From 3ad48bdb3e6372f55579808535ce2c17809178ff Mon Sep 17 00:00:00 2001 From: Yoshiya Hinosawa Date: Sat, 22 Sep 2018 01:06:59 +0900 Subject: [PATCH] Add benchmark tools (#777) --- .gitignore | 6 +++- .travis.yml | 15 +++++++- tools/benchmark.py | 85 ++++++++++++++++++++++++++++++++++++++++++++++ tools/format.py | 2 +- website/app.js | 26 ++++++++++++++ website/index.html | 14 ++++++++ 6 files changed, 145 insertions(+), 3 deletions(-) create mode 100755 tools/benchmark.py create mode 100644 website/app.js create mode 100644 website/index.html diff --git a/.gitignore b/.gitignore index 51eb81ca8b..bd850eb16d 100644 --- a/.gitignore +++ b/.gitignore @@ -6,7 +6,11 @@ Cargo.lock yarn.lock # npm deps node_modules +# editor files .idea - # RLS generated files /target/ +# export dir for gh-pages +/gh-pages +# temp benchmark data +/website/data.json diff --git a/.travis.yml b/.travis.yml index 27f50544eb..ae8d3c4b18 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,11 +37,24 @@ script: - bash -c "sleep 2100; pkill ninja" & - "./tools/build.py -j2" - "./tools/test.py $DENO_BUILD_PATH" +after_success: +# TODO: Include hyperfine in //third_party +- cargo install hyperfine +- ./tools/benchmark.py $DENO_BUILD_PATH +# export website assets for deployment +- cp website/*.* gh-pages/ before_deploy: | # gzip and name release to denote platform gzip -c $DENO_BUILD_PATH/deno > $DENO_BUILD_PATH/deno_${TRAVIS_OS_NAME}_x64.gz deploy: - provider: releases +- provider: pages + local-dir: gh-pages + skip-cleanup: true + github-token: $GITHUB_TOKEN + keep-history: true + on: + branch: master +- provider: releases api_key: secure: RIwv515oDcPAlEvt7uG8FeSFi6Tz6ODJUOXcFj6FYUPszxJ7Cg1kBLKln+fNW5OeOc52VsaZb/vPZ85skyEM6zk2ijL9FcSnnfNEm548w77iH6G0sk09NgBTy6KRXES6NZHD9jN1YTWYkT2G1NQi7mLqxR8a8pnWTbeK5HhtSWGsZPtXqf5iQbvnWsmKA0/w+FIgKupU0xe/qsYjh0eMLYpZDUWoKO0VxBKJ/ix5Uz91aJTjMIcHHij+ALg4pk+FkDotdyx39XB9b25KDxGuaI7NxWjSPzDxs/ZBHP6QYDLO0ti93ftvLAxRoBKPFoZrXqAu3KG9anr9WvxE40DO9OdV0VX2ZUatMUQm3DpSheN8ml2sErFqjIInqlpkdOVDYORz7FikPxkb9DKt+iuyFfxPRa4YWJv2tg8+Hy/nRCQw69OoKqrSNJ8KJDB3OjYbRBtdHz79RLJhTsGZla6RiyXfM7crR7CbFjbwdbW3Pt60t24fhvXQ0SwR0QTgzS/ieYEQHq/9GtSQA/Tn4kdIkyN6BdOMrQd/aUtgKmNdqbSlfmWGNyNZIxHdB+3RrTNT1tagkRI4UHEUfEujpIdYKwLjv0Xmi/VtTM+zOSkzHsIWGPfHBmIGnXfAItUHqivQYJ15E+dzg3T1CEbBxkDQtvwien9Fa8/pBsMkyovl8ps= file: "$DENO_BUILD_PATH/deno_${TRAVIS_OS_NAME}_x64.gz" diff --git a/tools/benchmark.py b/tools/benchmark.py new file mode 100755 index 0000000000..f0f9e4ac62 --- /dev/null +++ b/tools/benchmark.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python +# Performs benchmark and append data to //website/data.json. +# If //website/data.json doesn't exist, this script tries to import it from gh-pages branch. +# To view the results locally run ./tools/http_server.py and visit +# http://localhost:4545/website + +import os +import sys +import json +import time +import shutil +from util import run, run_output, root_path, build_path + +# The list of the tuples of the benchmark name and arguments +benchmarks = [("hello", ["tests/002_hello.ts", "--reload"]), + ("relative_import", ["tests/003_relative_import.ts", + "--reload"])] + +gh_pages_data_file = "gh-pages/data.json" +data_file = "website/data.json" + + +def read_json(filename): + with open(filename) as json_file: + return json.load(json_file) + + +def write_json(filename, data): + with open(filename, 'w') as outfile: + json.dump(data, outfile) + + +def import_data_from_gh_pages(): + if os.path.exists(data_file): + return + try: + run([ + "git", "clone", "--depth", "1", "-b", "gh-pages", + "https://github.com/denoland/deno.git", "gh-pages" + ]) + shutil.copy(gh_pages_data_file, data_file) + except: + write_json(data_file, []) # writes empty json data + + +def main(argv): + if len(argv) == 2: + build_dir = sys.argv[1] + elif len(argv) == 1: + build_dir = build_path() + else: + print "Usage: tools/benchmark.py [build_dir]" + sys.exit(1) + + deno_path = os.path.join(build_dir, "deno") + benchmark_file = os.path.join(build_dir, "benchmark.json") + + os.chdir(root_path) + import_data_from_gh_pages() + # TODO: Use hyperfine in //third_party + run(["hyperfine", "--export-json", benchmark_file, "--warmup", "3"] + + [deno_path + " " + " ".join(args) for [_, args] in benchmarks]) + all_data = read_json(data_file) + benchmark_data = read_json(benchmark_file) + sha1 = run_output(["git", "rev-parse", "HEAD"]).strip() + new_data = { + "created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ"), + "sha1": sha1, + "benchmark": {} + } + for [[name, _], data] in zip(benchmarks, benchmark_data["results"]): + new_data["benchmark"][name] = { + "mean": data["mean"], + "stddev": data["stddev"], + "user": data["user"], + "system": data["system"], + "min": data["min"], + "max": data["max"] + } + all_data.append(new_data) + write_json(data_file, all_data) + + +if __name__ == '__main__': + main(sys.argv) diff --git a/tools/format.py b/tools/format.py index 47774c44c1..b91e75635a 100755 --- a/tools/format.py +++ b/tools/format.py @@ -29,7 +29,7 @@ for fn in ["BUILD.gn", ".gn"] + find_exts("build_extra", ".gn", ".gni"): run(["yapf", "-i"] + glob("tools/*.py") + find_exts("build_extra", ".py")) run(["node", prettier, "--write"] + find_exts("js/", ".js", ".ts") + - find_exts("tests/", ".js", ".ts") + + find_exts("tests/", ".js", ".ts") + find_exts("website/", ".js", ".ts") + ["rollup.config.js", "tsconfig.json", "tslint.json"]) # Requires rustfmt 0.8.2 (flags were different in previous versions) diff --git a/website/app.js b/website/app.js new file mode 100644 index 0000000000..7a6566a61b --- /dev/null +++ b/website/app.js @@ -0,0 +1,26 @@ +const benchmarkNames = ["hello", "relative_import"]; + +(async () => { + const data = await (await fetch("./data.json")).json(); + + const benchmarkColumns = benchmarkNames.map(name => [ + name, + ...data.map(d => { + const benchmark = d.benchmark[name]; + return benchmark ? benchmark.mean : 0; + }) + ]); + + const sha1List = data.map(d => d.sha1); + + c3.generate({ + bindto: "#benchmark-chart", + data: { columns: benchmarkColumns }, + axis: { + x: { + type: "category", + categories: sha1List + } + } + }); +})(); diff --git a/website/index.html b/website/index.html new file mode 100644 index 0000000000..b15a9c9498 --- /dev/null +++ b/website/index.html @@ -0,0 +1,14 @@ + + + + deno benchmark + + + +
+ + + + + +