Add thread count benchmark (#811)

.travis.yml

@@ -3,6 +3,7 @@ matrix:
   include:
   - os: linux
     env: BENCHMARK=1
+    sudo: required
   - os: osx
 env:
   global:

tools/benchmark.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+# Copyright 2018 the Deno authors. All rights reserved. MIT license.
 # Performs benchmark and append data to //website/data.json.
 # If //website/data.json doesn't exist, this script tries to import it from gh-pages branch.
 # To view the results locally run ./tools/http_server.py and visit
@@ -10,6 +11,7 @@ import json
 import time
 import shutil
 from util import run, run_output, root_path, build_path
+import tempfile
 
 # The list of the tuples of the benchmark name and arguments
 benchmarks = [("hello", ["tests/002_hello.ts", "--reload"]),
@@ -43,6 +45,23 @@ def import_data_from_gh_pages():
         write_json(data_file, [])  # writes empty json data
 
 
+# run strace with test_args and record times a syscall record appears in out file
+# based on syscall_line_matcher. Should be reusable
+def count_strace_syscall(syscall_name, syscall_line_matcher, test_args):
+    f = tempfile.NamedTemporaryFile()
+    run(["strace", "-f", "-o", f.name, "-e", "trace=" + syscall_name] +
+        test_args)
+    return len(filter(syscall_line_matcher, f))
+
+
+def run_thread_count_benchmark(deno_path):
+    thread_count_map = {}
+    thread_count_map["set_timeout"] = count_strace_syscall(
+        "clone", lambda line: "clone(" in line,
+        [deno_path, "tests/004_set_timeout.ts", "--reload"]) + 1
+    return thread_count_map
+
+
 def main(argv):
     if len(argv) == 2:
         build_dir = sys.argv[1]
@@ -67,6 +86,7 @@ def main(argv):
         "created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ"),
         "sha1": sha1,
         "binary_size": os.path.getsize(deno_path),
+        "thread_count": {},
         "benchmark": {}
     }
     for [[name, _], data] in zip(benchmarks, benchmark_data["results"]):
@@ -78,6 +98,11 @@ def main(argv):
             "min": data["min"],
             "max": data["max"]
         }
+
+    if "linux" in sys.platform:
+        # Thread count test, only on linux
+        new_data["thread_count"] = run_thread_count_benchmark(deno_path)
+
     all_data.append(new_data)
     write_json(data_file, all_data)
 

tools/benchmark_test.py

@@ -0,0 +1,10 @@
+import sys
+import os
+from benchmark import run_thread_count_benchmark
+
+
+def benchmark_test(deno_path):
+    if "linux" in sys.platform:
+        thread_count_dict = run_thread_count_benchmark(deno_path)
+        assert "set_timeout" in thread_count_dict
+        assert thread_count_dict["set_timeout"] > 1

tools/test.py

@@ -8,6 +8,7 @@ from setup_test import setup_test
 from util import build_path, enable_ansi_colors, executable_suffix, run
 from unit_tests import unit_tests
 from util_test import util_test
+from benchmark_test import benchmark_test
 import subprocess
 import http_server
 
@@ -32,9 +33,15 @@ def main(argv):
 
     http_server.spawn()
 
+    deno_exe = os.path.join(build_dir, "deno" + executable_suffix)
+    check_exists(deno_exe)
+    deno_ns_exe = os.path.join(build_dir, "deno_ns" + executable_suffix)
+    check_exists(deno_ns_exe)
+
     # Internal tools testing
     setup_test()
     util_test()
+    benchmark_test(deno_exe)
 
     test_cc = os.path.join(build_dir, "test_cc" + executable_suffix)
     check_exists(test_cc)
@@ -44,15 +51,9 @@ def main(argv):
     check_exists(test_rs)
     run([test_rs])
 
-    deno_exe = os.path.join(build_dir, "deno" + executable_suffix)
-    check_exists(deno_exe)
     unit_tests(deno_exe)
 
-    check_exists(deno_exe)
     check_output_test(deno_exe)
-
-    deno_ns_exe = os.path.join(build_dir, "deno_ns" + executable_suffix)
-    check_exists(deno_ns_exe)
     check_output_test(deno_ns_exe)
 
 

website/app.js

@@ -20,6 +20,20 @@ export function createBinarySizeColumns(data) {
   return [["binary_size", ...data.map(d => d.binary_size || 0)]];
 }
 
+const threadCountNames = ["set_timeout"];
+export function createThreadCountColumns(data) {
+  return threadCountNames.map(name => [
+    name,
+    ...data.map(d => {
+      const threadCountData = d["thread_count"];
+      if (!threadCountData) {
+        return 0;
+      }
+      return threadCountData[name] || 0;
+    })
+  ]);
+}
+
 export function createSha1List(data) {
   return data.map(d => d.sha1);
 }
@@ -40,6 +54,7 @@ export async function main() {
 
   const execTimeColumns = createExecTimeColumns(data);
   const binarySizeColumns = createBinarySizeColumns(data);
+  const threadCountColumns = createThreadCountColumns(data);
   const sha1List = createSha1List(data);
 
   c3.generate({
@@ -68,4 +83,15 @@ export async function main() {
       }
     }
   });
+
+  c3.generate({
+    bindto: "#thread-count-chart",
+    data: { columns: threadCountColumns },
+    axis: {
+      x: {
+        type: "category",
+        categories: sha1List
+      }
+    }
+  });
 }

website/app_test.js

@@ -1,9 +1,10 @@
 // Copyright 2018 the Deno authors. All rights reserved. MIT license.
 
-import { test, testPerm, assertEqual } from "../js/test_util.ts";
+import { test, assertEqual } from "../js/test_util.ts";
 import {
   createBinarySizeColumns,
   createExecTimeColumns,
+  createThreadCountColumns,
   createSha1List,
   formatBytes
 } from "./app.js";
@@ -20,6 +21,9 @@ const regularData = [
       relative_import: {
         mean: 0.06
       }
+    },
+    thread_count: {
+      set_timeout: 4
     }
   },
   {
@@ -33,6 +37,9 @@ const regularData = [
       relative_import: {
         mean: 0.065
       }
+    },
+    thread_count: {
+      set_timeout: 5
     }
   }
 ];
@@ -44,7 +51,8 @@ const irregularData = [
     benchmark: {
       hello: {},
       relative_import: {}
-    }
+    },
+    thread_count: {}
   },
   {
     created_at: "2018-02-01T01:00:00Z",
@@ -76,6 +84,16 @@ test(function createBinarySizeColumnsIrregularData() {
   assertEqual(columns, [["binary_size", 0, 0]]);
 });
 
+test(function createThreadCountColumnsRegularData() {
+  const columns = createThreadCountColumns(regularData);
+  assertEqual(columns, [["set_timeout", 4, 5]]);
+});
+
+test(function createThreadCountColumnsIrregularData() {
+  const columns = createThreadCountColumns(irregularData);
+  assertEqual(columns, [["set_timeout", 0, 0]]);
+});
+
 test(function createSha1ListRegularData() {
   const sha1List = createSha1List(regularData);
   assertEqual(sha1List, ["abcdef", "012345"]);

website/index.html

@@ -9,6 +9,8 @@
   <div id="exec-time-chart"></div>
   <h2>Binary size chart</h2>
   <div id="binary-size-chart"></div>
+  <h2>Thread count chart</h2>
+  <div id="thread-count-chart"></div>
   <script src="https://unpkg.com/d3@5.7.0/dist/d3.min.js"></script>
   <script src="https://unpkg.com/c3@0.6.7/c3.min.js"></script>
   <script type="module">