Add max memory benchmark (#2061)

tools/benchmark.py

@@ -17,6 +17,7 @@ import http_server
 import throughput_benchmark
 from http_benchmark import http_benchmark
 import prebuilt
+import subprocess
 
 # The list of the tuples of the benchmark name and arguments
 exec_time_benchmarks = [
@@ -154,6 +155,21 @@ def run_syscall_count_benchmark(deno_path):
     return syscall_count_map
 
 
+# Takes the output from "/usr/bin/time -v" as input and extracts the 'maximum
+# resident set size' and returns it in bytes.
+def find_max_mem_in_bytes(time_v_output):
+    for line in time_v_output.split('\n'):
+        if 'maximum resident set size (kbytes)' in line.lower():
+            _, value = line.split(': ')
+            return int(value) * 1024
+
+
+def run_max_mem_benchmark(deno_path):
+    cmd = ["/usr/bin/time", "-v", deno_path, "--reload", "tests/002_hello.ts"]
+    out = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+    return find_max_mem_in_bytes(out)
+
+
 def main(argv):
     if len(argv) == 2:
         build_dir = sys.argv[1]
@@ -222,6 +238,7 @@ def main(argv):
         # Thread count test, only on linux
         new_data["thread_count"] = run_thread_count_benchmark(deno_path)
         new_data["syscall_count"] = run_syscall_count_benchmark(deno_path)
+        new_data["max_memory"] = run_max_mem_benchmark(deno_path)
 
     all_data.append(new_data)
     write_json(all_data_file, all_data)

tools/benchmark_test.py

@@ -1,7 +1,9 @@
+#!/usr/bin/env python
 # Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
 import sys
 import os
 import benchmark
+from util import build_path, executable_suffix
 
 
 def strace_parse_test():
@@ -20,6 +22,12 @@ def strace_parse_test():
         assert summary["total"]["calls"] == 704
 
 
+def max_mem_parse_test():
+    with open(os.path.join(sys.path[0], "testdata/time.out"), "r") as f:
+        data = f.read()
+        assert benchmark.find_max_mem_in_bytes(data) == 120380 * 1024
+
+
 def binary_size_test(build_dir):
     binary_size_dict = benchmark.get_binary_sizes(build_dir)
     assert binary_size_dict["deno"] > 0
@@ -43,6 +51,24 @@ def syscall_count_test(deno_path):
 def benchmark_test(build_dir, deno_path):
     strace_parse_test()
     binary_size_test(build_dir)
+    max_mem_parse_test()
     if "linux" in sys.platform:
         thread_count_test(deno_path)
         syscall_count_test(deno_path)
+
+
+# This test assumes tools/http_server.py is running in the background.
+def main():
+    if len(sys.argv) == 2:
+        build_dir = sys.argv[1]
+    elif len(sys.argv) == 1:
+        build_dir = build_path()
+    else:
+        print "Usage: tools/benchmark_test.py [build_dir]"
+        sys.exit(1)
+    deno_exe = os.path.join(build_dir, "deno" + executable_suffix)
+    benchmark_test(build_dir, deno_exe)
+
+
+if __name__ == '__main__':
+    main()

tools/testdata/time.out

@@ -0,0 +1,18 @@
+Hello
+    Command being timed: "./target/debug/deno tests/003_relative_import.ts"
+    User time (seconds): 2.43
+    System time (seconds): 0.05
+    Percent of CPU this job got: 156%
+    Elapsed (wall clock) time (h:mm:ss or m:ss): 0:01.59
+    Average shared text size (kbytes): 0
+    Average unshared data size (kbytes): 0
+    Average stack size (kbytes): 0
+    Average total size (kbytes): 0
+    Maximum resident set size (kbytes): 120380
+    Average resident set size (kbytes): 0
+    Major (requiring I/O) page faults: 0
+    Minor (reclaiming a frame) page faults: 41452
+    Voluntary context switches: 75
+    Involuntary context switches: 42
+    Swaps: 0
+    File system inputs: 0

website/app.js

@@ -50,6 +50,10 @@ export function createMaxLatencyColumns(data) {
   return createColumns(data, "max_latency");
 }
 
+export function createMaxMemoryColumns(data) {
+  return createColumns(data, "max_memory");
+}
+
 export function createBinarySizeColumns(data) {
   const propName = "binary_size";
   const binarySizeNames = Object.keys(data[data.length - 1][propName]);
@@ -203,6 +207,7 @@ export async function drawChartsFromBenchmarkData(dataUrl) {
   const throughputColumns = createThroughputColumns(data);
   const reqPerSecColumns = createReqPerSecColumns(data);
   const maxLatencyColumns = createMaxLatencyColumns(data);
+  const maxMemoryColumns = createMaxMemoryColumns(data);
   const binarySizeColumns = createBinarySizeColumns(data);
   const threadCountColumns = createThreadCountColumns(data);
   const syscallCountColumns = createSyscallCountColumns(data);
@@ -231,6 +236,7 @@ export async function drawChartsFromBenchmarkData(dataUrl) {
   gen("#throughput-chart", throughputColumns, "seconds", logScale);
   gen("#req-per-sec-chart", reqPerSecColumns, "1000 req/sec", formatReqSec);
   gen("#max-latency-chart", maxLatencyColumns, "milliseconds", logScale);
+  gen("#max-memory-chart", maxMemoryColumns, "megabytes", formatMB);
   gen("#binary-size-chart", binarySizeColumns, "megabytes", formatMB);
   gen("#thread-count-chart", threadCountColumns, "threads");
   gen("#syscall-count-chart", syscallCountColumns, "syscalls");

website/benchmarks.html

@@ -125,6 +125,14 @@
 
       <div id="max-latency-chart"></div>
 
+      <h3 id="max-memory">Max Memory Usage <a href="#max-memory">#</a></h3>
+
+      <p>
+        Max memory usage during execution. Smaller is better.
+      </p>
+
+      <div id="max-memory-chart"></div>
+
       <h3 id="size">Executable size <a href="#size">#</a></h3>
       <p>deno ships only a single binary. We track its size here.</p>
       <div id="binary-size-chart"></div>