From 0c463582206881b6461742633a67f51632db614e Mon Sep 17 00:00:00 2001 From: Dmitry Sharshakov Date: Tue, 16 Apr 2019 20:57:05 +0300 Subject: [PATCH] Add max memory benchmark (#2061) --- tools/benchmark.py | 17 +++++++++++++++++ tools/benchmark_test.py | 26 ++++++++++++++++++++++++++ tools/testdata/time.out | 18 ++++++++++++++++++ website/app.js | 6 ++++++ website/benchmarks.html | 8 ++++++++ 5 files changed, 75 insertions(+) mode change 100644 => 100755 tools/benchmark_test.py create mode 100644 tools/testdata/time.out diff --git a/tools/benchmark.py b/tools/benchmark.py index 2e3ac26e4b..43476c50b0 100755 --- a/tools/benchmark.py +++ b/tools/benchmark.py @@ -17,6 +17,7 @@ import http_server import throughput_benchmark from http_benchmark import http_benchmark import prebuilt +import subprocess # The list of the tuples of the benchmark name and arguments exec_time_benchmarks = [ @@ -154,6 +155,21 @@ def run_syscall_count_benchmark(deno_path): return syscall_count_map +# Takes the output from "/usr/bin/time -v" as input and extracts the 'maximum +# resident set size' and returns it in bytes. +def find_max_mem_in_bytes(time_v_output): + for line in time_v_output.split('\n'): + if 'maximum resident set size (kbytes)' in line.lower(): + _, value = line.split(': ') + return int(value) * 1024 + + +def run_max_mem_benchmark(deno_path): + cmd = ["/usr/bin/time", "-v", deno_path, "--reload", "tests/002_hello.ts"] + out = subprocess.check_output(cmd, stderr=subprocess.STDOUT) + return find_max_mem_in_bytes(out) + + def main(argv): if len(argv) == 2: build_dir = sys.argv[1] @@ -222,6 +238,7 @@ def main(argv): # Thread count test, only on linux new_data["thread_count"] = run_thread_count_benchmark(deno_path) new_data["syscall_count"] = run_syscall_count_benchmark(deno_path) + new_data["max_memory"] = run_max_mem_benchmark(deno_path) all_data.append(new_data) write_json(all_data_file, all_data) diff --git a/tools/benchmark_test.py b/tools/benchmark_test.py old mode 100644 new mode 100755 index b2c4aa9a6f..bc3ce8a71d --- a/tools/benchmark_test.py +++ b/tools/benchmark_test.py @@ -1,7 +1,9 @@ +#!/usr/bin/env python # Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. import sys import os import benchmark +from util import build_path, executable_suffix def strace_parse_test(): @@ -20,6 +22,12 @@ def strace_parse_test(): assert summary["total"]["calls"] == 704 +def max_mem_parse_test(): + with open(os.path.join(sys.path[0], "testdata/time.out"), "r") as f: + data = f.read() + assert benchmark.find_max_mem_in_bytes(data) == 120380 * 1024 + + def binary_size_test(build_dir): binary_size_dict = benchmark.get_binary_sizes(build_dir) assert binary_size_dict["deno"] > 0 @@ -43,6 +51,24 @@ def syscall_count_test(deno_path): def benchmark_test(build_dir, deno_path): strace_parse_test() binary_size_test(build_dir) + max_mem_parse_test() if "linux" in sys.platform: thread_count_test(deno_path) syscall_count_test(deno_path) + + +# This test assumes tools/http_server.py is running in the background. +def main(): + if len(sys.argv) == 2: + build_dir = sys.argv[1] + elif len(sys.argv) == 1: + build_dir = build_path() + else: + print "Usage: tools/benchmark_test.py [build_dir]" + sys.exit(1) + deno_exe = os.path.join(build_dir, "deno" + executable_suffix) + benchmark_test(build_dir, deno_exe) + + +if __name__ == '__main__': + main() diff --git a/tools/testdata/time.out b/tools/testdata/time.out new file mode 100644 index 0000000000..3ff409bd71 --- /dev/null +++ b/tools/testdata/time.out @@ -0,0 +1,18 @@ +Hello + Command being timed: "./target/debug/deno tests/003_relative_import.ts" + User time (seconds): 2.43 + System time (seconds): 0.05 + Percent of CPU this job got: 156% + Elapsed (wall clock) time (h:mm:ss or m:ss): 0:01.59 + Average shared text size (kbytes): 0 + Average unshared data size (kbytes): 0 + Average stack size (kbytes): 0 + Average total size (kbytes): 0 + Maximum resident set size (kbytes): 120380 + Average resident set size (kbytes): 0 + Major (requiring I/O) page faults: 0 + Minor (reclaiming a frame) page faults: 41452 + Voluntary context switches: 75 + Involuntary context switches: 42 + Swaps: 0 + File system inputs: 0 \ No newline at end of file diff --git a/website/app.js b/website/app.js index c012dfb4a8..7663e41f02 100644 --- a/website/app.js +++ b/website/app.js @@ -50,6 +50,10 @@ export function createMaxLatencyColumns(data) { return createColumns(data, "max_latency"); } +export function createMaxMemoryColumns(data) { + return createColumns(data, "max_memory"); +} + export function createBinarySizeColumns(data) { const propName = "binary_size"; const binarySizeNames = Object.keys(data[data.length - 1][propName]); @@ -203,6 +207,7 @@ export async function drawChartsFromBenchmarkData(dataUrl) { const throughputColumns = createThroughputColumns(data); const reqPerSecColumns = createReqPerSecColumns(data); const maxLatencyColumns = createMaxLatencyColumns(data); + const maxMemoryColumns = createMaxMemoryColumns(data); const binarySizeColumns = createBinarySizeColumns(data); const threadCountColumns = createThreadCountColumns(data); const syscallCountColumns = createSyscallCountColumns(data); @@ -231,6 +236,7 @@ export async function drawChartsFromBenchmarkData(dataUrl) { gen("#throughput-chart", throughputColumns, "seconds", logScale); gen("#req-per-sec-chart", reqPerSecColumns, "1000 req/sec", formatReqSec); gen("#max-latency-chart", maxLatencyColumns, "milliseconds", logScale); + gen("#max-memory-chart", maxMemoryColumns, "megabytes", formatMB); gen("#binary-size-chart", binarySizeColumns, "megabytes", formatMB); gen("#thread-count-chart", threadCountColumns, "threads"); gen("#syscall-count-chart", syscallCountColumns, "syscalls"); diff --git a/website/benchmarks.html b/website/benchmarks.html index 560b96c7cb..2432fedce8 100644 --- a/website/benchmarks.html +++ b/website/benchmarks.html @@ -125,6 +125,14 @@
+

Max Memory Usage #

+ +

+ Max memory usage during execution. Smaller is better. +

+ +
+

Executable size #

deno ships only a single binary. We track its size here.