diff --git a/.github/workflows/bench_cron.yml b/.github/workflows/bench_cron.yml new file mode 100644 index 0000000000..4439b6d8f2 --- /dev/null +++ b/.github/workflows/bench_cron.yml @@ -0,0 +1,43 @@ +name: bench + +on: + # Runs at minute 9 past hour 0, 6, 12, and 18. + schedule: + - cron: 9 0,6,12,18 * * * + workflow_dispatch: + +jobs: + bench: + name: bench / ${{ matrix.os }} / ${{ matrix.deno-version }} + if: github.repository == 'denoland/deno' + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-20.04-xl] + + env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: full + CI: true + GOOGLE_SVC_KEY: ${{ secrets.GOOGLE_SVC_KEY }} + + steps: + - name: Clone repository + uses: actions/checkout@v2 + with: + submodules: true + persist-credentials: false + + - uses: dtolnay/rust-toolchain@stable + + - name: Build release + run: cargo build --release --locked --all-targets + + - name: Worker info + run: | + cat /proc/cpuinfo + cat /proc/meminfo + + - name: Run and Post benchmarks + run: cargo bench --locked \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 53f4a5ea1b..c0aa18bb92 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -603,4 +603,4 @@ jobs: - name: Upload canary version file to dl.deno.land run: | echo ${{ github.sha }} > canary-latest.txt - gsutil -h "Cache-Control: no-cache" cp canary-latest.txt gs://dl.deno.land/canary-latest.txt + gsutil -h "Cache-Control: no-cache" cp canary-latest.txt gs://dl.deno.land/canary-latest.txt \ No newline at end of file diff --git a/.gitignore b/.gitignore index 31a0049398..7c43145088 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ *.orig *.pyc *.swp +.env /.cargo_home/ /.idea/ diff --git a/Cargo.lock b/Cargo.lock index 5165c654e2..0910acd14b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -354,6 +354,18 @@ dependencies = [ "alloc-stdlib", ] +[[package]] +name = "bstr" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223" +dependencies = [ + "lazy_static", + "memchr", + "regex-automata", + "serde", +] + [[package]] name = "bumpalo" version = "3.10.0" @@ -617,6 +629,28 @@ dependencies = [ "typenum", ] +[[package]] +name = "csv" +version = "1.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22813a6dc45b335f9bade10bf7271dc477e81113e89eb251a0bc2a8a81c536e1" +dependencies = [ + "bstr", + "csv-core", + "itoa 0.4.8", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90" +dependencies = [ + "memchr", +] + [[package]] name = "ctor" version = "0.1.22" @@ -735,6 +769,7 @@ dependencies = [ "clap", "clap_complete", "clap_complete_fig", + "csv", "data-url", "deno_ast", "deno_bench_util", @@ -747,6 +782,7 @@ dependencies = [ "deno_snapshots", "deno_task_shell", "dissimilar", + "dotenv", "dprint-plugin-json", "dprint-plugin-markdown", "dprint-plugin-typescript", @@ -756,19 +792,20 @@ dependencies = [ "fancy-regex", "flaky_test", "fwdansi", + "google-storage1", "http", "import_map", "indexmap", "jsonc-parser", "libc", - "log", + "log 0.4.17", "mitata", "nix", "node_resolver", "notify", "once_cell", "os_pipe", - "percent-encoding", + "percent-encoding 2.1.0", "pin-project", "pretty_assertions", "rand", @@ -827,7 +864,7 @@ dependencies = [ "swc_ecma_utils", "swc_ecma_visit", "text_lines", - "url", + "url 2.2.2", ] [[package]] @@ -836,6 +873,7 @@ version = "0.52.0" dependencies = [ "bencher", "deno_core", + "once_cell", "tokio", ] @@ -866,7 +904,7 @@ dependencies = [ "futures", "indexmap", "libc", - "log", + "log 0.4.17", "once_cell", "parking_lot 0.12.1", "pin-project", @@ -875,7 +913,7 @@ dependencies = [ "serde_v8", "sourcemap", "tokio", - "url", + "url 2.2.2", "v8", ] @@ -990,7 +1028,7 @@ dependencies = [ "serde_json", "sourcemap", "termcolor", - "url", + "url 2.2.2", ] [[package]] @@ -1008,8 +1046,8 @@ dependencies = [ "flate2", "fly-accept-encoding", "hyper", - "mime", - "percent-encoding", + "mime 0.3.16", + "percent-encoding 2.1.0", "phf", "ring", "serde", @@ -1027,7 +1065,7 @@ dependencies = [ "deno_ast", "derive_more", "if_chain", - "log", + "log 0.4.17", "once_cell", "regex", "serde", @@ -1040,7 +1078,7 @@ version = "0.50.0" dependencies = [ "deno_core", "deno_tls", - "log", + "log 0.4.17", "serde", "socket2", "tokio", @@ -1088,7 +1126,7 @@ dependencies = [ "http", "hyper", "libc", - "log", + "log 0.4.17", "lzzzz", "netif", "nix", @@ -1141,7 +1179,7 @@ dependencies = [ "once_cell", "rustls", "rustls-native-certs", - "rustls-pemfile", + "rustls-pemfile 1.0.0", "serde", "webpki", "webpki-roots", @@ -1297,6 +1335,12 @@ dependencies = [ "syn 0.15.44", ] +[[package]] +name = "dotenv" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" + [[package]] name = "dprint-core" version = "0.58.2" @@ -1458,7 +1502,7 @@ checksum = "0b2cf0344971ee6c64c31be0d530793fba457d322dfec2810c453d0ef228f9c3" dependencies = [ "atty", "humantime", - "log", + "log 0.4.17", "regex", "termcolor", ] @@ -1521,7 +1565,7 @@ dependencies = [ "sha2", "thiserror", "tokio", - "url", + "url 2.2.2", ] [[package]] @@ -1654,7 +1698,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5fc25a87fa4fd2094bffb06925852034d90a17f0d1e05197d4956d3555752191" dependencies = [ "matches", - "percent-encoding", + "percent-encoding 2.1.0", ] [[package]] @@ -1850,6 +1894,23 @@ dependencies = [ "web-sys", ] +[[package]] +name = "google-storage1" +version = "3.1.0+20220228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "838c83adf6e1f0e35fa17b343e32762867f8ee8ea1a63e97c5ef5efda751bab9" +dependencies = [ + "hyper", + "hyper-rustls", + "itertools", + "mime 0.2.6", + "serde", + "serde_derive", + "serde_json", + "url 1.7.2", + "yup-oauth2", +] + [[package]] name = "gpu-alloc" version = "0.5.3" @@ -2001,7 +2062,7 @@ checksum = "31f4c6746584866f0feabcc69893c5b51beef3831656a968ed7ae254cdc4fd03" dependencies = [ "bytes", "fnv", - "itoa", + "itoa 1.0.2", ] [[package]] @@ -2048,7 +2109,7 @@ dependencies = [ "http-body", "httparse", "httpdate", - "itoa", + "itoa 1.0.2", "pin-project-lite", "socket2", "tokio", @@ -2065,7 +2126,9 @@ checksum = "d87c48c02e0dc5e3b849a2041db3029fd066650f8f717c07bf8ed78ccb895cac" dependencies = [ "http", "hyper", + "log 0.4.17", "rustls", + "rustls-native-certs", "tokio", "tokio-rustls", ] @@ -2076,6 +2139,17 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" +[[package]] +name = "idna" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38f09e0f0b1fb55fdee1f17470ad800da77af5186a1a76c026b679358b7e844e" +dependencies = [ + "matches", + "unicode-bidi", + "unicode-normalization", +] + [[package]] name = "idna" version = "0.2.3" @@ -2100,10 +2174,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5247edf057fe57036112a1fec3864baa68052b52116760dbea4909115731272f" dependencies = [ "indexmap", - "log", + "log 0.4.17", "serde", "serde_json", - "url", + "url 2.2.2", ] [[package]] @@ -2208,6 +2282,12 @@ dependencies = [ "either", ] +[[package]] +name = "itoa" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" + [[package]] name = "itoa" version = "1.0.2" @@ -2438,6 +2518,15 @@ dependencies = [ "scopeguard", ] +[[package]] +name = "log" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e19e8d5c34a3e0e2223db8e060f9e8264aeeb5c5fc64a4ee9965c062211c024b" +dependencies = [ + "log 0.4.17", +] + [[package]] name = "log" version = "0.4.17" @@ -2467,7 +2556,7 @@ dependencies = [ "serde", "serde_json", "serde_repr", - "url", + "url 2.2.2", ] [[package]] @@ -2525,10 +2614,19 @@ dependencies = [ "block", "core-graphics-types", "foreign-types", - "log", + "log 0.4.17", "objc", ] +[[package]] +name = "mime" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba626b8a6de5da682e1caa06bdb42a335aee5a84db8e5046a3e8ab17ba0a3ae0" +dependencies = [ + "log 0.3.9", +] + [[package]] name = "mime" version = "0.3.16" @@ -2551,7 +2649,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "713d550d9b44d89174e066b7a6217ae06234c10cb47819a88290d2b353c31799" dependencies = [ "libc", - "log", + "log 0.4.17", "wasi 0.11.0+wasi-snapshot-preview1", "windows-sys 0.36.1", ] @@ -2576,7 +2674,7 @@ dependencies = [ "codespan-reporting", "hexf-parse", "indexmap", - "log", + "log 0.4.17", "num-traits", "rustc-hash", "serde", @@ -2893,6 +2991,12 @@ dependencies = [ "base64ct", ] +[[package]] +name = "percent-encoding" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31010dd2e1ac33d5b46a5b413495239882813e0369f8ed8a5e266f173602f831" + [[package]] name = "percent-encoding" version = "2.1.0" @@ -3258,6 +3362,12 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" + [[package]] name = "regex-syntax" version = "0.6.26" @@ -3305,12 +3415,12 @@ dependencies = [ "ipnet", "js-sys", "lazy_static", - "log", - "mime", - "percent-encoding", + "log 0.4.17", + "mime 0.3.16", + "percent-encoding 2.1.0", "pin-project-lite", "rustls", - "rustls-pemfile", + "rustls-pemfile 1.0.0", "serde", "serde_json", "serde_urlencoded", @@ -3318,7 +3428,7 @@ dependencies = [ "tokio-rustls", "tokio-util", "tower-service", - "url", + "url 2.2.2", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", @@ -3458,7 +3568,7 @@ version = "0.20.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5aab8ee6c7097ed6057f43c187a62418d0c05a4bd5f18b3571db50ee0f9ce033" dependencies = [ - "log", + "log 0.4.17", "ring", "sct", "webpki", @@ -3471,11 +3581,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0167bac7a9f490495f3c33013e7722b53cb087ecbe082fb0c6387c96f634ea50" dependencies = [ "openssl-probe", - "rustls-pemfile", + "rustls-pemfile 1.0.0", "schannel", "security-framework", ] +[[package]] +name = "rustls-pemfile" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ee86d63972a7c661d1536fefe8c3c8407321c3df668891286de28abcd087360" +dependencies = [ + "base64 0.13.0", +] + [[package]] name = "rustls-pemfile" version = "1.0.0" @@ -3496,7 +3615,7 @@ dependencies = [ "clipboard-win", "fd-lock", "libc", - "log", + "log 0.4.17", "memchr", "nix", "radix_trie", @@ -3565,6 +3684,12 @@ dependencies = [ "untrusted", ] +[[package]] +name = "seahash" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" + [[package]] name = "sec1" version = "0.3.0" @@ -3668,7 +3793,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b7ce2b32a1aed03c558dc61a5cd328f15aff2dbc17daad8fb8af04d2100e15c" dependencies = [ "indexmap", - "itoa", + "itoa 1.0.2", "ryu", "serde", ] @@ -3691,7 +3816,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" dependencies = [ "form_urlencoded", - "itoa", + "itoa 1.0.2", "ryu", "serde", ] @@ -3818,7 +3943,7 @@ dependencies = [ "rustc_version 0.2.3", "serde", "serde_json", - "url", + "url 2.2.2", ] [[package]] @@ -3985,7 +4110,7 @@ dependencies = [ "swc_visit", "tracing", "unicode-width", - "url", + "url 2.2.2", ] [[package]] @@ -4421,7 +4546,7 @@ dependencies = [ "pretty_assertions", "pty", "regex", - "rustls-pemfile", + "rustls-pemfile 1.0.0", "serde", "serde_json", "tokio", @@ -4490,6 +4615,7 @@ checksum = "c2702e08a7a860f005826c6815dcac101b19b5eb330c27fe4a5928fec1d20ddd" dependencies = [ "libc", "num_threads", + "serde", ] [[package]] @@ -4567,7 +4693,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e80b39df6afcc12cdf752398ade96a6b9e99c903dfdc36e53ad10b9c366bca72" dependencies = [ "futures-util", - "log", + "log 0.4.17", "rustls", "tokio", "tokio-rustls", @@ -4631,7 +4757,7 @@ dependencies = [ "dashmap", "futures", "httparse", - "log", + "log 0.4.17", "lsp-types", "memchr", "serde", @@ -4702,7 +4828,7 @@ dependencies = [ "futures-channel", "futures-util", "lazy_static", - "log", + "log 0.4.17", "radix_trie", "rand", "thiserror", @@ -4724,17 +4850,17 @@ dependencies = [ "futures-channel", "futures-io", "futures-util", - "idna", + "idna 0.2.3", "ipnet", "lazy_static", - "log", + "log 0.4.17", "rand", "serde", "smallvec", "thiserror", "tinyvec", "tokio", - "url", + "url 2.2.2", ] [[package]] @@ -4747,7 +4873,7 @@ dependencies = [ "futures-util", "ipconfig", "lazy_static", - "log", + "log 0.4.17", "lru-cache", "parking_lot 0.12.1", "resolv-conf", @@ -4771,7 +4897,7 @@ dependencies = [ "env_logger", "futures-executor", "futures-util", - "log", + "log 0.4.17", "serde", "thiserror", "time 0.3.9", @@ -4798,12 +4924,12 @@ dependencies = [ "bytes", "http", "httparse", - "log", + "log 0.4.17", "rand", "rustls", "sha-1 0.9.8", "thiserror", - "url", + "url 2.2.2", "utf-8", "webpki", ] @@ -4957,6 +5083,17 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" +[[package]] +name = "url" +version = "1.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd4e7c0d531266369519a4aa4f399d748bd37043b00bde1e4ff1f60a120b355a" +dependencies = [ + "idna 0.1.5", + "matches", + "percent-encoding 1.0.1", +] + [[package]] name = "url" version = "2.2.2" @@ -4964,9 +5101,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a507c383b2d33b5fc35d1861e77e6b383d158b2da5e14fe51b83dfedf6fd578c" dependencies = [ "form_urlencoded", - "idna", + "idna 0.2.3", "matches", - "percent-encoding", + "percent-encoding 2.1.0", "serde", ] @@ -4980,7 +5117,7 @@ dependencies = [ "regex", "serde", "unic-ucd-ident", - "url", + "url 2.2.2", ] [[package]] @@ -5053,7 +5190,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" dependencies = [ - "log", + "log 0.4.17", "try-lock", ] @@ -5087,7 +5224,7 @@ checksum = "5491a68ab4500fa6b4d726bd67408630c3dbe9c4fe7bda16d5c82a1fd8c7340a" dependencies = [ "bumpalo", "lazy_static", - "log", + "log 0.4.17", "proc-macro2 1.0.39", "quote 1.0.18", "syn 1.0.96", @@ -5176,7 +5313,7 @@ dependencies = [ "codespan-reporting", "copyless", "fxhash", - "log", + "log 0.4.17", "naga", "parking_lot 0.11.2", "profiling", @@ -5210,7 +5347,7 @@ dependencies = [ "js-sys", "khronos-egl", "libloading", - "log", + "log 0.4.17", "metal", "naga", "objc", @@ -5410,6 +5547,32 @@ dependencies = [ "toml", ] +[[package]] +name = "yup-oauth2" +version = "6.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87d859e5b7fb10b5956b30eb1a6da981cdf0d3195fe2c6ff1840424522ca0d51" +dependencies = [ + "anyhow", + "async-trait", + "base64 0.13.0", + "futures", + "http", + "hyper", + "hyper-rustls", + "itertools", + "log 0.4.17", + "percent-encoding 2.1.0", + "rustls", + "rustls-pemfile 0.3.0", + "seahash", + "serde", + "serde_json", + "time 0.3.9", + "tokio", + "url 2.2.2", +] + [[package]] name = "zeroize" version = "1.5.5" diff --git a/bench_util/Cargo.toml b/bench_util/Cargo.toml index 3db751ebcb..de0472881a 100644 --- a/bench_util/Cargo.toml +++ b/bench_util/Cargo.toml @@ -16,6 +16,7 @@ path = "lib.rs" [dependencies] bencher = "0.1" deno_core = { version = "0.140.0", path = "../core" } +once_cell = "1.10.0" tokio = { version = "1.19", features = ["full"] } [[bench]] diff --git a/cli/Cargo.toml b/cli/Cargo.toml index eba22c3e97..d9b760ee55 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -100,8 +100,12 @@ fwdansi = "=1.1.0" winapi = { version = "=0.3.9", features = ["knownfolders", "mswsock", "objbase", "shlobj", "tlhelp32", "winbase", "winerror", "winsock2"] } [dev-dependencies] +csv = "1.1.6" deno_bench_util = { version = "0.52.0", path = "../bench_util" } +dotenv = "0.15.0" flaky_test = "=0.1.0" +google-storage1 = "3.1.0" +once_cell = "=1.12.0" os_pipe = "=1.0.1" pretty_assertions = "=1.2.1" test_util = { path = "../test_util" } diff --git a/cli/bench/README.md b/cli/bench/README.md new file mode 100644 index 0000000000..3117cff153 --- /dev/null +++ b/cli/bench/README.md @@ -0,0 +1,10 @@ +benchmark filtering: + +``` +cargo bench --bench deno_bench -- bundle +``` + +benchmark plots: + +new: https://denoland.grafana.net/d/vErC9VCnz/benchmarks?orgId=1 old: +deno.land/benchmarks diff --git a/cli/bench/http.rs b/cli/bench/http.rs index 7706429071..3646f472c9 100644 --- a/cli/bench/http.rs +++ b/cli/bench/http.rs @@ -44,7 +44,7 @@ pub fn benchmark( if name.starts_with("node") { // node res.insert( - name, + file_stem.to_string(), run( &["node", path, &port.to_string()], port, @@ -56,7 +56,7 @@ pub fn benchmark( } else { // deno run -A --unstable res.insert( - name, + file_stem.to_string(), run( &[ deno_exe, diff --git a/cli/bench/lsp.rs b/cli/bench/lsp.rs index 8397d23d7e..7aa35d0993 100644 --- a/cli/bench/lsp.rs +++ b/cli/bench/lsp.rs @@ -333,7 +333,7 @@ fn bench_startup_shutdown(deno_exe: &Path) -> Result { } /// Generate benchmarks for the LSP server. -pub fn benchmarks(deno_exe: &Path) -> Result, AnyError> { +pub fn benchmarks(deno_exe: &Path) -> Result, AnyError> { println!("-> Start benchmarking lsp"); let mut exec_times = HashMap::new(); @@ -343,7 +343,7 @@ pub fn benchmarks(deno_exe: &Path) -> Result, AnyError> { times.push(bench_startup_shutdown(deno_exe)?); } let mean = - (times.iter().sum::() / times.len() as u32).as_millis() as u64; + (times.iter().sum::() / times.len() as u32).as_millis() as i64; println!(" ({} runs, mean: {}ms)", times.len(), mean); exec_times.insert("startup_shutdown".to_string(), mean); @@ -353,7 +353,7 @@ pub fn benchmarks(deno_exe: &Path) -> Result, AnyError> { times.push(bench_big_file_edits(deno_exe)?); } let mean = - (times.iter().sum::() / times.len() as u32).as_millis() as u64; + (times.iter().sum::() / times.len() as u32).as_millis() as i64; println!(" ({} runs, mean: {}ms)", times.len(), mean); exec_times.insert("big_file_edits".to_string(), mean); @@ -363,7 +363,7 @@ pub fn benchmarks(deno_exe: &Path) -> Result, AnyError> { times.push(bench_find_replace(deno_exe)?); } let mean = - (times.iter().sum::() / times.len() as u32).as_millis() as u64; + (times.iter().sum::() / times.len() as u32).as_millis() as i64; println!(" ({} runs, mean: {}ms)", times.len(), mean); exec_times.insert("find_replace".to_string(), mean); @@ -373,7 +373,7 @@ pub fn benchmarks(deno_exe: &Path) -> Result, AnyError> { times.push(bench_code_lens(deno_exe)?); } let mean = - (times.iter().sum::() / times.len() as u32).as_millis() as u64; + (times.iter().sum::() / times.len() as u32).as_millis() as i64; println!(" ({} runs, mean: {}ms)", times.len(), mean); exec_times.insert("code_lens".to_string(), mean); diff --git a/cli/bench/main.rs b/cli/bench/main.rs index d6d5259f68..3e7cc61fc0 100644 --- a/cli/bench/main.rs +++ b/cli/bench/main.rs @@ -3,7 +3,6 @@ use deno_core::error::AnyError; use deno_core::serde_json; use deno_core::serde_json::Value; -use serde::Serialize; use std::collections::HashMap; use std::convert::From; use std::env; @@ -16,6 +15,7 @@ use std::time::SystemTime; mod http; mod lsp; +mod metrics; fn read_json(filename: &str) -> Result { let f = fs::File::open(filename)?; @@ -162,7 +162,7 @@ const RESULT_KEYS: &[&str] = fn run_exec_time( deno_exe: &Path, target_dir: &Path, -) -> Result>> { +) -> Result>> { let hyperfine_exe = test_util::prebuilt_tool_path("hyperfine"); let benchmark_file = target_dir.join("hyperfine_results.json"); @@ -203,7 +203,7 @@ fn run_exec_time( true, ); - let mut results = HashMap::>::new(); + let mut results = HashMap::>::new(); let hyperfine_results = read_json(benchmark_file)?; for ((name, _, _), data) in EXEC_TIME_BENCHMARKS.iter().zip( hyperfine_results @@ -220,7 +220,7 @@ fn run_exec_time( data .into_iter() .filter(|(key, _)| RESULT_KEYS.contains(&key.as_str())) - .map(|(key, val)| (key, val.as_f64().unwrap())) + .map(|(key, val)| (key, val.as_f64().unwrap() as i64)) .collect(), ); } @@ -228,7 +228,7 @@ fn run_exec_time( Ok(results) } -fn rlib_size(target_dir: &std::path::Path, prefix: &str) -> u64 { +fn rlib_size(target_dir: &std::path::Path, prefix: &str) -> i64 { let mut size = 0; let mut seen = std::collections::HashSet::new(); for entry in std::fs::read_dir(target_dir.join("deps")).unwrap() { @@ -247,18 +247,18 @@ fn rlib_size(target_dir: &std::path::Path, prefix: &str) -> u64 { } } assert!(size > 0); - size + size as i64 } const BINARY_TARGET_FILES: &[&str] = &["CLI_SNAPSHOT.bin", "COMPILER_SNAPSHOT.bin"]; -fn get_binary_sizes(target_dir: &Path) -> Result> { - let mut sizes = HashMap::::new(); +fn get_binary_sizes(target_dir: &Path) -> Result> { + let mut sizes = HashMap::::new(); let mut mtimes = HashMap::::new(); sizes.insert( "deno".to_string(), - test_util::deno_exe_path().metadata()?.len(), + test_util::deno_exe_path().metadata()?.len() as i64, ); // add up size for everything in target/release/deps/libswc* @@ -294,7 +294,7 @@ fn get_binary_sizes(target_dir: &Path) -> Result> { } mtimes.insert(filename.clone(), file_mtime); - sizes.insert(filename, meta.len()); + sizes.insert(filename, meta.len() as i64); } Ok(sizes) @@ -304,8 +304,8 @@ const BUNDLES: &[(&str, &str)] = &[ ("file_server", "./test_util/std/http/file_server.ts"), ("gist", "./test_util/std/examples/gist.ts"), ]; -fn bundle_benchmark(deno_exe: &Path) -> Result> { - let mut sizes = HashMap::::new(); +fn bundle_benchmark(deno_exe: &Path) -> Result> { + let mut sizes = HashMap::::new(); for (name, url) in BUNDLES { let path = format!("{}.bundle.js", name); @@ -325,74 +325,15 @@ fn bundle_benchmark(deno_exe: &Path) -> Result> { let file = PathBuf::from(path); assert!(file.is_file()); - sizes.insert(name.to_string(), file.metadata()?.len()); + sizes.insert(name.to_string(), file.metadata()?.len() as i64); let _ = fs::remove_file(file); } Ok(sizes) } -fn run_http(target_dir: &Path, new_data: &mut BenchResult) -> Result<()> { - let stats = http::benchmark(target_dir)?; - - new_data.req_per_sec = stats - .iter() - .map(|(name, result)| (name.clone(), result.requests)) - .collect(); - - new_data.max_latency = stats - .iter() - .map(|(name, result)| (name.clone(), result.latency)) - .collect(); - - Ok(()) -} - -fn run_strace_benchmarks( - deno_exe: &Path, - new_data: &mut BenchResult, -) -> Result<()> { - use std::io::Read; - - let mut thread_count = HashMap::::new(); - let mut syscall_count = HashMap::::new(); - - for (name, args, expected_exit_code) in EXEC_TIME_BENCHMARKS { - let mut file = secure_tempfile::NamedTempFile::new()?; - - let exit_status = Command::new("strace") - .args(&[ - "-c", - "-f", - "-o", - file.path().to_str().unwrap(), - deno_exe.to_str().unwrap(), - ]) - .args(args.iter()) - .stdout(Stdio::null()) - .spawn()? - .wait()?; - let expected_exit_code = expected_exit_code.unwrap_or(0); - assert_eq!(exit_status.code(), Some(expected_exit_code)); - - let mut output = String::new(); - file.as_file_mut().read_to_string(&mut output)?; - - let strace_result = test_util::parse_strace_output(&output); - let clone = strace_result.get("clone").map(|d| d.calls).unwrap_or(0) + 1; - let total = strace_result.get("total").unwrap().calls; - thread_count.insert(name.to_string(), clone); - syscall_count.insert(name.to_string(), total); - } - - new_data.thread_count = thread_count; - new_data.syscall_count = syscall_count; - - Ok(()) -} - -fn run_max_mem_benchmark(deno_exe: &Path) -> Result> { - let mut results = HashMap::::new(); +fn run_max_mem_benchmark(deno_exe: &Path) -> Result> { + let mut results = HashMap::::new(); for (name, args, return_code) in EXEC_TIME_BENCHMARKS { let proc = Command::new("time") @@ -408,7 +349,10 @@ fn run_max_mem_benchmark(deno_exe: &Path) -> Result> { } let out = String::from_utf8(proc_result.stderr)?; - results.insert(name.to_string(), test_util::parse_max_mem(&out).unwrap()); + results.insert( + name.to_string(), + test_util::parse_max_mem(&out).unwrap() as i64, + ); } Ok(results) @@ -429,7 +373,8 @@ fn cargo_deps() -> usize { count } -#[derive(Default, Serialize)] +// TODO(@littledivy): Remove this, denoland/benchmark_data is deprecated. +#[derive(Default, serde::Serialize)] struct BenchResult { created_at: String, sha1: String, @@ -437,27 +382,44 @@ struct BenchResult { // TODO(ry) The "benchmark" benchmark should actually be called "exec_time". // When this is changed, the historical data in gh-pages branch needs to be // changed too. - benchmark: HashMap>, - binary_size: HashMap, - bundle_size: HashMap, + benchmark: HashMap>, + binary_size: HashMap, + bundle_size: HashMap, cargo_deps: usize, - max_latency: HashMap, - max_memory: HashMap, - lsp_exec_time: HashMap, - req_per_sec: HashMap, - syscall_count: HashMap, - thread_count: HashMap, + max_latency: HashMap, + max_memory: HashMap, + lsp_exec_time: HashMap, + req_per_sec: HashMap, + syscall_count: HashMap, + thread_count: HashMap, } -/* - TODO(SyrupThinker) - Switch to the #[bench] attribute once - it is stabilized. - Before that the #[test] tests won't be run because - we replace the harness with our own runner here. -*/ -fn main() -> Result<()> { - if !env::args().any(|s| s == "--bench") { +#[tokio::main] +async fn main() -> Result<()> { + let mut args = env::args(); + + let mut benchmarks = vec![ + "bundle", + "exec_time", + "binary_size", + "cargo_deps", + "lsp", + "http", + "strace", + "mem_usage", + ]; + + let mut found_bench = false; + let filter = args.nth(1); + if let Some(filter) = filter { + if filter != "--bench" { + benchmarks.retain(|s| s == &filter); + } else { + found_bench = true; + } + } + + if !found_bench && !args.any(|s| s == "--bench") { return Ok(()); } @@ -465,7 +427,6 @@ fn main() -> Result<()> { let target_dir = test_util::target_dir(); let deno_exe = test_util::deno_exe_path(); - env::set_current_dir(&test_util::root_path())?; let mut new_data = BenchResult { @@ -481,27 +442,107 @@ fn main() -> Result<()> { .0 .trim() .to_string(), - benchmark: run_exec_time(&deno_exe, &target_dir)?, - binary_size: get_binary_sizes(&target_dir)?, - bundle_size: bundle_benchmark(&deno_exe)?, - cargo_deps: cargo_deps(), - lsp_exec_time: lsp::benchmarks(&deno_exe)?, ..Default::default() }; - if cfg!(not(target_os = "windows")) { - run_http(&target_dir, &mut new_data)?; + let mut reporter = metrics::Reporter::new().await; + + if benchmarks.contains(&"bundle") { + let bundle_size = bundle_benchmark(&deno_exe)?; + reporter.write("bundle_size", &bundle_size); + new_data.bundle_size = bundle_size; } - if cfg!(target_os = "linux") { - run_strace_benchmarks(&deno_exe, &mut new_data)?; - new_data.max_memory = run_max_mem_benchmark(&deno_exe)?; + if benchmarks.contains(&"exec_time") { + let exec_times = run_exec_time(&deno_exe, &target_dir)?; + for (name, data) in exec_times.iter() { + reporter.write_one("exec_time", name, *data.get("mean").unwrap()); + } + new_data.benchmark = exec_times; } - println!("===== "); - serde_json::to_writer_pretty(std::io::stdout(), &new_data)?; - println!("\n===== "); + if benchmarks.contains(&"binary_size") { + let binary_sizes = get_binary_sizes(&target_dir)?; + reporter.write("binary_size", &binary_sizes); + new_data.binary_size = binary_sizes; + } + if benchmarks.contains(&"cargo_deps") { + let cargo_deps = cargo_deps(); + reporter.write_one("cargo_deps", "cargo_deps", cargo_deps as i64); + new_data.cargo_deps = cargo_deps; + } + + if benchmarks.contains(&"lsp") { + let lsp_exec_times = lsp::benchmarks(&deno_exe)?; + reporter.write("lsp_exec_time", &lsp_exec_times); + new_data.lsp_exec_time = lsp_exec_times; + } + + if benchmarks.contains(&"http") && cfg!(not(target_os = "windows")) { + let stats = http::benchmark(&target_dir)?; + let req_per_sec = stats + .iter() + .map(|(name, result)| (name.clone(), result.requests as i64)) + .collect(); + reporter.write("req_per_sec", &req_per_sec); + new_data.req_per_sec = req_per_sec; + let max_latency = stats + .iter() + .map(|(name, result)| (name.clone(), result.latency as i64)) + .collect(); + + reporter.write("max_latency", &max_latency); + new_data.max_latency = max_latency; + } + + if cfg!(target_os = "linux") && benchmarks.contains(&"strace") { + use std::io::Read; + + let mut thread_count = HashMap::::new(); + let mut syscall_count = HashMap::::new(); + + for (name, args, expected_exit_code) in EXEC_TIME_BENCHMARKS { + let mut file = secure_tempfile::NamedTempFile::new()?; + + let exit_status = Command::new("strace") + .args(&[ + "-c", + "-f", + "-o", + file.path().to_str().unwrap(), + deno_exe.to_str().unwrap(), + ]) + .args(args.iter()) + .stdout(Stdio::null()) + .spawn()? + .wait()?; + let expected_exit_code = expected_exit_code.unwrap_or(0); + assert_eq!(exit_status.code(), Some(expected_exit_code)); + + let mut output = String::new(); + file.as_file_mut().read_to_string(&mut output)?; + + let strace_result = test_util::parse_strace_output(&output); + let clone = strace_result.get("clone").map(|d| d.calls).unwrap_or(0) + 1; + let total = strace_result.get("total").unwrap().calls; + thread_count.insert(name.to_string(), clone as i64); + syscall_count.insert(name.to_string(), total as i64); + } + + reporter.write("thread_count", &thread_count); + new_data.thread_count = thread_count; + reporter.write("syscall_count", &syscall_count); + new_data.syscall_count = syscall_count; + } + + if benchmarks.contains(&"mem_usage") { + let max_memory = run_max_mem_benchmark(&deno_exe)?; + reporter.write("max_memory", &max_memory); + new_data.max_memory = max_memory; + } + + reporter.submit().await; if let Some(filename) = target_dir.join("bench.json").to_str() { write_json(filename, &serde_json::to_value(&new_data)?)?; } else { diff --git a/cli/bench/metrics.rs b/cli/bench/metrics.rs new file mode 100644 index 0000000000..c49ca90208 --- /dev/null +++ b/cli/bench/metrics.rs @@ -0,0 +1,99 @@ +// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. + +use google_storage1::api::Object; +use google_storage1::hyper; +use google_storage1::hyper_rustls; +use google_storage1::oauth2; +use google_storage1::Storage; +use once_cell::sync::Lazy; +use std::collections::HashMap; +use std::io::Cursor; + +static GIT_HASH: Lazy = Lazy::new(|| { + test_util::run_collect(&["git", "rev-parse", "HEAD"], None, None, None, true) + .0 + .trim() + .to_string() +}); + +#[derive(serde::Serialize)] +struct Metric { + name: String, + value: i64, + sha1: String, + #[serde(rename = "type")] + type_: String, + time: i64, +} + +pub struct Reporter { + wtr: csv::Writer>, + gcloud_client: Option, +} + +impl Reporter { + pub async fn new() -> Self { + dotenv::dotenv().ok(); + let gcloud_client = + match std::env::var("CI").map(|_| std::env::var("GOOGLE_SVC_KEY")) { + Ok(Ok(key_str)) => { + let secret = oauth2::parse_service_account_key(key_str) + .expect("Failed to load service account key"); + let auth = + oauth2::authenticator::ServiceAccountAuthenticator::builder(secret) + .build() + .await + .unwrap(); + let client = hyper::Client::builder().build( + hyper_rustls::HttpsConnectorBuilder::new() + .with_native_roots() + .https_or_http() + .enable_http1() + .enable_http2() + .build(), + ); + Some(Storage::new(client, auth)) + } + _ => None, + }; + Self { + wtr: csv::Writer::from_writer(vec![]), + gcloud_client, + } + } + + pub fn write_one(&mut self, type_: &str, name: &str, value: i64) { + self + .wtr + .serialize(Metric { + name: name.to_string(), + type_: type_.to_string(), + value, + sha1: GIT_HASH.clone(), + time: chrono::Utc::now().timestamp_millis(), + }) + .unwrap(); + } + + pub fn write(&mut self, type_: &str, hashmap: &HashMap) { + for (name, value) in hashmap { + self.write_one(type_, name, *value); + } + } + + pub async fn submit(mut self) { + self.wtr.flush().unwrap(); + if let Some(client) = self.gcloud_client.take() { + let mut reader = Cursor::new(self.wtr.into_inner().unwrap()); + let object: Object = Object::default(); + client + .objects() + .insert(object, "deno_benchmark_data") + .name(&format!("{}.csv", *GIT_HASH)) + .param("uploadType", "multipart") + .upload(&mut reader, "text/csv".parse().unwrap()) + .await + .unwrap(); + } + } +}