From aa546189be730163ee5370029e4dfdb3b454ab96 Mon Sep 17 00:00:00 2001 From: snek Date: Wed, 13 Nov 2024 11:38:46 +0100 Subject: [PATCH] feat: OpenTelemetry Tracing API and Exporting (#26710) Initial import of OTEL code supporting tracing. Metrics soon to come. Implements APIs for https://jsr.io/@deno/otel so that code using OpenTelemetry.js just works tm. There is still a lot of work to do with configuration and adding built-in tracing to core APIs, which will come in followup PRs. --------- Co-authored-by: Luca Casonato --- Cargo.lock | 324 ++++++++-- Cargo.toml | 6 + cli/args/mod.rs | 18 + cli/factory.rs | 1 + cli/standalone/binary.rs | 3 + cli/standalone/mod.rs | 1 + cli/tsc/dts/lib.deno.unstable.d.ts | 102 ++++ cli/worker.rs | 6 + ext/http/00_serve.ts | 141 +++-- runtime/Cargo.toml | 7 + runtime/js/90_deno_ns.js | 19 +- runtime/js/99_main.js | 14 +- runtime/js/telemetry.js | 395 +++++++++++++ runtime/lib.rs | 16 +- runtime/ops/mod.rs | 1 + runtime/ops/os/mod.rs | 2 + runtime/ops/otel.rs | 686 ++++++++++++++++++++++ runtime/shared.rs | 1 + runtime/snapshot.rs | 1 + runtime/web_worker.rs | 3 + runtime/worker.rs | 3 + runtime/worker_bootstrap.rs | 11 + tests/specs/cli/otel_basic/__test__.jsonc | 4 + tests/specs/cli/otel_basic/child.ts | 20 + tests/specs/cli/otel_basic/deno.json | 4 + tests/specs/cli/otel_basic/main.ts | 76 +++ tools/core_import_map.json | 1 + 27 files changed, 1742 insertions(+), 124 deletions(-) create mode 100644 runtime/js/telemetry.js create mode 100644 runtime/ops/otel.rs create mode 100644 tests/specs/cli/otel_basic/__test__.jsonc create mode 100644 tests/specs/cli/otel_basic/child.ts create mode 100644 tests/specs/cli/otel_basic/deno.json create mode 100644 tests/specs/cli/otel_basic/main.ts diff --git a/Cargo.lock b/Cargo.lock index 6f7799bac8..00c1f0736a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -347,6 +347,53 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +[[package]] +name = "axum" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a6c9af12842a67734c9a2e355436e5d03b22383ed60cf13cd0c18fbfe3dcbcf" +dependencies = [ + "async-trait", + "axum-core", + "bytes", + "futures-util", + "http 1.1.0", + "http-body 1.0.0", + "http-body-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "sync_wrapper", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http 1.1.0", + "http-body 1.0.0", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper", + "tower-layer", + "tower-service", +] + [[package]] name = "backtrace" version = "0.3.73" @@ -1118,7 +1165,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ "cfg-if", - "hashbrown", + "hashbrown 0.14.5", "lock_api", "once_cell", "parking_lot_core", @@ -1207,7 +1254,7 @@ dependencies = [ "http-body-util", "hyper-util", "import_map", - "indexmap", + "indexmap 2.3.0", "jsonc-parser", "junction", "lazy-regex", @@ -1363,7 +1410,7 @@ dependencies = [ "base32", "deno_media_type", "deno_path_util", - "indexmap", + "indexmap 2.3.0", "log", "once_cell", "parking_lot", @@ -1398,7 +1445,7 @@ dependencies = [ "glob", "ignore", "import_map", - "indexmap", + "indexmap 2.3.0", "jsonc-parser", "log", "percent-encoding", @@ -1519,7 +1566,7 @@ dependencies = [ "handlebars", "html-escape", "import_map", - "indexmap", + "indexmap 2.3.0", "itoa", "lazy_static", "regex", @@ -1619,7 +1666,7 @@ dependencies = [ "encoding_rs", "futures", "import_map", - "indexmap", + "indexmap 2.3.0", "log", "monch", "once_cell", @@ -1715,7 +1762,7 @@ dependencies = [ "http-body-util", "log", "num-bigint", - "prost", + "prost 0.11.9", "prost-build", "rand", "rusqlite", @@ -1851,7 +1898,7 @@ dependencies = [ "hyper 1.4.1", "hyper-util", "idna 0.3.0", - "indexmap", + "indexmap 2.3.0", "ipnetwork", "k256", "lazy-regex", @@ -1941,7 +1988,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6cbc4c4d3eb0960b58e8f43f9fc2d3f620fcac9a03cd85203e08db5b04e83c1f" dependencies = [ "deno_semver", - "indexmap", + "indexmap 2.3.0", "serde", "serde_json", "thiserror", @@ -1997,6 +2044,7 @@ dependencies = [ name = "deno_runtime" version = "0.186.0" dependencies = [ + "async-trait", "color-print", "deno_ast", "deno_broadcast_channel", @@ -2042,7 +2090,13 @@ dependencies = [ "notify", "ntapi", "once_cell", + "opentelemetry", + "opentelemetry-http", + "opentelemetry-otlp", + "opentelemetry-semantic-conventions", + "opentelemetry_sdk", "percent-encoding", + "pin-project", "regex", "rustyline", "same-file", @@ -2268,7 +2322,7 @@ dependencies = [ "chrono", "futures", "num-bigint", - "prost", + "prost 0.11.9", "serde", "uuid", ] @@ -2288,7 +2342,7 @@ dependencies = [ "futures", "http 1.1.0", "log", - "prost", + "prost 0.11.9", "rand", "serde", "serde_json", @@ -2548,8 +2602,8 @@ checksum = "f3ab0dd2bedc109d25f0d21afb09b7d329f6c6fa83b095daf31d2d967e091548" dependencies = [ "anyhow", "bumpalo", - "hashbrown", - "indexmap", + "hashbrown 0.14.5", + "indexmap 2.3.0", "rustc-hash 1.1.0", "serde", "unicode-width", @@ -2755,7 +2809,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48cede2bb1b07dd598d269f973792c43e0cd92686d3b452bd6e01d7a8eb01211" dependencies = [ "debug-ignore", - "indexmap", + "indexmap 2.3.0", "log", "thiserror", "zerocopy", @@ -3392,7 +3446,7 @@ checksum = "9c08c1f623a8d0b722b8b99f821eb0ba672a1618f0d3b16ddbee1cedd2dd8557" dependencies = [ "bitflags 2.6.0", "gpu-descriptor-types", - "hashbrown", + "hashbrown 0.14.5", ] [[package]] @@ -3436,7 +3490,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.12", - "indexmap", + "indexmap 2.3.0", "slab", "tokio", "tokio-util", @@ -3455,7 +3509,7 @@ dependencies = [ "futures-sink", "futures-util", "http 1.1.0", - "indexmap", + "indexmap 2.3.0", "slab", "tokio", "tokio-util", @@ -3468,7 +3522,7 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8588661a8607108a5ca69cab034063441a0413a0b041c13618a7dd348021ef6f" dependencies = [ - "hashbrown", + "hashbrown 0.14.5", "serde", ] @@ -3487,6 +3541,12 @@ dependencies = [ "thiserror", ] +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + [[package]] name = "hashbrown" version = "0.14.5" @@ -3503,7 +3563,7 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af" dependencies = [ - "hashbrown", + "hashbrown 0.14.5", ] [[package]] @@ -3666,7 +3726,7 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a9de2bdef6354361892492bab5e316b2d78a0ee9971db4d36da9b1eb0e11999" dependencies = [ - "hashbrown", + "hashbrown 0.14.5", "new_debug_unreachable", "once_cell", "phf", @@ -3820,6 +3880,19 @@ dependencies = [ "webpki-roots", ] +[[package]] +name = "hyper-timeout" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3203a961e5c83b6f5498933e78b6b263e208c197b63e9c6c53cc82ffd3f63793" +dependencies = [ + "hyper 1.4.1", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + [[package]] name = "hyper-util" version = "0.1.7" @@ -3908,7 +3981,7 @@ version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "351a787decc56f38d65d16d32687265045d6d6a4531b4a0e1b649def3590354e" dependencies = [ - "indexmap", + "indexmap 2.3.0", "log", "percent-encoding", "serde", @@ -3917,6 +3990,16 @@ dependencies = [ "url", ] +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", +] + [[package]] name = "indexmap" version = "2.3.0" @@ -3924,7 +4007,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de3fc2e30ba82dd1b3911c8de1ffc143c74a914a14e99514d7637e3099df5ea0" dependencies = [ "equivalent", - "hashbrown", + "hashbrown 0.14.5", "serde", ] @@ -4406,6 +4489,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + [[package]] name = "md-5" version = "0.10.6" @@ -4533,7 +4622,7 @@ dependencies = [ "bitflags 2.6.0", "codespan-reporting", "hexf-parse", - "indexmap", + "indexmap 2.3.0", "log", "num-traits", "rustc-hash 1.1.0", @@ -4837,6 +4926,93 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +[[package]] +name = "opentelemetry" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f3cebff57f7dbd1255b44d8bddc2cebeb0ea677dbaa2e25a3070a91b318f660" +dependencies = [ + "futures-core", + "futures-sink", + "js-sys", + "once_cell", + "pin-project-lite", + "thiserror", +] + +[[package]] +name = "opentelemetry-http" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a8a7f5f6ba7c1b286c2fbca0454eaba116f63bbe69ed250b642d36fbb04d80" +dependencies = [ + "async-trait", + "bytes", + "http 1.1.0", + "opentelemetry", +] + +[[package]] +name = "opentelemetry-otlp" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91cf61a1868dacc576bf2b2a1c3e9ab150af7272909e80085c3173384fe11f76" +dependencies = [ + "async-trait", + "futures-core", + "http 1.1.0", + "opentelemetry", + "opentelemetry-http", + "opentelemetry-proto", + "opentelemetry_sdk", + "prost 0.13.3", + "serde_json", + "thiserror", + "tokio", + "tonic", + "tracing", +] + +[[package]] +name = "opentelemetry-proto" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6e05acbfada5ec79023c85368af14abd0b307c015e9064d249b2a950ef459a6" +dependencies = [ + "hex", + "opentelemetry", + "opentelemetry_sdk", + "prost 0.13.3", + "serde", + "tonic", +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc1b6902ff63b32ef6c489e8048c5e253e2e4a803ea3ea7e783914536eb15c52" + +[[package]] +name = "opentelemetry_sdk" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b742c1cae4693792cc564e58d75a2a0ba29421a34a85b50da92efa89ecb2bc" +dependencies = [ + "async-trait", + "futures-channel", + "futures-executor", + "futures-util", + "glob", + "once_cell", + "opentelemetry", + "percent-encoding", + "rand", + "serde_json", + "thiserror", + "tracing", +] + [[package]] name = "option-ext" version = "0.2.0" @@ -5062,7 +5238,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" dependencies = [ "fixedbitset", - "indexmap", + "indexmap 2.3.0", ] [[package]] @@ -5339,7 +5515,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" dependencies = [ "bytes", - "prost-derive", + "prost-derive 0.11.9", +] + +[[package]] +name = "prost" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b0487d90e047de87f984913713b85c601c05609aad5b0df4b4573fbf69aa13f" +dependencies = [ + "bytes", + "prost-derive 0.13.3", ] [[package]] @@ -5356,7 +5542,7 @@ dependencies = [ "multimap", "petgraph", "prettyplease 0.1.25", - "prost", + "prost 0.11.9", "prost-types", "regex", "syn 1.0.109", @@ -5377,13 +5563,26 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "prost-derive" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9552f850d5f0964a4e4d0bf306459ac29323ddfbae05e35a7c0d35cb0803cc5" +dependencies = [ + "anyhow", + "itertools 0.13.0", + "proc-macro2", + "quote", + "syn 2.0.72", +] + [[package]] name = "prost-types" version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13" dependencies = [ - "prost", + "prost 0.11.9", ] [[package]] @@ -5439,7 +5638,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d1a341ae463320e9f8f34adda49c8a85d81d4e8f34cce4397fb0350481552224" dependencies = [ "chrono", - "indexmap", + "indexmap 2.3.0", "quick-xml", "strip-ansi-escapes", "thiserror", @@ -5801,7 +6000,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a58fa8a7ccff2aec4f39cc45bf5f985cec7125ab271cf681c279fd00192b49" dependencies = [ "countme", - "hashbrown", + "hashbrown 0.14.5", "memoffset", "rustc-hash 1.1.0", "text-size", @@ -6206,7 +6405,7 @@ version = "1.0.122" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "784b6203951c57ff748476b126ccb5e8e2959a5c19e5c617ab1956be3dbc68da" dependencies = [ - "indexmap", + "indexmap 2.3.0", "itoa", "memchr", "ryu", @@ -6603,7 +6802,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adc8bd3075d1c6964010333fae9ddcd91ad422a4f8eb8b3206a9b2b6afb4209e" dependencies = [ "bumpalo", - "hashbrown", + "hashbrown 0.14.5", "ptr_meta", "rustc-hash 1.1.0", "triomphe", @@ -6629,7 +6828,7 @@ checksum = "c77c112c218a09635d99a45802a81b4f341d6c28c81076aa2c29ba3bcd9151a9" dependencies = [ "anyhow", "crc", - "indexmap", + "indexmap 2.3.0", "is-macro", "once_cell", "parking_lot", @@ -6699,7 +6898,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4740e53eaf68b101203c1df0937d5161a29f3c13bceed0836ddfe245b72dd000" dependencies = [ "anyhow", - "indexmap", + "indexmap 2.3.0", "serde", "serde_json", "swc_cached", @@ -6811,7 +7010,7 @@ checksum = "65f21494e75d0bd8ef42010b47cabab9caaed8f2207570e809f6f4eb51a710d1" dependencies = [ "better_scoped_tls", "bitflags 2.6.0", - "indexmap", + "indexmap 2.3.0", "once_cell", "phf", "rustc-hash 1.1.0", @@ -6859,7 +7058,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "98d8447ea20ef76958a8240feef95743702485a84331e6df5bdbe7e383c87838" dependencies = [ "dashmap", - "indexmap", + "indexmap 2.3.0", "once_cell", "petgraph", "rustc-hash 1.1.0", @@ -6904,7 +7103,7 @@ checksum = "76c76d8b9792ce51401d38da0fa62158d61f6d80d16d68fe5b03ce4bf5fba383" dependencies = [ "base64 0.21.7", "dashmap", - "indexmap", + "indexmap 2.3.0", "once_cell", "serde", "sha1", @@ -6944,7 +7143,7 @@ version = "0.134.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "029eec7dd485923a75b5a45befd04510288870250270292fc2c1b3a9e7547408" dependencies = [ - "indexmap", + "indexmap 2.3.0", "num_cpus", "once_cell", "rustc-hash 1.1.0", @@ -6989,7 +7188,7 @@ version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "357e2c97bb51431d65080f25b436bc4e2fc1a7f64a643bc21a8353e478dc799f" dependencies = [ - "indexmap", + "indexmap 2.3.0", "petgraph", "rustc-hash 1.1.0", "swc_common", @@ -7210,7 +7409,7 @@ dependencies = [ "os_pipe", "parking_lot", "pretty_assertions", - "prost", + "prost 0.11.9", "prost-build", "regex", "reqwest", @@ -7402,9 +7601,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.15" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "267ac89e0bec6e691e5813911606935d77c476ff49024f98abcea3e7b15e37af" +checksum = "4f4e6ce100d0eb49a2734f8c0812bcd324cf357d21810932c5df6b96ef2b86f1" dependencies = [ "futures-core", "pin-project-lite", @@ -7422,7 +7621,7 @@ dependencies = [ "futures-io", "futures-sink", "futures-util", - "hashbrown", + "hashbrown 0.14.5", "pin-project-lite", "slab", "tokio", @@ -7438,6 +7637,36 @@ dependencies = [ "serde", ] +[[package]] +name = "tonic" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52" +dependencies = [ + "async-stream", + "async-trait", + "axum", + "base64 0.22.1", + "bytes", + "h2 0.4.4", + "http 1.1.0", + "http-body 1.0.0", + "http-body-util", + "hyper 1.4.1", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "prost 0.13.3", + "socket2", + "tokio", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "tower" version = "0.4.13" @@ -7446,11 +7675,16 @@ checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" dependencies = [ "futures-core", "futures-util", + "indexmap 1.9.3", "pin-project", "pin-project-lite", + "rand", + "slab", "tokio", + "tokio-util", "tower-layer", "tower-service", + "tracing", ] [[package]] @@ -7763,7 +7997,7 @@ checksum = "97599c400fc79925922b58303e98fcb8fa88f573379a08ddb652e72cbd2e70f6" dependencies = [ "bitflags 2.6.0", "encoding_rs", - "indexmap", + "indexmap 2.3.0", "num-bigint", "serde", "thiserror", @@ -7972,7 +8206,7 @@ dependencies = [ "cfg_aliases", "codespan-reporting", "document-features", - "indexmap", + "indexmap 2.3.0", "log", "naga", "once_cell", @@ -8521,7 +8755,7 @@ dependencies = [ "crossbeam-utils", "displaydoc", "flate2", - "indexmap", + "indexmap 2.3.0", "memchr", "thiserror", ] diff --git a/Cargo.toml b/Cargo.toml index e372e542bb..50e41145bf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -207,6 +207,12 @@ yoke = { version = "0.7.4", features = ["derive"] } zeromq = { version = "=0.4.1", default-features = false, features = ["tcp-transport", "tokio-runtime"] } zstd = "=0.12.4" +opentelemetry = "0.27.0" +opentelemetry-http = "0.27.0" +opentelemetry-otlp = { version = "0.27.0", features = ["logs", "http-proto", "http-json"] } +opentelemetry-semantic-conventions = { version = "0.27.0", features = ["semconv_experimental"] } +opentelemetry_sdk = "0.27.0" + # crypto hkdf = "0.12.3" rsa = { version = "0.9.3", default-features = false, features = ["std", "pem", "hazmat"] } # hazmat needed for PrehashSigner in ext/node diff --git a/cli/args/mod.rs b/cli/args/mod.rs index e19025f8b1..3aaf2bd438 100644 --- a/cli/args/mod.rs +++ b/cli/args/mod.rs @@ -27,6 +27,7 @@ use deno_npm::npm_rc::ResolvedNpmRc; use deno_npm::resolution::ValidSerializedNpmResolutionSnapshot; use deno_npm::NpmSystemInfo; use deno_path_util::normalize_path; +use deno_runtime::ops::otel::OtelConfig; use deno_semver::npm::NpmPackageReqReference; use import_map::resolve_import_map_value_from_specifier; @@ -1129,6 +1130,23 @@ impl CliOptions { } } + pub fn otel_config(&self) -> Option { + if self + .flags + .unstable_config + .features + .contains(&String::from("otel")) + { + Some(OtelConfig { + runtime_name: Cow::Borrowed("deno"), + runtime_version: Cow::Borrowed(crate::version::DENO_VERSION_INFO.deno), + ..Default::default() + }) + } else { + None + } + } + pub fn env_file_name(&self) -> Option<&String> { self.flags.env_file.as_ref() } diff --git a/cli/factory.rs b/cli/factory.rs index 4a36c75ba2..417f771a30 100644 --- a/cli/factory.rs +++ b/cli/factory.rs @@ -939,6 +939,7 @@ impl CliFactory { StorageKeyResolver::from_options(cli_options), cli_options.sub_command().clone(), self.create_cli_main_worker_options()?, + self.cli_options()?.otel_config(), )) } diff --git a/cli/standalone/binary.rs b/cli/standalone/binary.rs index 9e26512268..960aad1578 100644 --- a/cli/standalone/binary.rs +++ b/cli/standalone/binary.rs @@ -47,6 +47,7 @@ use deno_runtime::deno_fs::FileSystem; use deno_runtime::deno_fs::RealFs; use deno_runtime::deno_io::fs::FsError; use deno_runtime::deno_node::PackageJson; +use deno_runtime::ops::otel::OtelConfig; use deno_semver::npm::NpmVersionReqParseError; use deno_semver::package::PackageReq; use deno_semver::Version; @@ -185,6 +186,7 @@ pub struct Metadata { pub entrypoint_key: String, pub node_modules: Option, pub unstable_config: UnstableConfig, + pub otel_config: Option, // None means disabled. } fn write_binary_bytes( @@ -722,6 +724,7 @@ impl<'a> DenoCompileBinaryWriter<'a> { sloppy_imports: cli_options.unstable_sloppy_imports(), features: cli_options.unstable_features(), }, + otel_config: cli_options.otel_config(), }; write_binary_bytes( diff --git a/cli/standalone/mod.rs b/cli/standalone/mod.rs index 85610f4c20..bb0ab423dd 100644 --- a/cli/standalone/mod.rs +++ b/cli/standalone/mod.rs @@ -800,6 +800,7 @@ pub async fn run(data: StandaloneData) -> Result { serve_port: None, serve_host: None, }, + metadata.otel_config, ); // Initialize v8 once from the main thread. diff --git a/cli/tsc/dts/lib.deno.unstable.d.ts b/cli/tsc/dts/lib.deno.unstable.d.ts index 973a09d92a..6234268c39 100644 --- a/cli/tsc/dts/lib.deno.unstable.d.ts +++ b/cli/tsc/dts/lib.deno.unstable.d.ts @@ -1225,6 +1225,108 @@ declare namespace Deno { export {}; // only export exports } + /** + * @category Telemetry + * @experimental + */ + export namespace tracing { + /** + * Whether tracing is enabled. + * @category Telemetry + * @experimental + */ + export const enabled: boolean; + + /** + * Allowed attribute type. + * @category Telemetry + * @experimental + */ + export type AttributeValue = string | number | boolean | bigint; + + /** + * A tracing span. + * @category Telemetry + * @experimental + */ + export class Span implements Disposable { + readonly traceId: string; + readonly spanId: string; + readonly parentSpanId: string; + readonly kind: string; + readonly name: string; + readonly startTime: number; + readonly endTime: number; + readonly status: null | { code: 1 } | { code: 2; message: string }; + readonly attributes: Record; + readonly traceFlags: number; + + /** + * Construct a new Span and enter it as the "current" span. + */ + constructor( + name: string, + kind?: "internal" | "server" | "client" | "producer" | "consumer", + ); + + /** + * Set an attribute on this span. + */ + setAttribute( + name: string, + value: AttributeValue, + ): void; + + /** + * Enter this span as the "current" span. + */ + enter(): void; + + /** + * Exit this span as the "current" span and restore the previous one. + */ + exit(): void; + + /** + * End this span, and exit it as the "current" span. + */ + end(): void; + + [Symbol.dispose](): void; + + /** + * Get the "current" span, if one exists. + */ + static current(): Span | undefined | null; + } + + /** + * A SpanExporter compatible with OpenTelemetry.js + * https://open-telemetry.github.io/opentelemetry-js/interfaces/_opentelemetry_sdk_trace_base.SpanExporter.html + * @category Telemetry + * @experimental + */ + export class SpanExporter {} + + /** + * A ContextManager compatible with OpenTelemetry.js + * https://open-telemetry.github.io/opentelemetry-js/interfaces/_opentelemetry_api.ContextManager.html + * @category Telemetry + * @experimental + */ + export class ContextManager {} + + export {}; // only export exports + } + + /** + * @category Telemetry + * @experimental + */ + export namespace metrics { + export {}; // only export exports + } + export {}; // only export exports } diff --git a/cli/worker.rs b/cli/worker.rs index baacd681a1..402644a42c 100644 --- a/cli/worker.rs +++ b/cli/worker.rs @@ -30,6 +30,7 @@ use deno_runtime::deno_tls::RootCertStoreProvider; use deno_runtime::deno_web::BlobStore; use deno_runtime::fmt_errors::format_js_error; use deno_runtime::inspector_server::InspectorServer; +use deno_runtime::ops::otel::OtelConfig; use deno_runtime::ops::process::NpmProcessStateProviderRc; use deno_runtime::ops::worker_host::CreateWebWorkerCb; use deno_runtime::web_worker::WebWorker; @@ -142,6 +143,7 @@ struct SharedWorkerState { storage_key_resolver: StorageKeyResolver, options: CliMainWorkerOptions, subcommand: DenoSubcommand, + otel_config: Option, // `None` means OpenTelemetry is disabled. } impl SharedWorkerState { @@ -405,6 +407,7 @@ impl CliMainWorkerFactory { storage_key_resolver: StorageKeyResolver, subcommand: DenoSubcommand, options: CliMainWorkerOptions, + otel_config: Option, ) -> Self { Self { shared: Arc::new(SharedWorkerState { @@ -427,6 +430,7 @@ impl CliMainWorkerFactory { storage_key_resolver, options, subcommand, + otel_config, }), } } @@ -576,6 +580,7 @@ impl CliMainWorkerFactory { mode, serve_port: shared.options.serve_port, serve_host: shared.options.serve_host.clone(), + otel_config: shared.otel_config.clone(), }, extensions: custom_extensions, startup_snapshot: crate::js::deno_isolate_init(), @@ -775,6 +780,7 @@ fn create_web_worker_callback( mode: WorkerExecutionMode::Worker, serve_port: shared.options.serve_port, serve_host: shared.options.serve_host.clone(), + otel_config: shared.otel_config.clone(), }, extensions: vec![], startup_snapshot: crate::js::deno_isolate_init(), diff --git a/ext/http/00_serve.ts b/ext/http/00_serve.ts index 7bf83e49c3..fcdb87d092 100644 --- a/ext/http/00_serve.ts +++ b/ext/http/00_serve.ts @@ -42,6 +42,10 @@ const { Uint8Array, Promise, } = primordials; +const { + getAsyncContext, + setAsyncContext, +} = core; import { InnerBody } from "ext:deno_fetch/22_body.js"; import { Event } from "ext:deno_web/02_event.js"; @@ -397,8 +401,10 @@ class CallbackContext { /** @type {Promise | undefined} */ closing; listener; + asyncContext; constructor(signal, args, listener) { + this.asyncContext = getAsyncContext(); // The abort signal triggers a non-graceful shutdown signal?.addEventListener( "abort", @@ -508,82 +514,89 @@ function fastSyncResponseOrStream( */ function mapToCallback(context, callback, onError) { return async function (req) { - // Get the response from the user-provided callback. If that fails, use onError. If that fails, return a fallback - // 500 error. - let innerRequest; - let response; + const asyncContext = getAsyncContext(); + setAsyncContext(context.asyncContext); + try { - innerRequest = new InnerRequest(req, context); - const request = fromInnerRequest(innerRequest, "immutable"); - innerRequest.request = request; - response = await callback( - request, - new ServeHandlerInfo(innerRequest), - ); - - // Throwing Error if the handler return value is not a Response class - if (!ObjectPrototypeIsPrototypeOf(ResponsePrototype, response)) { - throw new TypeError( - "Return value from serve handler must be a response or a promise resolving to a response", - ); - } - - if (response.type === "error") { - throw new TypeError( - "Return value from serve handler must not be an error response (like Response.error())", - ); - } - - if (response.bodyUsed) { - throw new TypeError( - "The body of the Response returned from the serve handler has already been consumed", - ); - } - } catch (error) { + // Get the response from the user-provided callback. If that fails, use onError. If that fails, return a fallback + // 500 error. + let innerRequest; + let response; try { - response = await onError(error); + innerRequest = new InnerRequest(req, context); + const request = fromInnerRequest(innerRequest, "immutable"); + innerRequest.request = request; + response = await callback( + request, + new ServeHandlerInfo(innerRequest), + ); + + // Throwing Error if the handler return value is not a Response class if (!ObjectPrototypeIsPrototypeOf(ResponsePrototype, response)) { throw new TypeError( - "Return value from onError handler must be a response or a promise resolving to a response", + "Return value from serve handler must be a response or a promise resolving to a response", + ); + } + + if (response.type === "error") { + throw new TypeError( + "Return value from serve handler must not be an error response (like Response.error())", + ); + } + + if (response.bodyUsed) { + throw new TypeError( + "The body of the Response returned from the serve handler has already been consumed", ); } } catch (error) { - // deno-lint-ignore no-console - console.error("Exception in onError while handling exception", error); - response = internalServerError(); + try { + response = await onError(error); + if (!ObjectPrototypeIsPrototypeOf(ResponsePrototype, response)) { + throw new TypeError( + "Return value from onError handler must be a response or a promise resolving to a response", + ); + } + } catch (error) { + // deno-lint-ignore no-console + console.error("Exception in onError while handling exception", error); + response = internalServerError(); + } } - } - const inner = toInnerResponse(response); - if (innerRequest?.[_upgraded]) { - // We're done here as the connection has been upgraded during the callback and no longer requires servicing. - if (response !== UPGRADE_RESPONSE_SENTINEL) { - // deno-lint-ignore no-console - console.error("Upgrade response was not returned from callback"); - context.close(); + const inner = toInnerResponse(response); + if (innerRequest?.[_upgraded]) { + // We're done here as the connection has been upgraded during the callback and no longer requires servicing. + if (response !== UPGRADE_RESPONSE_SENTINEL) { + // deno-lint-ignore no-console + console.error("Upgrade response was not returned from callback"); + context.close(); + } + innerRequest?.[_upgraded](); + return; } - innerRequest?.[_upgraded](); - return; - } - // Did everything shut down while we were waiting? - if (context.closed) { - // We're shutting down, so this status shouldn't make it back to the client but "Service Unavailable" seems appropriate - innerRequest?.close(); - op_http_set_promise_complete(req, 503); - return; - } - - const status = inner.status; - const headers = inner.headerList; - if (headers && headers.length > 0) { - if (headers.length == 1) { - op_http_set_response_header(req, headers[0][0], headers[0][1]); - } else { - op_http_set_response_headers(req, headers); + // Did everything shut down while we were waiting? + if (context.closed) { + // We're shutting down, so this status shouldn't make it back to the client but "Service Unavailable" seems appropriate + innerRequest?.close(); + op_http_set_promise_complete(req, 503); + return; } - } - fastSyncResponseOrStream(req, inner.body, status, innerRequest); + const status = inner.status; + const headers = inner.headerList; + if (headers && headers.length > 0) { + if (headers.length == 1) { + op_http_set_response_header(req, headers[0][0], headers[0][1]); + } else { + op_http_set_response_headers(req, headers); + } + } + + fastSyncResponseOrStream(req, inner.body, status, innerRequest); + } finally { + setAsyncContext(asyncContext); + } }; } diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index ba236de149..b59cd14fa9 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -100,6 +100,7 @@ deno_websocket.workspace = true deno_webstorage.workspace = true node_resolver = { workspace = true, features = ["sync"] } +async-trait.workspace = true color-print.workspace = true dlopen2.workspace = true encoding_rs.workspace = true @@ -114,7 +115,13 @@ log.workspace = true netif = "0.1.6" notify.workspace = true once_cell.workspace = true +opentelemetry.workspace = true +opentelemetry-http.workspace = true +opentelemetry-otlp.workspace = true +opentelemetry-semantic-conventions.workspace = true +opentelemetry_sdk.workspace = true percent-encoding.workspace = true +pin-project.workspace = true regex.workspace = true rustyline = { workspace = true, features = ["custom-bindings"] } same-file = "1.0.6" diff --git a/runtime/js/90_deno_ns.js b/runtime/js/90_deno_ns.js index fd2ac00f20..11f618ce27 100644 --- a/runtime/js/90_deno_ns.js +++ b/runtime/js/90_deno_ns.js @@ -29,6 +29,7 @@ import * as tty from "ext:runtime/40_tty.js"; import * as kv from "ext:deno_kv/01_db.ts"; import * as cron from "ext:deno_cron/01_cron.ts"; import * as webgpuSurface from "ext:deno_webgpu/02_surface.js"; +import * as telemetry from "ext:runtime/telemetry.js"; const denoNs = { Process: process.Process, @@ -134,7 +135,7 @@ const denoNs = { createHttpClient: httpClient.createHttpClient, }; -// NOTE(bartlomieju): keep IDs in sync with `cli/main.rs` +// NOTE(bartlomieju): keep IDs in sync with `runtime/lib.rs` const unstableIds = { broadcastChannel: 1, cron: 2, @@ -143,11 +144,12 @@ const unstableIds = { http: 5, kv: 6, net: 7, - process: 8, - temporal: 9, - unsafeProto: 10, - webgpu: 11, - workerOptions: 12, + otel: 8, + process: 9, + temporal: 10, + unsafeProto: 11, + webgpu: 12, + workerOptions: 13, }; const denoNsUnstableById = { __proto__: null }; @@ -181,4 +183,9 @@ denoNsUnstableById[unstableIds.webgpu] = { // denoNsUnstableById[unstableIds.workerOptions] = { __proto__: null } +denoNsUnstableById[unstableIds.otel] = { + tracing: telemetry.tracing, + metrics: telemetry.metrics, +}; + export { denoNs, denoNsUnstableById, unstableIds }; diff --git a/runtime/js/99_main.js b/runtime/js/99_main.js index 6ddaa1335e..2da5c5398c 100644 --- a/runtime/js/99_main.js +++ b/runtime/js/99_main.js @@ -86,6 +86,8 @@ import { workerRuntimeGlobalProperties, } from "ext:runtime/98_global_scope_worker.js"; import { SymbolDispose, SymbolMetadata } from "ext:deno_web/00_infra.js"; +import { bootstrap as bootstrapOtel } from "ext:runtime/telemetry.js"; + // deno-lint-ignore prefer-primordials if (Symbol.metadata) { throw "V8 supports Symbol.metadata now, no need to shim it"; @@ -573,6 +575,7 @@ function bootstrapMainRuntime(runtimeOptions, warmup = false) { 10: serveHost, 11: serveIsMain, 12: serveWorkerCount, + 13: otelConfig, } = runtimeOptions; if (mode === executionModes.serve) { @@ -673,9 +676,10 @@ function bootstrapMainRuntime(runtimeOptions, warmup = false) { }); ObjectSetPrototypeOf(globalThis, Window.prototype); + bootstrapOtel(otelConfig); + if (inspectFlag) { - const consoleFromDeno = globalThis.console; - core.wrapConsole(consoleFromDeno, core.v8Console); + core.wrapConsole(globalThis.console, core.v8Console); } event.defineEventHandler(globalThis, "error"); @@ -855,6 +859,7 @@ function bootstrapWorkerRuntime( 5: hasNodeModulesDir, 6: argv0, 7: nodeDebug, + 13: otelConfig, } = runtimeOptions; performance.setTimeOrigin(); @@ -882,8 +887,9 @@ function bootstrapWorkerRuntime( } ObjectSetPrototypeOf(globalThis, DedicatedWorkerGlobalScope.prototype); - const consoleFromDeno = globalThis.console; - core.wrapConsole(consoleFromDeno, core.v8Console); + bootstrapOtel(otelConfig); + + core.wrapConsole(globalThis.console, core.v8Console); event.defineEventHandler(self, "message"); event.defineEventHandler(self, "error", undefined, true); diff --git a/runtime/js/telemetry.js b/runtime/js/telemetry.js new file mode 100644 index 0000000000..e9eb51f7ca --- /dev/null +++ b/runtime/js/telemetry.js @@ -0,0 +1,395 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +import { core, primordials } from "ext:core/mod.js"; +import { + op_otel_log, + op_otel_span_attribute, + op_otel_span_attribute2, + op_otel_span_attribute3, + op_otel_span_continue, + op_otel_span_flush, + op_otel_span_start, +} from "ext:core/ops"; +import { Console } from "ext:deno_console/01_console.js"; +import { performance } from "ext:deno_web/15_performance.js"; + +const { + SymbolDispose, + MathRandom, + Array, + ObjectEntries, + SafeMap, + ReflectApply, + SymbolFor, + Error, +} = primordials; +const { AsyncVariable, setAsyncContext } = core; + +const CURRENT = new AsyncVariable(); +let TRACING_ENABLED = false; + +const SPAN_ID_BYTES = 8; +const TRACE_ID_BYTES = 16; + +const TRACE_FLAG_SAMPLED = 1 << 0; + +const hexSliceLookupTable = (function () { + const alphabet = "0123456789abcdef"; + const table = new Array(256); + for (let i = 0; i < 16; ++i) { + const i16 = i * 16; + for (let j = 0; j < 16; ++j) { + table[i16 + j] = alphabet[i] + alphabet[j]; + } + } + return table; +})(); + +function generateId(bytes) { + let out = ""; + for (let i = 0; i < bytes / 4; i += 1) { + const r32 = (MathRandom() * 2 ** 32) >>> 0; + out += hexSliceLookupTable[(r32 >> 24) & 0xff]; + out += hexSliceLookupTable[(r32 >> 16) & 0xff]; + out += hexSliceLookupTable[(r32 >> 8) & 0xff]; + out += hexSliceLookupTable[r32 & 0xff]; + } + return out; +} + +function submit(span) { + if (!(span.traceFlags & TRACE_FLAG_SAMPLED)) return; + + op_otel_span_start( + span.traceId, + span.spanId, + span.parentSpanId ?? "", + span.kind, + span.name, + span.startTime, + span.endTime, + ); + + if (span.status !== null && span.status.code !== 0) { + op_otel_span_continue(span.code, span.message ?? ""); + } + + const attributes = ObjectEntries(span.attributes); + let i = 0; + while (i < attributes.length) { + if (i + 2 < attributes.length) { + op_otel_span_attribute3( + attributes.length, + attributes[i][0], + attributes[i][1], + attributes[i + 1][0], + attributes[i + 1][1], + attributes[i + 2][0], + attributes[i + 2][1], + ); + i += 3; + } else if (i + 1 < attributes.length) { + op_otel_span_attribute2( + attributes.length, + attributes[i][0], + attributes[i][1], + attributes[i + 1][0], + attributes[i + 1][1], + ); + i += 2; + } else { + op_otel_span_attribute( + attributes.length, + attributes[i][0], + attributes[i][1], + ); + i += 1; + } + } + + op_otel_span_flush(); +} + +const now = () => (performance.timeOrigin + performance.now()) / 1000; + +const INVALID_SPAN_ID = "0000000000000000"; +const INVALID_TRACE_ID = "00000000000000000000000000000000"; +const NO_ASYNC_CONTEXT = {}; + +class Span { + traceId; + spanId; + parentSpanId; + kind; + name; + startTime; + endTime; + status = null; + attributes = { __proto__: null }; + traceFlags = TRACE_FLAG_SAMPLED; + + enabled = TRACING_ENABLED; + #asyncContext = NO_ASYNC_CONTEXT; + + constructor(name, kind = "internal") { + if (!this.enabled) { + this.traceId = INVALID_TRACE_ID; + this.spanId = INVALID_SPAN_ID; + this.parentSpanId = INVALID_SPAN_ID; + return; + } + + this.startTime = now(); + + this.spanId = generateId(SPAN_ID_BYTES); + + let traceId; + let parentSpanId; + const parent = Span.current(); + if (parent) { + if (parent.spanId !== undefined) { + parentSpanId = parent.spanId; + traceId = parent.traceId; + } else { + const context = parent.spanContext(); + parentSpanId = context.spanId; + traceId = context.traceId; + } + } + if ( + traceId && traceId !== INVALID_TRACE_ID && parentSpanId && + parentSpanId !== INVALID_SPAN_ID + ) { + this.traceId = traceId; + this.parentSpanId = parentSpanId; + } else { + this.traceId = generateId(TRACE_ID_BYTES); + this.parentSpanId = INVALID_SPAN_ID; + } + + this.name = name; + + switch (kind) { + case "internal": + this.kind = 0; + break; + case "server": + this.kind = 1; + break; + case "client": + this.kind = 2; + break; + case "producer": + this.kind = 3; + break; + case "consumer": + this.kind = 4; + break; + default: + throw new Error(`Invalid span kind: ${kind}`); + } + + this.enter(); + } + + // helper function to match otel js api + spanContext() { + return { + traceId: this.traceId, + spanId: this.spanId, + traceFlags: this.traceFlags, + }; + } + + setAttribute(name, value) { + if (!this.enabled) return; + this.attributes[name] = value; + } + + enter() { + if (!this.enabled) return; + const context = (CURRENT.get() || ROOT_CONTEXT).setValue(SPAN_KEY, this); + this.#asyncContext = CURRENT.enter(context); + } + + exit() { + if (!this.enabled || this.#asyncContext === NO_ASYNC_CONTEXT) return; + setAsyncContext(this.#asyncContext); + this.#asyncContext = NO_ASYNC_CONTEXT; + } + + end() { + if (!this.enabled || this.endTime !== undefined) return; + this.exit(); + this.endTime = now(); + submit(this); + } + + [SymbolDispose]() { + this.end(); + } + + static current() { + return CURRENT.get()?.getValue(SPAN_KEY); + } +} + +function hrToSecs(hr) { + return ((hr[0] * 1e3 + hr[1] / 1e6) / 1000); +} + +// Exporter compatible with opentelemetry js library +class SpanExporter { + export(spans, resultCallback) { + try { + for (let i = 0; i < spans.length; i += 1) { + const span = spans[i]; + const context = span.spanContext(); + submit({ + spanId: context.spanId, + traceId: context.traceId, + traceFlags: context.traceFlags, + name: span.name, + kind: span.kind, + parentSpanId: span.parentSpanId, + startTime: hrToSecs(span.startTime), + endTime: hrToSecs(span.endTime), + status: span.status, + attributes: span.attributes, + }); + } + resultCallback({ code: 0 }); + } catch (error) { + resultCallback({ code: 1, error }); + } + } + + async shutdown() {} + + async forceFlush() {} +} + +// SPAN_KEY matches symbol in otel-js library +const SPAN_KEY = SymbolFor("OpenTelemetry Context Key SPAN"); + +// Context tracker compatible with otel-js api +class Context { + #data = new SafeMap(); + + constructor(data) { + this.#data = data ? new SafeMap(data) : new SafeMap(); + } + + getValue(key) { + return this.#data.get(key); + } + + setValue(key, value) { + const c = new Context(this.#data); + c.#data.set(key, value); + return c; + } + + deleteValue(key) { + const c = new Context(this.#data); + c.#data.delete(key); + return c; + } +} + +const ROOT_CONTEXT = new Context(); + +// Context manager for opentelemetry js library +class ContextManager { + active() { + return CURRENT.get() ?? ROOT_CONTEXT; + } + + with(context, fn, thisArg, ...args) { + const ctx = CURRENT.enter(context); + try { + return ReflectApply(fn, thisArg, args); + } finally { + setAsyncContext(ctx); + } + } + + bind(context, f) { + return (...args) => { + const ctx = CURRENT.enter(context); + try { + return ReflectApply(f, thisArg, args); + } finally { + setAsyncContext(ctx); + } + }; + } + + enable() { + return this; + } + + disable() { + return this; + } +} + +function otelLog(message, level) { + let traceId = ""; + let spanId = ""; + let traceFlags = 0; + const span = Span.current(); + if (span) { + if (span.spanId !== undefined) { + spanId = span.spanId; + traceId = span.traceId; + traceFlags = span.traceFlags; + } else { + const context = span.spanContext(); + spanId = context.spanId; + traceId = context.traceId; + traceFlags = context.traceFlags; + } + } + return op_otel_log(message, level, traceId, spanId, traceFlags); +} + +const otelConsoleConfig = { + ignore: 0, + capture: 1, + replace: 2, +}; + +export function bootstrap(config) { + if (config.length === 0) return; + const { 0: consoleConfig } = config; + + TRACING_ENABLED = true; + + switch (consoleConfig) { + case otelConsoleConfig.capture: + core.wrapConsole(globalThis.console, new Console(otelLog)); + break; + case otelConsoleConfig.replace: + ObjectDefineProperty( + globalThis, + "console", + core.propNonEnumerable(new Console(otelLog)), + ); + break; + default: + break; + } +} + +export const tracing = { + get enabled() { + return TRACING_ENABLED; + }, + Span, + SpanExporter, + ContextManager, +}; + +// TODO(devsnek): implement metrics +export const metrics = {}; diff --git a/runtime/lib.rs b/runtime/lib.rs index f0b1129ce3..21b61e1c05 100644 --- a/runtime/lib.rs +++ b/runtime/lib.rs @@ -99,18 +99,24 @@ pub static UNSTABLE_GRANULAR_FLAGS: &[UnstableGranularFlag] = &[ show_in_help: true, id: 7, }, + UnstableGranularFlag { + name: "otel", + help_text: "Enable unstable OpenTelemetry features", + show_in_help: false, + id: 8, + }, // TODO(bartlomieju): consider removing it UnstableGranularFlag { name: ops::process::UNSTABLE_FEATURE_NAME, help_text: "Enable unstable process APIs", show_in_help: false, - id: 8, + id: 9, }, UnstableGranularFlag { name: "temporal", help_text: "Enable unstable Temporal API", show_in_help: true, - id: 9, + id: 10, }, UnstableGranularFlag { name: "unsafe-proto", @@ -118,19 +124,19 @@ pub static UNSTABLE_GRANULAR_FLAGS: &[UnstableGranularFlag] = &[ show_in_help: true, // This number is used directly in the JS code. Search // for "unstableIds" to see where it's used. - id: 10, + id: 11, }, UnstableGranularFlag { name: deno_webgpu::UNSTABLE_FEATURE_NAME, help_text: "Enable unstable `WebGPU` APIs", show_in_help: true, - id: 11, + id: 12, }, UnstableGranularFlag { name: ops::worker_host::UNSTABLE_FEATURE_NAME, help_text: "Enable unstable Web Worker APIs", show_in_help: true, - id: 12, + id: 13, }, ]; diff --git a/runtime/ops/mod.rs b/runtime/ops/mod.rs index 67065b901b..c2e402f33c 100644 --- a/runtime/ops/mod.rs +++ b/runtime/ops/mod.rs @@ -4,6 +4,7 @@ pub mod bootstrap; pub mod fs_events; pub mod http; pub mod os; +pub mod otel; pub mod permissions; pub mod process; pub mod runtime; diff --git a/runtime/ops/os/mod.rs b/runtime/ops/os/mod.rs index 9bee9d8234..790962f38d 100644 --- a/runtime/ops/os/mod.rs +++ b/runtime/ops/os/mod.rs @@ -186,6 +186,8 @@ fn op_get_exit_code(state: &mut OpState) -> i32 { #[op2(fast)] fn op_exit(state: &mut OpState) { + crate::ops::otel::otel_drop_state(state); + let code = state.borrow::().get(); std::process::exit(code) } diff --git a/runtime/ops/otel.rs b/runtime/ops/otel.rs new file mode 100644 index 0000000000..6a4750acc2 --- /dev/null +++ b/runtime/ops/otel.rs @@ -0,0 +1,686 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +use crate::tokio_util::create_basic_runtime; +use deno_core::anyhow::anyhow; +use deno_core::anyhow::{self}; +use deno_core::futures::channel::mpsc; +use deno_core::futures::channel::mpsc::UnboundedSender; +use deno_core::futures::future::BoxFuture; +use deno_core::futures::stream; +use deno_core::futures::Stream; +use deno_core::futures::StreamExt; +use deno_core::op2; +use deno_core::v8; +use deno_core::OpState; +use once_cell::sync::Lazy; +use opentelemetry::logs::Severity; +use opentelemetry::trace::SpanContext; +use opentelemetry::trace::SpanId; +use opentelemetry::trace::SpanKind; +use opentelemetry::trace::Status as SpanStatus; +use opentelemetry::trace::TraceFlags; +use opentelemetry::trace::TraceId; +use opentelemetry::InstrumentationScope; +use opentelemetry::Key; +use opentelemetry::KeyValue; +use opentelemetry::StringValue; +use opentelemetry::Value; +use opentelemetry_otlp::HttpExporterBuilder; +use opentelemetry_otlp::Protocol; +use opentelemetry_otlp::WithExportConfig; +use opentelemetry_otlp::WithHttpConfig; +use opentelemetry_sdk::export::trace::SpanData; +use opentelemetry_sdk::logs::BatchLogProcessor; +use opentelemetry_sdk::logs::LogProcessor as LogProcessorTrait; +use opentelemetry_sdk::logs::LogRecord; +use opentelemetry_sdk::trace::BatchSpanProcessor; +use opentelemetry_sdk::trace::SpanProcessor as SpanProcessorTrait; +use opentelemetry_sdk::Resource; +use opentelemetry_semantic_conventions::resource::PROCESS_RUNTIME_NAME; +use opentelemetry_semantic_conventions::resource::PROCESS_RUNTIME_VERSION; +use opentelemetry_semantic_conventions::resource::TELEMETRY_SDK_LANGUAGE; +use opentelemetry_semantic_conventions::resource::TELEMETRY_SDK_NAME; +use opentelemetry_semantic_conventions::resource::TELEMETRY_SDK_VERSION; +use serde::Deserialize; +use serde::Serialize; +use std::borrow::Cow; +use std::env; +use std::fmt::Debug; +use std::pin::Pin; +use std::task::Context; +use std::task::Poll; +use std::thread; +use std::time::Duration; +use std::time::SystemTime; + +type SpanProcessor = BatchSpanProcessor; +type LogProcessor = BatchLogProcessor; + +deno_core::extension!( + deno_otel, + ops = [op_otel_log, op_otel_span_start, op_otel_span_continue, op_otel_span_attribute, op_otel_span_attribute2, op_otel_span_attribute3, op_otel_span_flush], + options = { + otel_config: Option, // `None` means OpenTelemetry is disabled. + }, + state = |state, options| { + if let Some(otel_config) = options.otel_config { + otel_create_globals(otel_config, state).unwrap(); + } + } +); + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct OtelConfig { + pub runtime_name: Cow<'static, str>, + pub runtime_version: Cow<'static, str>, + pub console: OtelConsoleConfig, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +#[repr(u8)] +pub enum OtelConsoleConfig { + Ignore = 0, + Capture = 1, + Replace = 2, +} + +impl Default for OtelConfig { + fn default() -> Self { + Self { + runtime_name: Cow::Borrowed(env!("CARGO_PKG_NAME")), + runtime_version: Cow::Borrowed(env!("CARGO_PKG_VERSION")), + console: OtelConsoleConfig::Capture, + } + } +} + +static OTEL_SHARED_RUNTIME_SPAWN_TASK_TX: Lazy< + UnboundedSender>, +> = Lazy::new(otel_create_shared_runtime); + +fn otel_create_shared_runtime() -> UnboundedSender> { + let (spawn_task_tx, mut spawn_task_rx) = + mpsc::unbounded::>(); + + thread::spawn(move || { + let rt = create_basic_runtime(); + rt.block_on(async move { + while let Some(task) = spawn_task_rx.next().await { + tokio::spawn(task); + } + }); + }); + + spawn_task_tx +} + +#[derive(Clone, Copy)] +struct OtelSharedRuntime; + +impl hyper::rt::Executor> for OtelSharedRuntime { + fn execute(&self, fut: BoxFuture<'static, ()>) { + (*OTEL_SHARED_RUNTIME_SPAWN_TASK_TX) + .unbounded_send(fut) + .expect("failed to send task to shared OpenTelemetry runtime"); + } +} + +impl opentelemetry_sdk::runtime::Runtime for OtelSharedRuntime { + type Interval = Pin + Send + 'static>>; + type Delay = Pin>; + + fn interval(&self, period: Duration) -> Self::Interval { + stream::repeat(()) + .then(move |_| tokio::time::sleep(period)) + .boxed() + } + + fn spawn(&self, future: BoxFuture<'static, ()>) { + (*OTEL_SHARED_RUNTIME_SPAWN_TASK_TX) + .unbounded_send(future) + .expect("failed to send task to shared OpenTelemetry runtime"); + } + + fn delay(&self, duration: Duration) -> Self::Delay { + Box::pin(tokio::time::sleep(duration)) + } +} + +impl opentelemetry_sdk::runtime::RuntimeChannel for OtelSharedRuntime { + type Receiver = BatchMessageChannelReceiver; + type Sender = BatchMessageChannelSender; + + fn batch_message_channel( + &self, + capacity: usize, + ) -> (Self::Sender, Self::Receiver) { + let (batch_tx, batch_rx) = tokio::sync::mpsc::channel::(capacity); + (batch_tx.into(), batch_rx.into()) + } +} + +#[derive(Debug)] +pub struct BatchMessageChannelSender { + sender: tokio::sync::mpsc::Sender, +} + +impl From> + for BatchMessageChannelSender +{ + fn from(sender: tokio::sync::mpsc::Sender) -> Self { + Self { sender } + } +} + +impl opentelemetry_sdk::runtime::TrySend + for BatchMessageChannelSender +{ + type Message = T; + + fn try_send( + &self, + item: Self::Message, + ) -> Result<(), opentelemetry_sdk::runtime::TrySendError> { + self.sender.try_send(item).map_err(|err| match err { + tokio::sync::mpsc::error::TrySendError::Full(_) => { + opentelemetry_sdk::runtime::TrySendError::ChannelFull + } + tokio::sync::mpsc::error::TrySendError::Closed(_) => { + opentelemetry_sdk::runtime::TrySendError::ChannelClosed + } + }) + } +} + +pub struct BatchMessageChannelReceiver { + receiver: tokio::sync::mpsc::Receiver, +} + +impl From> + for BatchMessageChannelReceiver +{ + fn from(receiver: tokio::sync::mpsc::Receiver) -> Self { + Self { receiver } + } +} + +impl Stream for BatchMessageChannelReceiver { + type Item = T; + + fn poll_next( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll> { + self.receiver.poll_recv(cx) + } +} + +mod hyper_client { + use http_body_util::BodyExt; + use http_body_util::Full; + use hyper::body::Body as HttpBody; + use hyper::body::Frame; + use hyper_util::client::legacy::connect::HttpConnector; + use hyper_util::client::legacy::Client; + use opentelemetry_http::Bytes; + use opentelemetry_http::HttpError; + use opentelemetry_http::Request; + use opentelemetry_http::Response; + use opentelemetry_http::ResponseExt; + use std::fmt::Debug; + use std::pin::Pin; + use std::task::Poll; + use std::task::{self}; + + use super::OtelSharedRuntime; + + // same as opentelemetry_http::HyperClient except it uses OtelSharedRuntime + #[derive(Debug, Clone)] + pub struct HyperClient { + inner: Client, + } + + impl HyperClient { + pub fn new() -> Self { + Self { + inner: Client::builder(OtelSharedRuntime).build(HttpConnector::new()), + } + } + } + + #[async_trait::async_trait] + impl opentelemetry_http::HttpClient for HyperClient { + async fn send( + &self, + request: Request>, + ) -> Result, HttpError> { + let (parts, body) = request.into_parts(); + let request = Request::from_parts(parts, Body(Full::from(body))); + let mut response = self.inner.request(request).await?; + let headers = std::mem::take(response.headers_mut()); + + let mut http_response = Response::builder() + .status(response.status()) + .body(response.into_body().collect().await?.to_bytes())?; + *http_response.headers_mut() = headers; + + Ok(http_response.error_for_status()?) + } + } + + #[pin_project::pin_project] + pub struct Body(#[pin] Full); + + impl HttpBody for Body { + type Data = Bytes; + type Error = Box; + + #[inline] + fn poll_frame( + self: Pin<&mut Self>, + cx: &mut task::Context<'_>, + ) -> Poll, Self::Error>>> { + self.project().0.poll_frame(cx).map_err(Into::into) + } + + #[inline] + fn is_end_stream(&self) -> bool { + self.0.is_end_stream() + } + + #[inline] + fn size_hint(&self) -> hyper::body::SizeHint { + self.0.size_hint() + } + } +} + +fn otel_create_globals( + config: OtelConfig, + op_state: &mut OpState, +) -> anyhow::Result<()> { + // Parse the `OTEL_EXPORTER_OTLP_PROTOCOL` variable. The opentelemetry_* + // crates don't do this automatically. + // TODO(piscisaureus): enable GRPC support. + let protocol = match env::var("OTEL_EXPORTER_OTLP_PROTOCOL").as_deref() { + Ok("http/protobuf") => Protocol::HttpBinary, + Ok("http/json") => Protocol::HttpJson, + Ok("") | Err(env::VarError::NotPresent) => { + return Ok(()); + } + Ok(protocol) => { + return Err(anyhow!( + "Env var OTEL_EXPORTER_OTLP_PROTOCOL specifies an unsupported protocol: {}", + protocol + )); + } + Err(err) => { + return Err(anyhow!( + "Failed to read env var OTEL_EXPORTER_OTLP_PROTOCOL: {}", + err + )) + } + }; + + // Define the resource attributes that will be attached to all log records. + // These attributes are sourced as follows (in order of precedence): + // * The `service.name` attribute from the `OTEL_SERVICE_NAME` env var. + // * Additional attributes from the `OTEL_RESOURCE_ATTRIBUTES` env var. + // * Default attribute values defined here. + // TODO(piscisaureus): add more default attributes (e.g. script path). + let mut resource = Resource::default(); + + // Add the runtime name and version to the resource attributes. Also override + // the `telemetry.sdk` attributes to include the Deno runtime. + resource = resource.merge(&Resource::new(vec![ + KeyValue::new(PROCESS_RUNTIME_NAME, config.runtime_name), + KeyValue::new(PROCESS_RUNTIME_VERSION, config.runtime_version.clone()), + KeyValue::new( + TELEMETRY_SDK_LANGUAGE, + format!( + "deno-{}", + resource.get(Key::new(TELEMETRY_SDK_LANGUAGE)).unwrap() + ), + ), + KeyValue::new( + TELEMETRY_SDK_NAME, + format!( + "deno-{}", + resource.get(Key::new(TELEMETRY_SDK_NAME)).unwrap() + ), + ), + KeyValue::new( + TELEMETRY_SDK_VERSION, + format!( + "{}-{}", + config.runtime_version, + resource.get(Key::new(TELEMETRY_SDK_VERSION)).unwrap() + ), + ), + ])); + + // The OTLP endpoint is automatically picked up from the + // `OTEL_EXPORTER_OTLP_ENDPOINT` environment variable. Additional headers can + // be specified using `OTEL_EXPORTER_OTLP_HEADERS`. + + let client = hyper_client::HyperClient::new(); + + let span_exporter = HttpExporterBuilder::default() + .with_http_client(client.clone()) + .with_protocol(protocol) + .build_span_exporter()?; + let mut span_processor = + BatchSpanProcessor::builder(span_exporter, OtelSharedRuntime).build(); + span_processor.set_resource(&resource); + op_state.put::(span_processor); + + let log_exporter = HttpExporterBuilder::default() + .with_http_client(client) + .with_protocol(protocol) + .build_log_exporter()?; + let log_processor = + BatchLogProcessor::builder(log_exporter, OtelSharedRuntime).build(); + log_processor.set_resource(&resource); + op_state.put::(log_processor); + + Ok(()) +} + +/// This function is called by the runtime whenever it is about to call +/// `os::process::exit()`, to ensure that all OpenTelemetry logs are properly +/// flushed before the process terminates. +pub fn otel_drop_state(state: &mut OpState) { + if let Some(processor) = state.try_take::() { + let _ = processor.force_flush(); + drop(processor); + } + if let Some(processor) = state.try_take::() { + let _ = processor.force_flush(); + drop(processor); + } +} + +#[op2(fast)] +fn op_otel_log( + state: &mut OpState, + #[string] message: String, + #[smi] level: i32, + #[string] trace_id: &str, + #[string] span_id: &str, + #[smi] trace_flags: u8, +) { + let Some(logger) = state.try_borrow::() else { + log::error!("op_otel_log: OpenTelemetry Logger not available"); + return; + }; + + // Convert the integer log level that ext/console uses to the corresponding + // OpenTelemetry log severity. + let severity = match level { + ..=0 => Severity::Debug, + 1 => Severity::Info, + 2 => Severity::Warn, + 3.. => Severity::Error, + }; + + let mut log_record = LogRecord::default(); + + log_record.observed_timestamp = Some(SystemTime::now()); + log_record.body = Some(message.into()); + log_record.severity_number = Some(severity); + log_record.severity_text = Some(severity.name()); + if let (Ok(trace_id), Ok(span_id)) = + (TraceId::from_hex(trace_id), SpanId::from_hex(span_id)) + { + let span_context = SpanContext::new( + trace_id, + span_id, + TraceFlags::new(trace_flags), + false, + Default::default(), + ); + log_record.trace_context = Some((&span_context).into()); + } + logger.emit( + &mut log_record, + &InstrumentationScope::builder("deno").build(), + ); +} + +struct TemporarySpan(SpanData); + +#[allow(clippy::too_many_arguments)] +#[op2(fast)] +fn op_otel_span_start<'s>( + scope: &mut v8::HandleScope<'s>, + state: &mut OpState, + trace_id: v8::Local<'s, v8::Value>, + span_id: v8::Local<'s, v8::Value>, + parent_span_id: v8::Local<'s, v8::Value>, + #[smi] span_kind: u8, + name: v8::Local<'s, v8::Value>, + start_time: f64, + end_time: f64, +) -> Result<(), anyhow::Error> { + if let Some(temporary_span) = state.try_take::() { + let Some(span_processor) = state.try_borrow::() else { + return Ok(()); + }; + span_processor.on_end(temporary_span.0); + }; + + let trace_id = { + let x = v8::ValueView::new(scope, trace_id.try_cast()?); + match x.data() { + v8::ValueViewData::OneByte(bytes) => { + TraceId::from_hex(&String::from_utf8_lossy(bytes))? + } + _ => return Err(anyhow!("invalid trace_id")), + } + }; + + let span_id = { + let x = v8::ValueView::new(scope, span_id.try_cast()?); + match x.data() { + v8::ValueViewData::OneByte(bytes) => { + SpanId::from_hex(&String::from_utf8_lossy(bytes))? + } + _ => return Err(anyhow!("invalid span_id")), + } + }; + + let parent_span_id = { + let x = v8::ValueView::new(scope, parent_span_id.try_cast()?); + match x.data() { + v8::ValueViewData::OneByte(bytes) => { + let s = String::from_utf8_lossy(bytes); + if s.is_empty() { + SpanId::INVALID + } else { + SpanId::from_hex(&s)? + } + } + _ => return Err(anyhow!("invalid parent_span_id")), + } + }; + + let name = { + let x = v8::ValueView::new(scope, name.try_cast()?); + match x.data() { + v8::ValueViewData::OneByte(bytes) => { + String::from_utf8_lossy(bytes).into_owned() + } + v8::ValueViewData::TwoByte(bytes) => String::from_utf16_lossy(bytes), + } + }; + + let temporary_span = TemporarySpan(SpanData { + span_context: SpanContext::new( + trace_id, + span_id, + TraceFlags::SAMPLED, + false, + Default::default(), + ), + parent_span_id, + span_kind: match span_kind { + 0 => SpanKind::Internal, + 1 => SpanKind::Server, + 2 => SpanKind::Client, + 3 => SpanKind::Producer, + 4 => SpanKind::Consumer, + _ => return Err(anyhow!("invalid span kind")), + }, + name: Cow::Owned(name), + start_time: SystemTime::UNIX_EPOCH + .checked_add(std::time::Duration::from_secs_f64(start_time)) + .ok_or_else(|| anyhow!("invalid start time"))?, + end_time: SystemTime::UNIX_EPOCH + .checked_add(std::time::Duration::from_secs_f64(end_time)) + .ok_or_else(|| anyhow!("invalid start time"))?, + attributes: Vec::new(), + dropped_attributes_count: 0, + events: Default::default(), + links: Default::default(), + status: SpanStatus::Unset, + instrumentation_scope: InstrumentationScope::builder("deno").build(), + }); + state.put(temporary_span); + + Ok(()) +} + +#[op2(fast)] +fn op_otel_span_continue( + state: &mut OpState, + #[smi] status: u8, + #[string] error_description: Cow<'_, str>, +) { + if let Some(temporary_span) = state.try_borrow_mut::() { + temporary_span.0.status = match status { + 0 => SpanStatus::Unset, + 1 => SpanStatus::Ok, + 2 => SpanStatus::Error { + description: Cow::Owned(error_description.into_owned()), + }, + _ => return, + }; + } +} + +macro_rules! attr { + ($scope:ident, $temporary_span:ident, $name:ident, $value:ident) => { + let name = if let Ok(name) = $name.try_cast() { + let view = v8::ValueView::new($scope, name); + match view.data() { + v8::ValueViewData::OneByte(bytes) => { + Some(String::from_utf8_lossy(bytes).into_owned()) + } + v8::ValueViewData::TwoByte(bytes) => { + Some(String::from_utf16_lossy(bytes)) + } + } + } else { + None + }; + let value = if let Ok(string) = $value.try_cast::() { + Some(Value::String(StringValue::from({ + let x = v8::ValueView::new($scope, string); + match x.data() { + v8::ValueViewData::OneByte(bytes) => { + String::from_utf8_lossy(bytes).into_owned() + } + v8::ValueViewData::TwoByte(bytes) => String::from_utf16_lossy(bytes), + } + }))) + } else if let Ok(number) = $value.try_cast::() { + Some(Value::F64(number.value())) + } else if let Ok(boolean) = $value.try_cast::() { + Some(Value::Bool(boolean.is_true())) + } else if let Ok(bigint) = $value.try_cast::() { + let (i64_value, _lossless) = bigint.i64_value(); + Some(Value::I64(i64_value)) + } else { + None + }; + if let (Some(name), Some(value)) = (name, value) { + $temporary_span + .0 + .attributes + .push(KeyValue::new(name, value)); + } else { + $temporary_span.0.dropped_attributes_count += 1; + } + }; +} + +#[op2(fast)] +fn op_otel_span_attribute<'s>( + scope: &mut v8::HandleScope<'s>, + state: &mut OpState, + #[smi] capacity: u32, + key: v8::Local<'s, v8::Value>, + value: v8::Local<'s, v8::Value>, +) { + if let Some(temporary_span) = state.try_borrow_mut::() { + temporary_span.0.attributes.reserve_exact( + (capacity as usize) - temporary_span.0.attributes.capacity(), + ); + attr!(scope, temporary_span, key, value); + } +} + +#[op2(fast)] +fn op_otel_span_attribute2<'s>( + scope: &mut v8::HandleScope<'s>, + state: &mut OpState, + #[smi] capacity: u32, + key1: v8::Local<'s, v8::Value>, + value1: v8::Local<'s, v8::Value>, + key2: v8::Local<'s, v8::Value>, + value2: v8::Local<'s, v8::Value>, +) { + if let Some(temporary_span) = state.try_borrow_mut::() { + temporary_span.0.attributes.reserve_exact( + (capacity as usize) - temporary_span.0.attributes.capacity(), + ); + attr!(scope, temporary_span, key1, value1); + attr!(scope, temporary_span, key2, value2); + } +} + +#[allow(clippy::too_many_arguments)] +#[op2(fast)] +fn op_otel_span_attribute3<'s>( + scope: &mut v8::HandleScope<'s>, + state: &mut OpState, + #[smi] capacity: u32, + key1: v8::Local<'s, v8::Value>, + value1: v8::Local<'s, v8::Value>, + key2: v8::Local<'s, v8::Value>, + value2: v8::Local<'s, v8::Value>, + key3: v8::Local<'s, v8::Value>, + value3: v8::Local<'s, v8::Value>, +) { + if let Some(temporary_span) = state.try_borrow_mut::() { + temporary_span.0.attributes.reserve_exact( + (capacity as usize) - temporary_span.0.attributes.capacity(), + ); + attr!(scope, temporary_span, key1, value1); + attr!(scope, temporary_span, key2, value2); + attr!(scope, temporary_span, key3, value3); + } +} + +#[op2(fast)] +fn op_otel_span_flush(state: &mut OpState) { + let Some(temporary_span) = state.try_take::() else { + return; + }; + + let Some(span_processor) = state.try_borrow::() else { + return; + }; + + span_processor.on_end(temporary_span.0); +} diff --git a/runtime/shared.rs b/runtime/shared.rs index f7d76f67a7..c05f352f1c 100644 --- a/runtime/shared.rs +++ b/runtime/shared.rs @@ -47,6 +47,7 @@ extension!(runtime, "40_signals.js", "40_tty.js", "41_prompt.js", + "telemetry.js", "90_deno_ns.js", "98_global_scope_shared.js", "98_global_scope_window.js", diff --git a/runtime/snapshot.rs b/runtime/snapshot.rs index 251ee5f41c..bb9bf9166f 100644 --- a/runtime/snapshot.rs +++ b/runtime/snapshot.rs @@ -312,6 +312,7 @@ pub fn create_runtime_snapshot( ), ops::fs_events::deno_fs_events::init_ops(), ops::os::deno_os::init_ops(Default::default()), + ops::otel::deno_otel::init_ops(None), ops::permissions::deno_permissions::init_ops(), ops::process::deno_process::init_ops(None), ops::signal::deno_signal::init_ops(), diff --git a/runtime/web_worker.rs b/runtime/web_worker.rs index 61e5c77029..d81c82c501 100644 --- a/runtime/web_worker.rs +++ b/runtime/web_worker.rs @@ -505,6 +505,9 @@ impl WebWorker { ), ops::fs_events::deno_fs_events::init_ops_and_esm(), ops::os::deno_os_worker::init_ops_and_esm(), + ops::otel::deno_otel::init_ops_and_esm( + options.bootstrap.otel_config.clone(), + ), ops::permissions::deno_permissions::init_ops_and_esm(), ops::process::deno_process::init_ops_and_esm( services.npm_process_state_provider, diff --git a/runtime/worker.rs b/runtime/worker.rs index 88a61fa938..82df755faf 100644 --- a/runtime/worker.rs +++ b/runtime/worker.rs @@ -422,6 +422,9 @@ impl MainWorker { ), ops::fs_events::deno_fs_events::init_ops_and_esm(), ops::os::deno_os::init_ops_and_esm(exit_code.clone()), + ops::otel::deno_otel::init_ops_and_esm( + options.bootstrap.otel_config.clone(), + ), ops::permissions::deno_permissions::init_ops_and_esm(), ops::process::deno_process::init_ops_and_esm( services.npm_process_state_provider, diff --git a/runtime/worker_bootstrap.rs b/runtime/worker_bootstrap.rs index 3f3c25c5ea..dc989a1c0f 100644 --- a/runtime/worker_bootstrap.rs +++ b/runtime/worker_bootstrap.rs @@ -1,5 +1,6 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +use crate::ops::otel::OtelConfig; use deno_core::v8; use deno_core::ModuleSpecifier; use serde::Serialize; @@ -118,6 +119,8 @@ pub struct BootstrapOptions { // Used by `deno serve` pub serve_port: Option, pub serve_host: Option, + // OpenTelemetry output options. If `None`, OpenTelemetry is disabled. + pub otel_config: Option, } impl Default for BootstrapOptions { @@ -152,6 +155,7 @@ impl Default for BootstrapOptions { mode: WorkerExecutionMode::None, serve_port: Default::default(), serve_host: Default::default(), + otel_config: None, } } } @@ -193,6 +197,8 @@ struct BootstrapV8<'a>( Option, // serve worker count Option, + // OTEL config + Box<[u8]>, ); impl BootstrapOptions { @@ -219,6 +225,11 @@ impl BootstrapOptions { self.serve_host.as_deref(), serve_is_main, serve_worker_count, + if let Some(otel_config) = self.otel_config.as_ref() { + Box::new([otel_config.console as u8]) + } else { + Box::new([]) + }, ); bootstrap.serialize(ser).unwrap() diff --git a/tests/specs/cli/otel_basic/__test__.jsonc b/tests/specs/cli/otel_basic/__test__.jsonc new file mode 100644 index 0000000000..a9d4fff049 --- /dev/null +++ b/tests/specs/cli/otel_basic/__test__.jsonc @@ -0,0 +1,4 @@ +{ + "args": "run -A main.ts", + "output": "processed\n" +} diff --git a/tests/specs/cli/otel_basic/child.ts b/tests/specs/cli/otel_basic/child.ts new file mode 100644 index 0000000000..72cffd9f0b --- /dev/null +++ b/tests/specs/cli/otel_basic/child.ts @@ -0,0 +1,20 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +async function inner() { + using _span = new Deno.tracing.Span("inner span"); + console.log("log 1"); + await 1; + console.log("log 2"); +} + +Deno.serve({ + port: 0, + onListen({ port }) { + console.log(port.toString()); + }, + handler: async (_req) => { + using _span = new Deno.tracing.Span("outer span"); + await inner(); + return new Response(null, { status: 200 }); + }, +}); diff --git a/tests/specs/cli/otel_basic/deno.json b/tests/specs/cli/otel_basic/deno.json new file mode 100644 index 0000000000..105514e133 --- /dev/null +++ b/tests/specs/cli/otel_basic/deno.json @@ -0,0 +1,4 @@ +{ + "lock": false, + "importMap": "../../../../import_map.json" +} diff --git a/tests/specs/cli/otel_basic/main.ts b/tests/specs/cli/otel_basic/main.ts new file mode 100644 index 0000000000..66ef5c79cc --- /dev/null +++ b/tests/specs/cli/otel_basic/main.ts @@ -0,0 +1,76 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +import { assert, assertEquals } from "@std/assert"; +import { TextLineStream } from "@std/streams/text-line-stream"; + +const logs = []; +const spans = []; +let child: Deno.ChildProcess; + +Deno.serve( + { + port: 0, + async onListen({ port }) { + const command = new Deno.Command(Deno.execPath(), { + args: ["run", "-A", "--unstable-otel", "child.ts"], + env: { + OTEL_EXPORTER_OTLP_PROTOCOL: "http/json", + OTEL_EXPORTER_OTLP_ENDPOINT: `http://localhost:${port}`, + OTEL_BSP_SCHEDULE_DELAY: "10", + OTEL_BLRP_SCHEDULE_DELAY: "10", + }, + stdin: "piped", + stdout: "piped", + stderr: "inherit", + }); + child = command.spawn(); + const lines = child.stdout + .pipeThrough(new TextDecoderStream()) + .pipeThrough(new TextLineStream()) + .getReader(); + const line = await lines.read(); + await fetch(`http://localhost:${line.value}/`); + }, + async handler(req) { + try { + const body = await req.json(); + if (body.resourceLogs) { + logs.push(...body.resourceLogs[0].scopeLogs[0].logRecords); + } + if (body.resourceSpans) { + spans.push(...body.resourceSpans[0].scopeSpans[0].spans); + } + + if (logs.length > 2 && spans.length > 1) { + child.kill(); + + const inner = spans.find((s) => s.name === "inner span"); + const outer = spans.find((s) => s.name === "outer span"); + + assertEquals(inner.traceId, outer.traceId); + assertEquals(inner.parentSpanId, outer.spanId); + + assertEquals(logs[1].body.stringValue, "log 1\n"); + assertEquals(logs[1].traceId, inner.traceId); + assertEquals(logs[1].spanId, inner.spanId); + + assertEquals(logs[2].body.stringValue, "log 2\n"); + assertEquals(logs[2].traceId, inner.traceId); + assertEquals(logs[2].spanId, inner.spanId); + + console.log("processed"); + Deno.exit(0); + } + + return Response.json({ partialSuccess: {} }, { status: 200 }); + } catch (e) { + console.error(e); + Deno.exit(1); + } + }, + }, +); + +setTimeout(() => { + assert(false, "test did not finish in time"); +}, 10e3); diff --git a/tools/core_import_map.json b/tools/core_import_map.json index aae4e63a45..0811672b16 100644 --- a/tools/core_import_map.json +++ b/tools/core_import_map.json @@ -247,6 +247,7 @@ "ext:runtime/41_prompt.js": "../runtime/js/41_prompt.js", "ext:runtime/90_deno_ns.js": "../runtime/js/90_deno_ns.js", "ext:runtime/98_global_scope.js": "../runtime/js/98_global_scope.js", + "ext:runtime/telemetry.js": "../runtime/js/telemetry.js", "ext:deno_node/_util/std_fmt_colors.ts": "../ext/node/polyfills/_util/std_fmt_colors.ts", "@std/archive": "../tests/util/std/archive/mod.ts", "@std/archive/tar": "../tests/util/std/archive/tar.ts",