From bf0760411336ce5ebb1c103f766c8154af478414 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Sat, 16 Sep 2023 02:42:09 +0200 Subject: [PATCH] feat: Add "deno jupyter" subcommand (#20337) This commit adds "deno jupyter" subcommand which provides a Deno kernel for Jupyter notebooks. The implementation is mostly based on Deno's REPL and reuses large parts of it (though there's some clean up that needs to happen in follow up PRs). Not all functionality of Jupyter kernel is implemented and some message type are still not implemented (eg. "inspect_request") but the kernel is fully working and provides all the capatibilities that the Deno REPL has; including TypeScript transpilation and npm packages support. Closes https://github.com/denoland/deno/issues/13016 --------- Co-authored-by: Adam Powers Co-authored-by: Kyle Kelley --- .gitignore | 4 + Cargo.lock | 418 ++++++++-- Cargo.toml | 1 + cli/Cargo.toml | 3 + cli/args/flags.rs | 117 ++- cli/main.rs | 3 + cli/module_loader.rs | 5 +- .../testdata/jupyter/integration_test.ipynb | 620 +++++++++++++++ cli/tools/jupyter/install.rs | 95 +++ cli/tools/jupyter/jupyter_msg.rs | 268 +++++++ cli/tools/jupyter/mod.rs | 139 ++++ .../jupyter/resources/deno-logo-32x32.png | Bin 0 -> 1029 bytes .../jupyter/resources/deno-logo-64x64.png | Bin 0 -> 2066 bytes cli/tools/jupyter/server.rs | 724 ++++++++++++++++++ cli/tools/mod.rs | 1 + cli/tools/repl/mod.rs | 7 +- cli/tools/repl/session.rs | 63 +- ext/node/Cargo.toml | 2 +- 18 files changed, 2364 insertions(+), 106 deletions(-) create mode 100644 cli/tests/testdata/jupyter/integration_test.ipynb create mode 100644 cli/tools/jupyter/install.rs create mode 100644 cli/tools/jupyter/jupyter_msg.rs create mode 100644 cli/tools/jupyter/mod.rs create mode 100644 cli/tools/jupyter/resources/deno-logo-32x32.png create mode 100644 cli/tools/jupyter/resources/deno-logo-64x64.png create mode 100644 cli/tools/jupyter/server.rs diff --git a/.gitignore b/.gitignore index 417bed71c8..eab627b387 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,7 @@ gclient_config.py_entries # JUnit files produced by deno test --junit junit.xml + +# Jupyter files +.ipynb_checkpoints/ +Untitled*.ipynb \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index fbf9ee8f6a..329df26b55 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -57,7 +57,7 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac1f845298e95f983ff1944b728ae08b8cebab80d684f0a832ed0fc74dfa27e2" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "cipher", "cpufeatures", ] @@ -85,13 +85,22 @@ dependencies = [ "aes", ] +[[package]] +name = "ahash" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8fd72866655d1904d6b0997d0b07ba561047d070fbe29de039031c641b61217" +dependencies = [ + "const-random", +] + [[package]] name = "ahash" version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "once_cell", "version_check", ] @@ -301,6 +310,19 @@ dependencies = [ "syn 2.0.33", ] +[[package]] +name = "asynchronous-codec" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06a0daa378f5fd10634e44b0a29b2a87b890657658e072a30d6f26e57ddee182" +dependencies = [ + "bytes", + "futures-sink", + "futures-util", + "memchr", + "pin-project-lite", +] + [[package]] name = "auto_impl" version = "0.5.0" @@ -339,7 +361,7 @@ checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" dependencies = [ "addr2line", "cc", - "cfg-if", + "cfg-if 1.0.0", "libc", "miniz_oxide", "object", @@ -525,6 +547,12 @@ dependencies = [ "libc", ] +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + [[package]] name = "cfg-if" version = "1.0.0" @@ -645,6 +673,28 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28c122c3980598d243d63d9a704629a2d748d101f278052ff068be5a4423ab6f" +[[package]] +name = "const-random" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f590d95d011aa80b063ffe3253422ed5aa462af4e9867d43ce8337562bac77c4" +dependencies = [ + "const-random-macro", + "proc-macro-hack", +] + +[[package]] +name = "const-random-macro" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "615f6e27d000a2bffbc7f2f6a8669179378fa27ee4d0a509e985dfc0a7defb40" +dependencies = [ + "getrandom 0.2.10", + "lazy_static", + "proc-macro-hack", + "tiny-keccak", +] + [[package]] name = "convert_case" version = "0.4.0" @@ -697,28 +747,107 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", +] + +[[package]] +name = "crossbeam" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69323bff1fb41c635347b8ead484a5ca6c3f11914d784170b158d8449ab07f8e" +dependencies = [ + "cfg-if 0.1.10", + "crossbeam-channel 0.4.4", + "crossbeam-deque", + "crossbeam-epoch", + "crossbeam-queue", + "crossbeam-utils 0.7.2", ] [[package]] name = "crossbeam-channel" -version = "0.5.8" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" +checksum = "b153fe7cbef478c567df0f972e02e6d736db11affe43dfc9c56a9374d1adfb87" dependencies = [ - "cfg-if", - "crossbeam-utils", + "crossbeam-utils 0.7.2", + "maybe-uninit", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c02a4d71819009c192cf4872265391563fd6a84c81ff2c0f2a7026ca4c1d85c" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-utils 0.8.10", +] + +[[package]] +name = "crossbeam-deque" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20ff29ded3204c5106278a81a38f4b482636ed4fa1e6cfbeef193291beb29ed" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils 0.7.2", + "maybe-uninit", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace" +dependencies = [ + "autocfg", + "cfg-if 0.1.10", + "crossbeam-utils 0.7.2", + "lazy_static", + "maybe-uninit", + "memoffset 0.5.6", + "scopeguard", +] + +[[package]] +name = "crossbeam-queue" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "774ba60a54c213d409d5353bda12d49cd68d14e45036a285234c8d6f91f92570" +dependencies = [ + "cfg-if 0.1.10", + "crossbeam-utils 0.7.2", + "maybe-uninit", ] [[package]] name = "crossbeam-utils" -version = "0.8.16" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" +checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" dependencies = [ - "cfg-if", + "autocfg", + "cfg-if 0.1.10", + "lazy_static", ] +[[package]] +name = "crossbeam-utils" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d82ee10ce34d7bc12c2122495e7593a9c41347ecdd64185af4ecf72cb1a7f83" +dependencies = [ + "cfg-if 1.0.0", + "once_cell", +] + +[[package]] +name = "crunchy" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" + [[package]] name = "crypto-bigint" version = "0.4.9" @@ -782,7 +911,7 @@ version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "622178105f911d937a42cdb140730ba4a3ed2becd8ae6ce39c7d28b5d75d4588" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "cpufeatures", "curve25519-dalek-derive", "fiat-crypto", @@ -803,13 +932,24 @@ dependencies = [ "syn 2.0.33", ] +[[package]] +name = "dashmap" +version = "3.11.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f260e2fc850179ef410018660006951c1b55b79e8087e87111a2c388994b9b5" +dependencies = [ + "ahash 0.3.8", + "cfg-if 0.1.10", + "num_cpus", +] + [[package]] name = "dashmap" version = "5.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "hashbrown 0.14.0", "lock_api", "once_cell", @@ -835,7 +975,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d" dependencies = [ "serde", - "uuid", + "uuid 1.4.1", ] [[package]] @@ -846,12 +986,14 @@ dependencies = [ "base32", "base64 0.13.1", "bincode", + "bytes", "cache_control", "chrono", "clap", "clap_complete", "clap_complete_fig", "console_static_text", + "data-encoding", "data-url", "deno_ast", "deno_bench_util", @@ -903,7 +1045,7 @@ dependencies = [ "pin-project", "pretty_assertions", "quick-junit", - "rand", + "rand 0.8.5", "regex", "ring", "rustyline", @@ -919,16 +1061,17 @@ dependencies = [ "text_lines", "thiserror", "tokio", - "tokio-util", + "tokio-util 0.7.8", "tower-lsp", "trust-dns-client", "trust-dns-server", "twox-hash", "typed-arena", - "uuid", + "uuid 1.4.1", "walkdir", "winapi", "winres", + "zeromq", "zstd", ] @@ -1012,7 +1155,7 @@ dependencies = [ "async-trait", "deno_core", "tokio", - "uuid", + "uuid 1.4.1", ] [[package]] @@ -1117,7 +1260,7 @@ dependencies = [ "once_cell", "p256 0.11.1", "p384 0.11.2", - "rand", + "rand 0.8.5", "ring", "rsa", "sec1 0.3.0", @@ -1128,7 +1271,7 @@ dependencies = [ "signature 1.6.4", "spki 0.6.0", "tokio", - "uuid", + "uuid 1.4.1", "x25519-dalek", ] @@ -1138,7 +1281,7 @@ version = "0.66.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cc42f49e0aa338e438f59b8367c0ca73c789e9321bd6e1ee086d57733826190" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "deno_ast", "deno_graph", "futures", @@ -1180,7 +1323,7 @@ dependencies = [ "reqwest", "serde", "tokio", - "tokio-util", + "tokio-util 0.7.8", ] [[package]] @@ -1211,7 +1354,7 @@ dependencies = [ "libc", "log", "nix 0.26.2", - "rand", + "rand 0.8.5", "serde", "tokio", "winapi", @@ -1266,14 +1409,14 @@ dependencies = [ "percent-encoding", "phf", "pin-project", - "rand", + "rand 0.8.5", "ring", "serde", "slab", "smallvec", "thiserror", "tokio", - "tokio-util", + "tokio-util 0.7.8", ] [[package]] @@ -1304,7 +1447,7 @@ dependencies = [ "num-bigint", "prost", "prost-build", - "rand", + "rand 0.8.5", "reqwest", "rusqlite", "serde", @@ -1312,7 +1455,7 @@ dependencies = [ "termcolor", "tokio", "url", - "uuid", + "uuid 1.4.1", ] [[package]] @@ -1422,7 +1565,7 @@ dependencies = [ "p384 0.13.0", "path-clean", "pbkdf2", - "rand", + "rand 0.8.5", "regex", "reqwest", "ring", @@ -1529,7 +1672,7 @@ dependencies = [ "test_util", "tokio", "tokio-metrics", - "uuid", + "uuid 1.4.1", "which", "winapi", "winres", @@ -1561,7 +1704,7 @@ dependencies = [ "os_pipe", "path-dedot", "tokio", - "tokio-util", + "tokio-util 0.7.8", ] [[package]] @@ -1624,7 +1767,7 @@ dependencies = [ "futures", "serde", "tokio", - "uuid", + "uuid 1.4.1", "windows-sys", ] @@ -1993,7 +2136,7 @@ version = "0.8.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", ] [[package]] @@ -2014,6 +2157,17 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "enum-primitive-derive" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c375b9c5eadb68d0a6efee2999fef292f45854c3444c86f09d8ab086ba942b0e" +dependencies = [ + "num-traits", + "quote 1.0.33", + "syn 1.0.109", +] + [[package]] name = "env_logger" version = "0.10.0" @@ -2142,7 +2296,7 @@ dependencies = [ "base64 0.21.4", "hyper 0.14.27", "pin-project", - "rand", + "rand 0.8.5", "sha1", "simdutf8", "thiserror", @@ -2156,7 +2310,7 @@ version = "3.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef033ed5e9bad94e55838ca0ca906db0e043f517adda0c8b79c7a8c66c93c1b5" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "rustix", "windows-sys", ] @@ -2193,7 +2347,7 @@ version = "0.2.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4029edd3e734da6fe05b6cd7bd2960760a616bd2ddd0d59a0124746d6272af0" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "libc", "redox_syscall 0.3.5", "windows-sys", @@ -2420,7 +2574,7 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "libc", "wasi 0.9.0+wasi-snapshot-preview1", ] @@ -2431,7 +2585,7 @@ version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "libc", "wasi 0.11.0+wasi-snapshot-preview1", ] @@ -2504,7 +2658,7 @@ dependencies = [ "indexmap 1.9.3", "slab", "tokio", - "tokio-util", + "tokio-util 0.7.8", "tracing", ] @@ -2520,7 +2674,7 @@ version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" dependencies = [ - "ahash", + "ahash 0.8.3", "allocator-api2", ] @@ -2764,7 +2918,7 @@ version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "632089ec08bd62e807311104122fb26d5c911ab172e2b9864be154a575979e29" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "indexmap 1.9.3", "log", "serde", @@ -2830,7 +2984,7 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", ] [[package]] @@ -3010,7 +3164,7 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "winapi", ] @@ -3118,6 +3272,12 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" +[[package]] +name = "maybe-uninit" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" + [[package]] name = "md-5" version = "0.10.5" @@ -3157,6 +3317,15 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a64a92489e2744ce060c349162be1c5f33c6969234104dbd99ddb5feb08b8c15" +[[package]] +name = "memoffset" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "043175f069eda7b85febe4a74abbaeff828d9f8b448515d3151a14a3542811aa" +dependencies = [ + "autocfg", +] + [[package]] name = "memoffset" version = "0.7.1" @@ -3275,7 +3444,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa52e972a9a719cecb6864fb88568781eb706bac2cd1d4f04a648542dbf78069" dependencies = [ "bitflags 1.3.2", - "cfg-if", + "cfg-if 1.0.0", "libc", ] @@ -3286,9 +3455,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfdda3d196821d6af13126e40375cdf7da646a96114af134d5f417a9a1dc8e1a" dependencies = [ "bitflags 1.3.2", - "cfg-if", + "cfg-if 1.0.0", "libc", - "memoffset", + "memoffset 0.7.1", "pin-utils", "static_assertions", ] @@ -3310,7 +3479,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2c66da08abae1c024c01d635253e402341b4060a12e99b31c7594063bf490a" dependencies = [ "bitflags 1.3.2", - "crossbeam-channel", + "crossbeam-channel 0.5.5", "filetime", "fsevent-sys", "inotify", @@ -3339,7 +3508,7 @@ dependencies = [ "autocfg", "num-integer", "num-traits", - "rand", + "rand 0.8.5", "serde", ] @@ -3355,7 +3524,7 @@ dependencies = [ "num-integer", "num-iter", "num-traits", - "rand", + "rand 0.8.5", "serde", "smallvec", "zeroize", @@ -3548,7 +3717,7 @@ version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "instant", "libc", "redox_syscall 0.2.16", @@ -3562,7 +3731,7 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "libc", "redox_syscall 0.3.5", "smallvec", @@ -3663,7 +3832,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d5285893bb5eb82e6aaf5d59ee909a06a16737a8970984dd7746ba9283498d6" dependencies = [ "phf_shared", - "rand", + "rand 0.8.5", ] [[package]] @@ -3782,7 +3951,7 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d52cff9d1d4dee5fe6d03729099f4a310a41179e0a10dbf542039873f2e826fb" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "cpufeatures", "opaque-debug", "universal-hash", @@ -3978,7 +4147,7 @@ dependencies = [ "nextest-workspace-hack", "quick-xml", "thiserror", - "uuid", + "uuid 1.4.1", ] [[package]] @@ -4024,6 +4193,19 @@ dependencies = [ "nibble_vec", ] +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom 0.1.16", + "libc", + "rand_chacha 0.2.2", + "rand_core 0.5.1", + "rand_hc", +] + [[package]] name = "rand" version = "0.8.5" @@ -4031,10 +4213,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", - "rand_chacha", + "rand_chacha 0.3.1", "rand_core 0.6.4", ] +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core 0.5.1", +] + [[package]] name = "rand_chacha" version = "0.3.1" @@ -4063,6 +4255,15 @@ dependencies = [ "getrandom 0.2.10", ] +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core 0.5.1", +] + [[package]] name = "redox_syscall" version = "0.2.16" @@ -4148,7 +4349,7 @@ dependencies = [ "tokio", "tokio-rustls", "tokio-socks", - "tokio-util", + "tokio-util 0.7.8", "tower-service", "url", "wasm-bindgen", @@ -4357,7 +4558,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d1cd5ae51d3f7bf65d7969d579d502168ef578f289452bd8ccc91de28fda20e" dependencies = [ "bitflags 1.3.2", - "cfg-if", + "cfg-if 1.0.0", "clipboard-win", "fd-lock", "libc", @@ -4484,7 +4685,7 @@ version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25996b82292a7a57ed3508f052cfff8640d38d32018784acd714758b43da9c8f" dependencies = [ - "rand", + "rand 0.8.5", "secp256k1-sys", ] @@ -4637,7 +4838,7 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "028f48d513f9678cda28f6e4064755b3fbb2af6acd672f2c209b62323f7aea0f" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "cpufeatures", "digest 0.10.7", ] @@ -4648,7 +4849,7 @@ version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "cpufeatures", "digest 0.10.7", ] @@ -4659,7 +4860,7 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "cpufeatures", "digest 0.10.7", ] @@ -4828,7 +5029,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c886bd4480155fd3ef527d45e9ac8dd7118a898a46530b7b94c3e21866259fce" dependencies = [ "cc", - "cfg-if", + "cfg-if 1.0.0", "libc", "psm", "winapi", @@ -4971,7 +5172,7 @@ checksum = "39cb7fcd56655c8ae7dcf2344f0be6cbff4d9c7cb401fe3ec8e56e1de8dfe582" dependencies = [ "ast_node", "better_scoped_tls", - "cfg-if", + "cfg-if 1.0.0", "either", "from_variant", "new_debug_unreachable", @@ -5165,7 +5366,7 @@ version = "0.192.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "880fd2a588ac88a61cd1d21b10203bbabe31d7adacbd22de3bb4d702bf2c42b4" dependencies = [ - "dashmap", + "dashmap 5.5.3", "indexmap 1.9.3", "once_cell", "petgraph", @@ -5210,7 +5411,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "675b5c755b0448268830e85e59429095d3423c0ce4a850b209c6f0eeab069f63" dependencies = [ "base64 0.13.1", - "dashmap", + "dashmap 5.5.3", "indexmap 1.9.3", "once_cell", "serde", @@ -5410,7 +5611,7 @@ version = "3.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "fastrand", "redox_syscall 0.3.5", "rustix", @@ -5548,6 +5749,15 @@ dependencies = [ "time-core", ] +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + [[package]] name = "tinyvec" version = "1.6.0" @@ -5638,6 +5848,21 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-util" +version = "0.6.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36943ee01a6d67977dd3f84a5a1d2efeb4ada3a1ae771cadfaa535d9d9fc6507" +dependencies = [ + "bytes", + "futures-core", + "futures-io", + "futures-sink", + "log", + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-util" version = "0.7.8" @@ -5707,7 +5932,7 @@ dependencies = [ "async-trait", "auto_impl 0.5.0", "bytes", - "dashmap", + "dashmap 5.5.3", "futures", "httparse", "log", @@ -5716,7 +5941,7 @@ dependencies = [ "serde", "serde_json", "tokio", - "tokio-util", + "tokio-util 0.7.8", "tower", "tower-lsp-macros", ] @@ -5744,7 +5969,7 @@ version = "0.1.37" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -5786,13 +6011,13 @@ version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c408c32e6a9dbb38037cece35740f2cf23c875d8ca134d33631cec83f74d3fe" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "data-encoding", "futures-channel", "futures-util", "lazy_static", "radix_trie", - "rand", + "rand 0.8.5", "thiserror", "time", "tokio", @@ -5807,7 +6032,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4f7f83d1e4a0e4358ac54c5c3681e5d7da5efc5a7a632c90bb6d6669ddd9bc26" dependencies = [ "async-trait", - "cfg-if", + "cfg-if 1.0.0", "data-encoding", "enum-as-inner", "futures-channel", @@ -5816,7 +6041,7 @@ dependencies = [ "idna 0.2.3", "ipnet", "lazy_static", - "rand", + "rand 0.8.5", "serde", "smallvec", "thiserror", @@ -5832,7 +6057,7 @@ version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aff21aa4dcefb0a1afbfac26deb0adc93888c7d295fb63ab273ef276ba2b7cfe" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "futures-util", "ipconfig", "lazy_static", @@ -5855,7 +6080,7 @@ checksum = "99022f9befa6daec2a860be68ac28b1f0d9d7ccf441d8c5a695e35a58d88840d" dependencies = [ "async-trait", "bytes", - "cfg-if", + "cfg-if 1.0.0", "enum-as-inner", "futures-executor", "futures-util", @@ -5881,8 +6106,8 @@ version = "1.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" dependencies = [ - "cfg-if", - "rand", + "cfg-if 1.0.0", + "rand 0.8.5", "static_assertions", ] @@ -6052,6 +6277,15 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" +[[package]] +name = "uuid" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" +dependencies = [ + "getrandom 0.2.10", +] + [[package]] name = "uuid" version = "1.4.1" @@ -6151,7 +6385,7 @@ version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "wasm-bindgen-macro", ] @@ -6176,7 +6410,7 @@ version = "0.4.37" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c02dbc21516f9f1f04f187958890d7e6026df8d16540b7ad9492bc34a67cea03" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "js-sys", "wasm-bindgen", "web-sys", @@ -6389,7 +6623,7 @@ version = "0.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "windows-sys", ] @@ -6466,6 +6700,32 @@ dependencies = [ "syn 2.0.33", ] +[[package]] +name = "zeromq" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "667ece59294ccaf617fcf2e5decc9114a06427c1f68990028b9f12d322686bdc" +dependencies = [ + "async-trait", + "asynchronous-codec", + "bytes", + "crossbeam", + "dashmap 3.11.10", + "enum-primitive-derive", + "futures", + "futures-util", + "lazy_static", + "log", + "num-traits", + "parking_lot 0.11.2", + "rand 0.7.3", + "regex", + "thiserror", + "tokio", + "tokio-util 0.6.10", + "uuid 0.8.2", +] + [[package]] name = "zstd" version = "0.12.4" diff --git a/Cargo.toml b/Cargo.toml index b601edcc22..e896d4c985 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -85,6 +85,7 @@ cbc = { version = "=0.1.2", features = ["alloc"] } chrono = { version = "0.4", default-features = false, features = ["std", "serde", "clock"] } console_static_text = "=0.8.1" data-url = "=0.3.0" +data-encoding = "2.3.3" dlopen = "0.1.8" encoding_rs = "=0.8.33" ecb = "=0.1.2" diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 8906488e9f..58a45538a6 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -65,12 +65,14 @@ async-trait.workspace = true base32 = "=0.4.0" base64.workspace = true bincode = "=1.3.3" +bytes.workspace = true cache_control.workspace = true chrono.workspace = true clap = { version = "=4.3.3", features = ["string"] } clap_complete = "=4.3.1" clap_complete_fig = "=4.3.1" console_static_text.workspace = true +data-encoding.workspace = true data-url.workspace = true dissimilar = "=1.0.4" dprint-plugin-json = "=0.17.4" @@ -120,6 +122,7 @@ twox-hash = "=1.6.3" typed-arena = "=2.0.1" uuid = { workspace = true, features = ["serde"] } walkdir = "=2.3.2" +zeromq = { version = "=0.3.3", default-features = false, features = ["tcp-transport", "tokio-runtime"] } zstd.workspace = true [target.'cfg(windows)'.dependencies] diff --git a/cli/args/flags.rs b/cli/args/flags.rs index b69f1ce8f8..40aa7b8e33 100644 --- a/cli/args/flags.rs +++ b/cli/args/flags.rs @@ -158,6 +158,13 @@ pub struct InstallFlags { pub force: bool, } +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct JupyterFlags { + pub install: bool, + pub kernel: bool, + pub conn_file: Option, +} + #[derive(Clone, Debug, Eq, PartialEq)] pub struct UninstallFlags { pub name: String, @@ -276,6 +283,7 @@ pub enum DenoSubcommand { Init(InitFlags), Info(InfoFlags), Install(InstallFlags), + Jupyter(JupyterFlags), Uninstall(UninstallFlags), Lsp, Lint(LintFlags), @@ -678,7 +686,8 @@ impl Flags { std::env::current_dir().ok() } Bundle(_) | Completions(_) | Doc(_) | Fmt(_) | Init(_) | Install(_) - | Uninstall(_) | Lsp | Lint(_) | Types | Upgrade(_) | Vendor(_) => None, + | Uninstall(_) | Jupyter(_) | Lsp | Lint(_) | Types | Upgrade(_) + | Vendor(_) => None, } } @@ -818,6 +827,7 @@ pub fn flags_from_vec(args: Vec) -> clap::error::Result { "init" => init_parse(&mut flags, &mut m), "info" => info_parse(&mut flags, &mut m), "install" => install_parse(&mut flags, &mut m), + "jupyter" => jupyter_parse(&mut flags, &mut m), "lint" => lint_parse(&mut flags, &mut m), "lsp" => lsp_parse(&mut flags, &mut m), "repl" => repl_parse(&mut flags, &mut m), @@ -919,6 +929,7 @@ fn clap_root() -> Command { .subcommand(init_subcommand()) .subcommand(info_subcommand()) .subcommand(install_subcommand()) + .subcommand(jupyter_subcommand()) .subcommand(uninstall_subcommand()) .subcommand(lsp_subcommand()) .subcommand(lint_subcommand()) @@ -1613,6 +1624,33 @@ These must be added to the path manually if required.") ) } +fn jupyter_subcommand() -> Command { + Command::new("jupyter") + .arg( + Arg::new("install") + .long("install") + .help("Installs kernelspec, requires 'jupyter' command to be available.") + .conflicts_with("kernel") + .action(ArgAction::SetTrue) + ) + .arg( + Arg::new("kernel") + .long("kernel") + .help("Start the kernel") + .conflicts_with("install") + .requires("conn") + .action(ArgAction::SetTrue) + ) + .arg( + Arg::new("conn") + .long("conn") + .help("Path to JSON file describing connection parameters, provided by Jupyter") + .value_parser(value_parser!(PathBuf)) + .value_hint(ValueHint::FilePath) + .conflicts_with("install")) + .about("Deno kernel for Jupyter notebooks") +} + fn uninstall_subcommand() -> Command { Command::new("uninstall") .about("Uninstall a script previously installed with deno install") @@ -3166,6 +3204,18 @@ fn install_parse(flags: &mut Flags, matches: &mut ArgMatches) { }); } +fn jupyter_parse(flags: &mut Flags, matches: &mut ArgMatches) { + let conn_file = matches.remove_one::("conn"); + let kernel = matches.get_flag("kernel"); + let install = matches.get_flag("install"); + + flags.subcommand = DenoSubcommand::Jupyter(JupyterFlags { + install, + kernel, + conn_file, + }); +} + fn uninstall_parse(flags: &mut Flags, matches: &mut ArgMatches) { let root = matches.remove_one::("root"); @@ -7829,4 +7879,69 @@ mod tests { } ); } + + #[test] + fn jupyter() { + let r = flags_from_vec(svec!["deno", "jupyter", "--unstable"]); + assert_eq!( + r.unwrap(), + Flags { + subcommand: DenoSubcommand::Jupyter(JupyterFlags { + install: false, + kernel: false, + conn_file: None, + }), + unstable: true, + ..Flags::default() + } + ); + + let r = flags_from_vec(svec!["deno", "jupyter", "--unstable", "--install"]); + assert_eq!( + r.unwrap(), + Flags { + subcommand: DenoSubcommand::Jupyter(JupyterFlags { + install: true, + kernel: false, + conn_file: None, + }), + unstable: true, + ..Flags::default() + } + ); + + let r = flags_from_vec(svec![ + "deno", + "jupyter", + "--unstable", + "--kernel", + "--conn", + "path/to/conn/file" + ]); + assert_eq!( + r.unwrap(), + Flags { + subcommand: DenoSubcommand::Jupyter(JupyterFlags { + install: false, + kernel: true, + conn_file: Some(PathBuf::from("path/to/conn/file")), + }), + unstable: true, + ..Flags::default() + } + ); + + let r = flags_from_vec(svec![ + "deno", + "jupyter", + "--install", + "--conn", + "path/to/conn/file" + ]); + r.unwrap_err(); + let r = flags_from_vec(svec!["deno", "jupyter", "--kernel",]); + r.unwrap_err(); + let r = flags_from_vec(svec!["deno", "jupyter", "--install", "--kernel",]); + r.unwrap_err(); + } } diff --git a/cli/main.rs b/cli/main.rs index df5dd0b261..98a2e5d480 100644 --- a/cli/main.rs +++ b/cli/main.rs @@ -134,6 +134,9 @@ async fn run_subcommand(flags: Flags) -> Result { DenoSubcommand::Install(install_flags) => spawn_subcommand(async { tools::installer::install_command(flags, install_flags).await }), + DenoSubcommand::Jupyter(jupyter_flags) => spawn_subcommand(async { + tools::jupyter::kernel(flags, jupyter_flags).await + }), DenoSubcommand::Uninstall(uninstall_flags) => spawn_subcommand(async { tools::installer::uninstall(uninstall_flags.name, uninstall_flags.root) }), diff --git a/cli/module_loader.rs b/cli/module_loader.rs index 67811304bc..4a1e0b671b 100644 --- a/cli/module_loader.rs +++ b/cli/module_loader.rs @@ -330,7 +330,10 @@ impl CliModuleLoaderFactory { lib_window: options.ts_type_lib_window(), lib_worker: options.ts_type_lib_worker(), is_inspecting: options.is_inspecting(), - is_repl: matches!(options.sub_command(), DenoSubcommand::Repl(_)), + is_repl: matches!( + options.sub_command(), + DenoSubcommand::Repl(_) | DenoSubcommand::Jupyter(_) + ), prepared_module_loader: PreparedModuleLoader { emitter, graph_container: graph_container.clone(), diff --git a/cli/tests/testdata/jupyter/integration_test.ipynb b/cli/tests/testdata/jupyter/integration_test.ipynb new file mode 100644 index 0000000000..ec6b279735 --- /dev/null +++ b/cli/tests/testdata/jupyter/integration_test.ipynb @@ -0,0 +1,620 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "id": "182aef1d", + "metadata": {}, + "source": [ + "# Integration Tests for Deno Jupyter\n", + "This notebook contains a number of tests to ensure that Jupyter is working as expected. You should be able to select \"Kernel -> Restart Kernel and Run All\" in Jupyter's notebook UI to run the tests." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "d7705d88", + "metadata": {}, + "source": [ + "## Passing Tests" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "669f972e", + "metadata": { + "heading_collapsed": true + }, + "source": [ + "### Simple Tests" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "e7e8a512", + "metadata": { + "hidden": true + }, + "source": [ + "#### This test should print \"hi\".\n", + "If this doesn't work, everything else will probably fail :)" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "a5d38758", + "metadata": { + "hidden": true + }, + "outputs": [ + { + "data": {}, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "hi\n" + ] + } + ], + "source": [ + "console.log(\"hi\")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "bc5ce8e3", + "metadata": { + "hidden": true + }, + "source": [ + "#### Top-level await" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "f7fa885a", + "metadata": { + "hidden": true + }, + "outputs": [ + { + "data": {}, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "x is \u001b[33m42\u001b[39m\n" + ] + } + ], + "source": [ + "let x = await Promise.resolve(42);\n", + "console.log(\"x is\", x);" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "c21455ae", + "metadata": { + "hidden": true + }, + "source": [ + "#### TypeScript transpiling\n", + "Credit to [typescriptlang.org](https://www.typescriptlang.org/docs/handbook/interfaces.html) for this code" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "08a17340", + "metadata": { + "hidden": true + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{ color: \u001b[32m\"red\"\u001b[39m, area: \u001b[33m10000\u001b[39m }" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "interface SquareConfig {\n", + " color?: string;\n", + " width?: number;\n", + "}\n", + " \n", + "function createSquare(config: SquareConfig): { color: string; area: number } {\n", + " return {\n", + " color: config.color || \"red\",\n", + " area: config.width ? config.width * config.width : 20,\n", + " };\n", + "}\n", + " \n", + "createSquare({ colour: \"red\", width: 100 });" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "eaa0ebc0", + "metadata": { + "heading_collapsed": true + }, + "source": [ + "### Return Values" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "52876276", + "metadata": { + "hidden": true + }, + "source": [ + "#### undefined should not return a value" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "bbf2c09b", + "metadata": { + "hidden": true + }, + "outputs": [ + { + "data": {}, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "undefined" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "e175c803", + "metadata": { + "hidden": true + }, + "source": [ + "#### null should return \"null\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "d9801d80", + "metadata": { + "hidden": true + }, + "outputs": [ + { + "data": { + "text/plain": [ + "\u001b[1mnull\u001b[22m" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "null" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "a2a716dc", + "metadata": { + "hidden": true + }, + "source": [ + "#### boolean should return the boolean" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "cfaac330", + "metadata": { + "hidden": true + }, + "outputs": [ + { + "data": { + "text/plain": [ + "\u001b[33mtrue\u001b[39m" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "true" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "8d9f1aba", + "metadata": { + "hidden": true + }, + "source": [ + "#### number should return the number" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "ec3be2da", + "metadata": { + "hidden": true + }, + "outputs": [ + { + "data": { + "text/plain": [ + "\u001b[33m42\u001b[39m" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "42" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "60965915", + "metadata": { + "hidden": true + }, + "source": [ + "#### string should return the string" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "997cf2d7", + "metadata": { + "hidden": true + }, + "outputs": [ + { + "data": { + "text/plain": [ + "\u001b[32m\"this is a test of the emergency broadcast system\"\u001b[39m" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"this is a test of the emergency broadcast system\"" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "fe38dc27", + "metadata": { + "hidden": true + }, + "source": [ + "#### bigint should return the bigint in literal format" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "44b63807", + "metadata": { + "hidden": true + }, + "outputs": [ + { + "data": { + "text/plain": [ + "\u001b[33m31337n\u001b[39m" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "31337n" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "843ccb6c", + "metadata": { + "hidden": true + }, + "source": [ + "#### symbol should return a string describing the symbol" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "e10c0d31", + "metadata": { + "hidden": true + }, + "outputs": [ + { + "data": { + "text/plain": [ + "\u001b[32mSymbol(foo)\u001b[39m" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "Symbol(\"foo\")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "171b817f", + "metadata": { + "hidden": true + }, + "source": [ + "#### object should describe the object inspection" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "81c99233", + "metadata": { + "hidden": true + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{ foo: \u001b[32m\"bar\"\u001b[39m }" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "{foo: \"bar\"}" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "6caeb583", + "metadata": { + "hidden": true + }, + "source": [ + "#### resolve returned promise" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "43c1581b", + "metadata": { + "hidden": true + }, + "outputs": [ + { + "data": { + "text/plain": [ + "Promise { \u001b[32m\"it worked!\"\u001b[39m }" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "Promise.resolve(\"it worked!\")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "9a34b725", + "metadata": { + "hidden": true + }, + "outputs": [ + { + "data": { + "text/plain": [ + "Promise {\n", + " \u001b[36m\u001b[39m Error: it failed!\n", + " at :2:16\n", + "}" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "Promise.reject(new Error(\"it failed!\"));" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "b5c7b819", + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "ename": "Error: this is a test\n at foo (:3:9)\n at :4:3", + "evalue": "", + "output_type": "error", + "traceback": [] + } + ], + "source": [ + "(function foo() {\n", + " throw new Error(\"this is a test\")\n", + "})()" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "72d01fdd", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Promise {\n", + " \u001b[36m\u001b[39m TypeError: Expected string at position 0\n", + " at Object.readFile (ext:deno_fs/30_fs.js:716:29)\n", + " at :2:6\n", + "}" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "Deno.readFile(1)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "28cf59d0-6908-4edc-bb10-c325beeee362", + "metadata": {}, + "outputs": [], + "source": [ + "console.log(\"Hello from Deno!\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8d5485c3-0da3-43fe-8ef5-a61e672f5e81", + "metadata": {}, + "outputs": [], + "source": [ + "console.log(\"%c Hello Deno \", \"background-color: #15803d; color: white;\");" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1401d9d5-6994-4c7b-b55a-db3c16a1e2dc", + "metadata": {}, + "outputs": [], + "source": [ + "\"Cool 🫡\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7afdaa0a-a2a0-4f52-8c7d-b6c5f237aa0d", + "metadata": {}, + "outputs": [], + "source": [ + "console.table([1, 2, 3])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8e93df23-06eb-414b-98d4-51fbebb53d1f", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Deno", + "language": "typescript", + "name": "deno" + }, + "language_info": { + "file_extension": ".ts", + "mimetype": "text/x.typescript", + "name": "typescript", + "nb_converter": "script", + "pygments_lexer": "typescript", + "version": "5.2.2" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": true + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/cli/tools/jupyter/install.rs b/cli/tools/jupyter/install.rs new file mode 100644 index 0000000000..d1777d92d5 --- /dev/null +++ b/cli/tools/jupyter/install.rs @@ -0,0 +1,95 @@ +// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. + +use deno_core::anyhow::bail; +use deno_core::anyhow::Context; +use deno_core::error::AnyError; +use deno_core::serde_json; +use deno_core::serde_json::json; +use std::env::current_exe; +use std::io::Write; +use std::path::Path; +use tempfile::TempDir; + +const DENO_ICON_32: &[u8] = include_bytes!("./resources/deno-logo-32x32.png"); +const DENO_ICON_64: &[u8] = include_bytes!("./resources/deno-logo-64x64.png"); + +pub fn status() -> Result<(), AnyError> { + let output = std::process::Command::new("jupyter") + .args(["kernelspec", "list", "--json"]) + .output() + .context("Failed to get list of installed kernelspecs")?; + let json_output: serde_json::Value = + serde_json::from_slice(&output.stdout) + .context("Failed to parse JSON from kernelspec list")?; + + if let Some(specs) = json_output.get("kernelspecs") { + if let Some(specs_obj) = specs.as_object() { + if specs_obj.contains_key("deno") { + println!("✅ Deno kernel already installed"); + return Ok(()); + } + } + } + + println!("ℹ️ Deno kernel is not yet installed, run `deno jupyter --unstable --install` to set it up"); + Ok(()) +} + +fn install_icon( + dir_path: &Path, + filename: &str, + icon_data: &[u8], +) -> Result<(), AnyError> { + let path = dir_path.join(filename); + let mut file = std::fs::File::create(path)?; + file.write_all(icon_data)?; + Ok(()) +} + +pub fn install() -> Result<(), AnyError> { + let temp_dir = TempDir::new().unwrap(); + let kernel_json_path = temp_dir.path().join("kernel.json"); + + // TODO(bartlomieju): add remaining fields as per + // https://jupyter-client.readthedocs.io/en/stable/kernels.html#kernel-specs + // FIXME(bartlomieju): replace `current_exe` before landing? + let json_data = json!({ + "argv": [current_exe().unwrap().to_string_lossy(), "--unstable", "jupyter", "--kernel", "--conn", "{connection_file}"], + "display_name": "Deno", + "language": "typescript", + }); + + let f = std::fs::File::create(kernel_json_path)?; + serde_json::to_writer_pretty(f, &json_data)?; + install_icon(temp_dir.path(), "icon-32x32.png", DENO_ICON_32)?; + install_icon(temp_dir.path(), "icon-64x64.png", DENO_ICON_64)?; + + let child_result = std::process::Command::new("jupyter") + .args([ + "kernelspec", + "install", + "--user", + "--name", + "deno", + &temp_dir.path().to_string_lossy(), + ]) + .spawn(); + + if let Ok(mut child) = child_result { + let wait_result = child.wait(); + match wait_result { + Ok(status) => { + if !status.success() { + bail!("Failed to install kernelspec, try again."); + } + } + Err(err) => { + bail!("Failed to install kernelspec: {}", err); + } + } + } + + let _ = std::fs::remove_dir(temp_dir); + println!("✅ Deno kernelspec installed successfully."); + Ok(()) +} diff --git a/cli/tools/jupyter/jupyter_msg.rs b/cli/tools/jupyter/jupyter_msg.rs new file mode 100644 index 0000000000..c28dd3b485 --- /dev/null +++ b/cli/tools/jupyter/jupyter_msg.rs @@ -0,0 +1,268 @@ +// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. + +// This file is forked/ported from +// Copyright 2020 The Evcxr Authors. MIT license. + +use bytes::Bytes; +use chrono::Utc; +use data_encoding::HEXLOWER; +use deno_core::anyhow::anyhow; +use deno_core::anyhow::bail; +use deno_core::error::AnyError; +use deno_core::serde_json; +use deno_core::serde_json::json; +use ring::hmac; +use std::fmt; +use uuid::Uuid; + +pub(crate) struct Connection { + pub(crate) socket: S, + /// Will be None if our key was empty (digest authentication disabled). + pub(crate) mac: Option, +} + +impl Connection { + pub(crate) fn new(socket: S, key: &str) -> Self { + let mac = if key.is_empty() { + None + } else { + Some(hmac::Key::new(hmac::HMAC_SHA256, key.as_bytes())) + }; + Connection { socket, mac } + } +} + +struct RawMessage { + zmq_identities: Vec, + jparts: Vec, +} + +impl RawMessage { + pub(crate) async fn read( + connection: &mut Connection, + ) -> Result { + Self::from_multipart(connection.socket.recv().await?, connection) + } + + pub(crate) fn from_multipart( + multipart: zeromq::ZmqMessage, + connection: &Connection, + ) -> Result { + let delimiter_index = multipart + .iter() + .position(|part| &part[..] == DELIMITER) + .ok_or_else(|| anyhow!("Missing delimiter"))?; + let mut parts = multipart.into_vec(); + let jparts: Vec<_> = parts.drain(delimiter_index + 2..).collect(); + let expected_hmac = parts.pop().unwrap(); + // Remove delimiter, so that what's left is just the identities. + parts.pop(); + let zmq_identities = parts; + + let raw_message = RawMessage { + zmq_identities, + jparts, + }; + + if let Some(key) = &connection.mac { + let sig = HEXLOWER.decode(&expected_hmac)?; + let mut msg = Vec::new(); + for part in &raw_message.jparts { + msg.extend(part); + } + + if let Err(err) = hmac::verify(key, msg.as_ref(), sig.as_ref()) { + bail!("{}", err); + } + } + + Ok(raw_message) + } + + async fn send( + self, + connection: &mut Connection, + ) -> Result<(), AnyError> { + let hmac = if let Some(key) = &connection.mac { + let ctx = self.digest(key); + let tag = ctx.sign(); + HEXLOWER.encode(tag.as_ref()) + } else { + String::new() + }; + let mut parts: Vec = Vec::new(); + for part in &self.zmq_identities { + parts.push(part.to_vec().into()); + } + parts.push(DELIMITER.into()); + parts.push(hmac.as_bytes().to_vec().into()); + for part in &self.jparts { + parts.push(part.to_vec().into()); + } + // ZmqMessage::try_from only fails if parts is empty, which it never + // will be here. + let message = zeromq::ZmqMessage::try_from(parts).unwrap(); + connection.socket.send(message).await?; + Ok(()) + } + + fn digest(&self, mac: &hmac::Key) -> hmac::Context { + let mut hmac_ctx = hmac::Context::with_key(mac); + for part in &self.jparts { + hmac_ctx.update(part); + } + hmac_ctx + } +} + +#[derive(Clone)] +pub(crate) struct JupyterMessage { + zmq_identities: Vec, + header: serde_json::Value, + parent_header: serde_json::Value, + metadata: serde_json::Value, + content: serde_json::Value, +} + +const DELIMITER: &[u8] = b""; + +impl JupyterMessage { + pub(crate) async fn read( + connection: &mut Connection, + ) -> Result { + Self::from_raw_message(RawMessage::read(connection).await?) + } + + fn from_raw_message( + raw_message: RawMessage, + ) -> Result { + if raw_message.jparts.len() < 4 { + bail!("Insufficient message parts {}", raw_message.jparts.len()); + } + + Ok(JupyterMessage { + zmq_identities: raw_message.zmq_identities, + header: serde_json::from_slice(&raw_message.jparts[0])?, + parent_header: serde_json::from_slice(&raw_message.jparts[1])?, + metadata: serde_json::from_slice(&raw_message.jparts[2])?, + content: serde_json::from_slice(&raw_message.jparts[3])?, + }) + } + + pub(crate) fn message_type(&self) -> &str { + self.header["msg_type"].as_str().unwrap_or("") + } + + pub(crate) fn code(&self) -> &str { + self.content["code"].as_str().unwrap_or("") + } + + pub(crate) fn cursor_pos(&self) -> usize { + self.content["cursor_pos"].as_u64().unwrap_or(0) as usize + } + + pub(crate) fn comm_id(&self) -> &str { + self.content["comm_id"].as_str().unwrap_or("") + } + + // Creates a new child message of this message. ZMQ identities are not transferred. + pub(crate) fn new_message(&self, msg_type: &str) -> JupyterMessage { + let mut header = self.header.clone(); + header["msg_type"] = serde_json::Value::String(msg_type.to_owned()); + header["username"] = serde_json::Value::String("kernel".to_owned()); + header["msg_id"] = serde_json::Value::String(Uuid::new_v4().to_string()); + header["date"] = serde_json::Value::String(Utc::now().to_rfc3339()); + + JupyterMessage { + zmq_identities: Vec::new(), + header, + parent_header: self.header.clone(), + metadata: json!({}), + content: json!({}), + } + } + + // Creates a reply to this message. This is a child with the message type determined + // automatically by replacing "request" with "reply". ZMQ identities are transferred. + pub(crate) fn new_reply(&self) -> JupyterMessage { + let mut reply = + self.new_message(&self.message_type().replace("_request", "_reply")); + reply.zmq_identities = self.zmq_identities.clone(); + reply + } + + #[must_use = "Need to send this message for it to have any effect"] + pub(crate) fn comm_close_message(&self) -> JupyterMessage { + self.new_message("comm_close").with_content(json!({ + "comm_id": self.comm_id() + })) + } + + pub(crate) fn with_content( + mut self, + content: serde_json::Value, + ) -> JupyterMessage { + self.content = content; + self + } + + pub(crate) async fn send( + &self, + connection: &mut Connection, + ) -> Result<(), AnyError> { + // If performance is a concern, we can probably avoid the clone and to_vec calls with a bit + // of refactoring. + let raw_message = RawMessage { + zmq_identities: self.zmq_identities.clone(), + jparts: vec![ + serde_json::to_string(&self.header) + .unwrap() + .as_bytes() + .to_vec() + .into(), + serde_json::to_string(&self.parent_header) + .unwrap() + .as_bytes() + .to_vec() + .into(), + serde_json::to_string(&self.metadata) + .unwrap() + .as_bytes() + .to_vec() + .into(), + serde_json::to_string(&self.content) + .unwrap() + .as_bytes() + .to_vec() + .into(), + ], + }; + raw_message.send(connection).await + } +} + +impl fmt::Debug for JupyterMessage { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + writeln!( + f, + "\nHeader: {}", + serde_json::to_string_pretty(&self.header).unwrap() + )?; + writeln!( + f, + "Parent header: {}", + serde_json::to_string_pretty(&self.parent_header).unwrap() + )?; + writeln!( + f, + "Metadata: {}", + serde_json::to_string_pretty(&self.metadata).unwrap() + )?; + writeln!( + f, + "Content: {}\n", + serde_json::to_string_pretty(&self.content).unwrap() + )?; + Ok(()) + } +} diff --git a/cli/tools/jupyter/mod.rs b/cli/tools/jupyter/mod.rs new file mode 100644 index 0000000000..b704d58cd5 --- /dev/null +++ b/cli/tools/jupyter/mod.rs @@ -0,0 +1,139 @@ +// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. + +use crate::args::Flags; +use crate::args::JupyterFlags; +use crate::tools::repl; +use crate::util::logger; +use crate::CliFactory; +use deno_core::anyhow::Context; +use deno_core::error::AnyError; +use deno_core::futures::channel::mpsc; +use deno_core::op; +use deno_core::resolve_url_or_path; +use deno_core::serde::Deserialize; +use deno_core::serde_json; +use deno_core::Op; +use deno_core::OpState; +use deno_runtime::permissions::Permissions; +use deno_runtime::permissions::PermissionsContainer; + +mod install; +mod jupyter_msg; +mod server; + +pub async fn kernel( + flags: Flags, + jupyter_flags: JupyterFlags, +) -> Result<(), AnyError> { + if !flags.unstable { + eprintln!( + "Unstable subcommand 'deno jupyter'. The --unstable flag must be provided." + ); + std::process::exit(70); + } + + if !jupyter_flags.install && !jupyter_flags.kernel { + install::status()?; + return Ok(()); + } + + if jupyter_flags.install { + install::install()?; + return Ok(()); + } + + let connection_filepath = jupyter_flags.conn_file.unwrap(); + + // This env var might be set by notebook + if std::env::var("DEBUG").is_ok() { + logger::init(Some(log::Level::Debug)); + } + + let factory = CliFactory::from_flags(flags).await?; + let cli_options = factory.cli_options(); + let main_module = + resolve_url_or_path("./$deno$jupyter.ts", cli_options.initial_cwd()) + .unwrap(); + // TODO(bartlomieju): should we run with all permissions? + let permissions = PermissionsContainer::new(Permissions::allow_all()); + let npm_resolver = factory.npm_resolver().await?.clone(); + let resolver = factory.resolver().await?.clone(); + let worker_factory = factory.create_cli_main_worker_factory().await?; + let (stdio_tx, stdio_rx) = mpsc::unbounded(); + + let conn_file = + std::fs::read_to_string(&connection_filepath).with_context(|| { + format!("Couldn't read connection file: {:?}", connection_filepath) + })?; + let spec: ConnectionSpec = + serde_json::from_str(&conn_file).with_context(|| { + format!( + "Connection file is not a valid JSON: {:?}", + connection_filepath + ) + })?; + + let mut worker = worker_factory + .create_custom_worker( + main_module.clone(), + permissions, + vec![deno_jupyter::init_ops(stdio_tx)], + Default::default(), + ) + .await?; + worker.setup_repl().await?; + let worker = worker.into_main_worker(); + let repl_session = + repl::ReplSession::initialize(cli_options, npm_resolver, resolver, worker) + .await?; + + server::JupyterServer::start(spec, stdio_rx, repl_session).await?; + + Ok(()) +} + +deno_core::extension!(deno_jupyter, + options = { + sender: mpsc::UnboundedSender, + }, + middleware = |op| match op.name { + "op_print" => op_print::DECL, + _ => op, + }, + state = |state, options| { + state.put(options.sender); + }, +); + +#[op] +pub fn op_print( + state: &mut OpState, + msg: String, + is_err: bool, +) -> Result<(), AnyError> { + let sender = state.borrow_mut::>(); + + if is_err { + if let Err(err) = sender.unbounded_send(server::StdioMsg::Stderr(msg)) { + eprintln!("Failed to send stderr message: {}", err); + } + return Ok(()); + } + + if let Err(err) = sender.unbounded_send(server::StdioMsg::Stdout(msg)) { + eprintln!("Failed to send stdout message: {}", err); + } + Ok(()) +} + +#[derive(Debug, Deserialize)] +pub struct ConnectionSpec { + ip: String, + transport: String, + control_port: u32, + shell_port: u32, + stdin_port: u32, + hb_port: u32, + iopub_port: u32, + key: String, +} diff --git a/cli/tools/jupyter/resources/deno-logo-32x32.png b/cli/tools/jupyter/resources/deno-logo-32x32.png new file mode 100644 index 0000000000000000000000000000000000000000..97871a02ee768c20879cadd36be22e7a6c3d3227 GIT binary patch literal 1029 zcmV+g1p51lP)CEpxm98uEtZIok{C>+3_xBpPC4 zV?&h7<)QthQb{Z=EkRa8%XY_rwz5Ib^?F?iKRG$^_q@HmDPJcNi9oKry1G*Fl1%y` zv?b%>Vh924r&Ati*=$ygoM4RYq(pcJMv_|!g@Qtd(J7CK8NR?_hal6}pi#;}J3BjS zj7FnsWLZHpr*-}C5zxNL*n-UwSWcq%;JOi%p?Dw{6k+l6^Rt2u|1rXPeuC}oZ50>T z2bhW5?Y1~NI#Q^0IvsIyb2Fd~vaoM&Z;wwGl?!x*OF}roh7o~Af)EF_S`E=`Jr3r< zkO9OF4-bdp!a_-_0v}_WJ$)z&xH&oJa>Ot!U9+U7shE2DRRX_*aj>FLRS8Sw&S=JRKWmvV{~E;nq;XIKX-kvLOejp7#{3l-9}k0gdwQ?UonEJ zDhP7GEF+rVB9Taf?txe=CKeVJ{F4I>T&LA)RT)L=lTx?a75De|N-<=y1N{!^QvBe` zbJTeA3MMEKs8}CWD=401R@s)m&!BsiMznd;X?8zmFmfM4xTDrPGl7~`=%yg7PI|TC z)IfsLX_9Qj@Vj? zwQsh#JIn$_a8hvEf|M$0I-SPMl2n&@6L^bJTSMqok#&GWtyC&fHi z7o4U(`QNF1oVr|OG_ip`PD4S{=zo2l{}W&Uxu;s1MrT3s00000NkvXXu0mjfCUy!|`xEZMVX zf1VGccq`eN8P48kueE>cx7J=~pOlxd)YaAXvJbQ@YmKb68@d!(yJ78|wWEiJhodCt zmr6k4TK4gswQs>^98zHIH$Is`9zUf4f|dT0wef^Om1k|08KM}+U|-@;Jd?@f*w?>x z@E#c%@dgG4yt%nK|M%tPrS!V)?ryKEtIJEJQr`Xjz3dX)W0Kty_xjU#2ne9Q zVzc5kH8tM#_4WJjg@pxgeSKYe?cUy=Zve>Y>8Uq3IB0{2A7Nc19a4Gt_O&hC`}_M# zenT;pp-6ei$;pY$@R-AOFsI}Vz-|P%@32;D+Xt{Fd3t*CE-o&l*R{2^Ss6+qn9EKu zHz%obg#4jJ7%LNgRSymhl-I#H{C7(Pl%Rpl6Z89O>k?H-E00xeXlSs4HIOknI_me) z8A@~;wA$-I1B`KJE&CjDbEd9KN@Ig zuHV?$@b7_$AcRDcUZ9|?(u7dK+8bInn|&jhpPzq|{ayT@*rukY-q?iS!pP&}Bo5>(G*FJAos@tx;@ zZ~P#%(y|yY%eE*Nk11YDc?9EY{5lLdL+5-M0{F0FLvm4+$M{1-Lq5b!$rSDbw^H3F zM0oxW0b*QqgpqbqI-$KkALHRZCV{cB`4SB#&f8>A%6rm$=z}5zf19KOL?zbyY~}g+ zx#YKvkMl^-rOOVviEn@`{X+b7S$%#(VyUC~t{jaB@fp9aDpCM)t}8N~=;QG``Tf5dxUVOf+Q z1M^uX)e2I`j6`3u*}L-(Fe)X;NhmB6f60)Ggo@DG)6-L8u(ViSlI`7zc+kjidA_KW z@B|E{WXg-xD>0k3DvK{IEtU8v;p zWO+JvQ9YfqD5n)+fSa)OKg40x_O#M&KoNpO;j%X{zmxhF9vue4J{@jR338P20Oo=-Mk=m??x~LzPIJ8Kvp83MY9wKg4H(zoNw6u+QyOCq|Rl z7;&;%mI?>a;3&){B+64;kXKw$1E{n>!4ZGXSlkONhOxHm>SLlHPxAc_M!}-8AOaMy zh>-Uz&;N$?f8-8Cpj5_+tGosTX!a<1G4GM>27j<-JYa|vn6g9HRG5z*FhHEe6&O;H z7C#2M=}Tf@M}~q4Pjg6$_61>%$A~@(<>Pb#GJ<4o6S9+|X+?@ya1D$6iUQ(z%q4qd zCfi`eNrW2c7i9lX7DZWyiQv?TQ+^3zBXWZ=#Pj|t@&pRv-~qU%u=+*lqPpHx5&CK> zn7GvFbjed3JRsSc7XuxXqAiODe1HLPSi08M|5Qrp>I+c}0QbRTc3}+o`>^T6xUnV} z4A5kQ1XBS;_u@oYBBX7&pgiW<#O8iu>TmU2ATWZ_9ZV%OUGn6L02mOD#lSHFf%>qw zePP1-l^G1kT8Az=@cwPe4rQAlTO*)0LB$sd+jqFUK;`_}+{??$?@T6xI^rL!Rf}-M zYQPY=T+TZ?J1c{0OG}II8mp_TWn=gC^?A+B&B_%=pRutqzyIs&tLX0|OmXKjbz-m0 zf$?2qyI5eM$MispCg}nqL%%i``OVSbj1i8X_QsLNgC~#}ffNlcft-FZ>bu2pzB-G9 w=`ZhNUY-3F*Gtv^zdW7rYJK|G=l>O80J5OP-I9b +// Copyright 2020 The Evcxr Authors. MIT license. + +use std::cell::RefCell; +use std::collections::HashMap; +use std::rc::Rc; +use std::sync::Arc; + +use crate::tools::repl; +use crate::tools::repl::cdp; +use deno_core::error::AnyError; +use deno_core::futures; +use deno_core::futures::channel::mpsc; +use deno_core::futures::StreamExt; +use deno_core::serde_json; +use deno_core::serde_json::json; +use deno_core::CancelFuture; +use deno_core::CancelHandle; +use tokio::sync::Mutex; +use zeromq::SocketRecv; +use zeromq::SocketSend; + +use super::jupyter_msg::Connection; +use super::jupyter_msg::JupyterMessage; +use super::ConnectionSpec; + +pub enum StdioMsg { + Stdout(String), + Stderr(String), +} + +pub struct JupyterServer { + execution_count: usize, + last_execution_request: Rc>>, + // This is Arc>, so we don't hold RefCell borrows across await + // points. + iopub_socket: Arc>>, + repl_session: repl::ReplSession, +} + +impl JupyterServer { + pub async fn start( + spec: ConnectionSpec, + mut stdio_rx: mpsc::UnboundedReceiver, + repl_session: repl::ReplSession, + ) -> Result<(), AnyError> { + let mut heartbeat = + bind_socket::(&spec, spec.hb_port).await?; + let shell_socket = + bind_socket::(&spec, spec.shell_port).await?; + let control_socket = + bind_socket::(&spec, spec.control_port).await?; + let _stdin_socket = + bind_socket::(&spec, spec.stdin_port).await?; + let iopub_socket = + bind_socket::(&spec, spec.iopub_port).await?; + let iopub_socket = Arc::new(Mutex::new(iopub_socket)); + let last_execution_request = Rc::new(RefCell::new(None)); + + let cancel_handle = CancelHandle::new_rc(); + let cancel_handle2 = CancelHandle::new_rc(); + + let mut server = Self { + execution_count: 0, + iopub_socket: iopub_socket.clone(), + last_execution_request: last_execution_request.clone(), + repl_session, + }; + + let handle1 = deno_core::unsync::spawn(async move { + if let Err(err) = Self::handle_heartbeat(&mut heartbeat).await { + eprintln!("Heartbeat error: {}", err); + } + }); + + let handle2 = deno_core::unsync::spawn(async move { + if let Err(err) = + Self::handle_control(control_socket, cancel_handle2).await + { + eprintln!("Control error: {}", err); + } + }); + + let handle3 = deno_core::unsync::spawn(async move { + if let Err(err) = server.handle_shell(shell_socket).await { + eprintln!("Shell error: {}", err); + } + }); + + let handle4 = deno_core::unsync::spawn(async move { + while let Some(stdio_msg) = stdio_rx.next().await { + Self::handle_stdio_msg( + iopub_socket.clone(), + last_execution_request.clone(), + stdio_msg, + ) + .await; + } + }); + + let join_fut = + futures::future::try_join_all(vec![handle1, handle2, handle3, handle4]); + + if let Ok(result) = join_fut.or_cancel(cancel_handle).await { + result?; + } + + Ok(()) + } + + async fn handle_stdio_msg( + iopub_socket: Arc>>, + last_execution_request: Rc>>, + stdio_msg: StdioMsg, + ) { + let maybe_exec_result = last_execution_request.borrow().clone(); + if let Some(exec_request) = maybe_exec_result { + let (name, text) = match stdio_msg { + StdioMsg::Stdout(text) => ("stdout", text), + StdioMsg::Stderr(text) => ("stderr", text), + }; + + let result = exec_request + .new_message("stream") + .with_content(json!({ + "name": name, + "text": text + })) + .send(&mut *iopub_socket.lock().await) + .await; + + if let Err(err) = result { + eprintln!("Output {} error: {}", name, err); + } + } + } + + async fn handle_heartbeat( + connection: &mut Connection, + ) -> Result<(), AnyError> { + loop { + connection.socket.recv().await?; + connection + .socket + .send(zeromq::ZmqMessage::from(b"ping".to_vec())) + .await?; + } + } + + async fn handle_control( + mut connection: Connection, + cancel_handle: Rc, + ) -> Result<(), AnyError> { + loop { + let msg = JupyterMessage::read(&mut connection).await?; + match msg.message_type() { + "kernel_info_request" => { + msg + .new_reply() + .with_content(kernel_info()) + .send(&mut connection) + .await?; + } + "shutdown_request" => { + cancel_handle.cancel(); + } + "interrupt_request" => { + eprintln!("Interrupt request currently not supported"); + } + _ => { + eprintln!( + "Unrecognized control message type: {}", + msg.message_type() + ); + } + } + } + } + + async fn handle_shell( + &mut self, + mut connection: Connection, + ) -> Result<(), AnyError> { + loop { + let msg = JupyterMessage::read(&mut connection).await?; + self.handle_shell_message(msg, &mut connection).await?; + } + } + + async fn handle_shell_message( + &mut self, + msg: JupyterMessage, + connection: &mut Connection, + ) -> Result<(), AnyError> { + msg + .new_message("status") + .with_content(json!({"execution_state": "busy"})) + .send(&mut *self.iopub_socket.lock().await) + .await?; + + match msg.message_type() { + "kernel_info_request" => { + msg + .new_reply() + .with_content(kernel_info()) + .send(connection) + .await?; + } + "is_complete_request" => { + msg + .new_reply() + .with_content(json!({"status": "complete"})) + .send(connection) + .await?; + } + "execute_request" => { + self + .handle_execution_request(msg.clone(), connection) + .await?; + } + "comm_open" => { + msg + .comm_close_message() + .send(&mut *self.iopub_socket.lock().await) + .await?; + } + "complete_request" => { + let user_code = msg.code(); + let cursor_pos = msg.cursor_pos(); + + let lsp_completions = self + .repl_session + .language_server + .completions(user_code, cursor_pos) + .await; + + if !lsp_completions.is_empty() { + let matches: Vec = lsp_completions + .iter() + .map(|item| item.new_text.clone()) + .collect(); + + let cursor_start = lsp_completions + .first() + .map(|item| item.range.start) + .unwrap_or(cursor_pos); + + let cursor_end = lsp_completions + .last() + .map(|item| item.range.end) + .unwrap_or(cursor_pos); + + msg + .new_reply() + .with_content(json!({ + "status": "ok", + "matches": matches, + "cursor_start": cursor_start, + "cursor_end": cursor_end, + "metadata": {}, + })) + .send(connection) + .await?; + } else { + let expr = get_expr_from_line_at_pos(user_code, cursor_pos); + // check if the expression is in the form `obj.prop` + let (completions, cursor_start) = if let Some(index) = expr.rfind('.') + { + let sub_expr = &expr[..index]; + let prop_name = &expr[index + 1..]; + let candidates = + get_expression_property_names(&mut self.repl_session, sub_expr) + .await + .into_iter() + .filter(|n| { + !n.starts_with("Symbol(") + && n.starts_with(prop_name) + && n != &*repl::REPL_INTERNALS_NAME + }) + .collect(); + + (candidates, cursor_pos - prop_name.len()) + } else { + // combine results of declarations and globalThis properties + let mut candidates = get_expression_property_names( + &mut self.repl_session, + "globalThis", + ) + .await + .into_iter() + .chain(get_global_lexical_scope_names(&mut self.repl_session).await) + .filter(|n| n.starts_with(expr) && n != &*repl::REPL_INTERNALS_NAME) + .collect::>(); + + // sort and remove duplicates + candidates.sort(); + candidates.dedup(); // make sure to sort first + + (candidates, cursor_pos - expr.len()) + }; + msg + .new_reply() + .with_content(json!({ + "status": "ok", + "matches": completions, + "cursor_start": cursor_start, + "cursor_end": cursor_pos, + "metadata": {}, + })) + .send(connection) + .await?; + } + } + "comm_msg" | "comm_info_request" | "history_request" => { + // We don't handle these messages + } + _ => { + eprintln!("Unrecognized shell message type: {}", msg.message_type()); + } + } + + msg + .new_message("status") + .with_content(json!({"execution_state": "idle"})) + .send(&mut *self.iopub_socket.lock().await) + .await?; + Ok(()) + } + + async fn handle_execution_request( + &mut self, + msg: JupyterMessage, + connection: &mut Connection, + ) -> Result<(), AnyError> { + self.execution_count += 1; + *self.last_execution_request.borrow_mut() = Some(msg.clone()); + + msg + .new_message("execute_input") + .with_content(json!({ + "execution_count": self.execution_count, + "code": msg.code() + })) + .send(&mut *self.iopub_socket.lock().await) + .await?; + + let result = self + .repl_session + .evaluate_line_with_object_wrapping(msg.code()) + .await; + + let evaluate_response = match result { + Ok(eval_response) => eval_response, + Err(err) => { + msg + .new_message("error") + .with_content(json!({ + "ename": err.to_string(), + "evalue": "", + "traceback": [], + })) + .send(&mut *self.iopub_socket.lock().await) + .await?; + msg + .new_reply() + .with_content(json!({ + "status": "error", + "execution_count": self.execution_count, + })) + .send(connection) + .await?; + return Ok(()); + } + }; + + let repl::cdp::EvaluateResponse { + result, + exception_details, + } = evaluate_response.value; + + if exception_details.is_none() { + let output = + get_jupyter_display_or_eval_value(&mut self.repl_session, &result) + .await?; + msg + .new_message("execute_result") + .with_content(json!({ + "execution_count": self.execution_count, + "data": output, + "metadata": {}, + })) + .send(&mut *self.iopub_socket.lock().await) + .await?; + msg + .new_reply() + .with_content(json!({ + "status": "ok", + "execution_count": self.execution_count, + })) + .send(connection) + .await?; + // Let's sleep here for a few ms, so we give a chance to the task that is + // handling stdout and stderr streams to receive and flush the content. + // Otherwise, executing multiple cells one-by-one might lead to output + // from various cells be grouped together in another cell result. + tokio::time::sleep(std::time::Duration::from_millis(5)).await; + } else { + let exception_details = exception_details.unwrap(); + let name = if let Some(exception) = exception_details.exception { + if let Some(description) = exception.description { + description + } else if let Some(value) = exception.value { + value.to_string() + } else { + "undefined".to_string() + } + } else { + "Unknown exception".to_string() + }; + + // TODO(bartlomieju): fill all the fields + msg + .new_message("error") + .with_content(json!({ + "ename": name, + "evalue": "", + "traceback": [], + })) + .send(&mut *self.iopub_socket.lock().await) + .await?; + msg + .new_reply() + .with_content(json!({ + "status": "error", + "execution_count": self.execution_count, + })) + .send(connection) + .await?; + } + + Ok(()) + } +} + +async fn bind_socket( + config: &ConnectionSpec, + port: u32, +) -> Result, AnyError> { + let endpoint = format!("{}://{}:{}", config.transport, config.ip, port); + let mut socket = S::new(); + socket.bind(&endpoint).await?; + Ok(Connection::new(socket, &config.key)) +} + +fn kernel_info() -> serde_json::Value { + json!({ + "status": "ok", + "protocol_version": "5.3", + "implementation_version": crate::version::deno(), + "implementation": "Deno kernel", + "language_info": { + "name": "typescript", + "version": crate::version::TYPESCRIPT, + "mimetype": "text/x.typescript", + "file_extension": ".ts", + "pygments_lexer": "typescript", + "nb_converter": "script" + }, + "help_links": [{ + "text": "Visit Deno manual", + "url": "https://deno.land/manual" + }], + "banner": "Welcome to Deno kernel", + }) +} + +async fn get_jupyter_display( + session: &mut repl::ReplSession, + evaluate_result: &cdp::RemoteObject, +) -> Result>, AnyError> { + let mut data = HashMap::default(); + let response = session + .call_function_on_args( + r#"function (object) {{ + return object[Symbol.for("Jupyter.display")](); + }}"# + .to_string(), + &[evaluate_result.clone()], + ) + .await?; + + if response.exception_details.is_some() { + return Ok(None); + } + + let object_id = response.result.object_id.unwrap(); + + let get_properties_response_result = session + .post_message_with_event_loop( + "Runtime.getProperties", + Some(cdp::GetPropertiesArgs { + object_id, + own_properties: Some(true), + accessor_properties_only: None, + generate_preview: None, + non_indexed_properties_only: Some(true), + }), + ) + .await; + + let Ok(get_properties_response) = get_properties_response_result else { + return Ok(None); + }; + + let get_properties_response: cdp::GetPropertiesResponse = + serde_json::from_value(get_properties_response).unwrap(); + + for prop in get_properties_response.result.into_iter() { + if let Some(value) = &prop.value { + data.insert( + prop.name.to_string(), + value + .value + .clone() + .unwrap_or_else(|| serde_json::Value::Null), + ); + } + } + + if !data.is_empty() { + return Ok(Some(data)); + } + + Ok(None) +} + +async fn get_jupyter_display_or_eval_value( + session: &mut repl::ReplSession, + evaluate_result: &cdp::RemoteObject, +) -> Result, AnyError> { + // Printing "undefined" generates a lot of noise, so let's skip + // these. + if evaluate_result.kind == "undefined" { + return Ok(HashMap::default()); + } + + if let Some(data) = get_jupyter_display(session, evaluate_result).await? { + return Ok(data); + } + + let response = session + .call_function_on_args( + format!( + r#"function (object) {{ + try {{ + return {0}.inspectArgs(["%o", object], {{ colors: !{0}.noColor }}); + }} catch (err) {{ + return {0}.inspectArgs(["%o", err]); + }} + }}"#, + *repl::REPL_INTERNALS_NAME + ), + &[evaluate_result.clone()], + ) + .await?; + let mut data = HashMap::default(); + if let Some(value) = response.result.value { + data.insert("text/plain".to_string(), value); + } + + Ok(data) +} + +// TODO(bartlomieju): dedup with repl::editor +fn get_expr_from_line_at_pos(line: &str, cursor_pos: usize) -> &str { + let start = line[..cursor_pos].rfind(is_word_boundary).unwrap_or(0); + let end = line[cursor_pos..] + .rfind(is_word_boundary) + .map(|i| cursor_pos + i) + .unwrap_or(cursor_pos); + + let word = &line[start..end]; + let word = word.strip_prefix(is_word_boundary).unwrap_or(word); + let word = word.strip_suffix(is_word_boundary).unwrap_or(word); + + word +} + +// TODO(bartlomieju): dedup with repl::editor +fn is_word_boundary(c: char) -> bool { + if matches!(c, '.' | '_' | '$') { + false + } else { + char::is_ascii_whitespace(&c) || char::is_ascii_punctuation(&c) + } +} + +// TODO(bartlomieju): dedup with repl::editor +async fn get_global_lexical_scope_names( + session: &mut repl::ReplSession, +) -> Vec { + let evaluate_response = session + .post_message_with_event_loop( + "Runtime.globalLexicalScopeNames", + Some(cdp::GlobalLexicalScopeNamesArgs { + execution_context_id: Some(session.context_id), + }), + ) + .await + .unwrap(); + let evaluate_response: cdp::GlobalLexicalScopeNamesResponse = + serde_json::from_value(evaluate_response).unwrap(); + evaluate_response.names +} + +// TODO(bartlomieju): dedup with repl::editor +async fn get_expression_property_names( + session: &mut repl::ReplSession, + expr: &str, +) -> Vec { + // try to get the properties from the expression + if let Some(properties) = get_object_expr_properties(session, expr).await { + return properties; + } + + // otherwise fall back to the prototype + let expr_type = get_expression_type(session, expr).await; + let object_expr = match expr_type.as_deref() { + // possibilities: https://chromedevtools.github.io/devtools-protocol/v8/Runtime/#type-RemoteObject + Some("object") => "Object.prototype", + Some("function") => "Function.prototype", + Some("string") => "String.prototype", + Some("boolean") => "Boolean.prototype", + Some("bigint") => "BigInt.prototype", + Some("number") => "Number.prototype", + _ => return Vec::new(), // undefined, symbol, and unhandled + }; + + get_object_expr_properties(session, object_expr) + .await + .unwrap_or_default() +} + +// TODO(bartlomieju): dedup with repl::editor +async fn get_expression_type( + session: &mut repl::ReplSession, + expr: &str, +) -> Option { + evaluate_expression(session, expr) + .await + .map(|res| res.result.kind) +} + +// TODO(bartlomieju): dedup with repl::editor +async fn get_object_expr_properties( + session: &mut repl::ReplSession, + object_expr: &str, +) -> Option> { + let evaluate_result = evaluate_expression(session, object_expr).await?; + let object_id = evaluate_result.result.object_id?; + + let get_properties_response = session + .post_message_with_event_loop( + "Runtime.getProperties", + Some(cdp::GetPropertiesArgs { + object_id, + own_properties: None, + accessor_properties_only: None, + generate_preview: None, + non_indexed_properties_only: Some(true), + }), + ) + .await + .ok()?; + let get_properties_response: cdp::GetPropertiesResponse = + serde_json::from_value(get_properties_response).ok()?; + Some( + get_properties_response + .result + .into_iter() + .map(|prop| prop.name) + .collect(), + ) +} + +// TODO(bartlomieju): dedup with repl::editor +async fn evaluate_expression( + session: &mut repl::ReplSession, + expr: &str, +) -> Option { + let evaluate_response = session + .post_message_with_event_loop( + "Runtime.evaluate", + Some(cdp::EvaluateArgs { + expression: expr.to_string(), + object_group: None, + include_command_line_api: None, + silent: None, + context_id: Some(session.context_id), + return_by_value: None, + generate_preview: None, + user_gesture: None, + await_promise: None, + throw_on_side_effect: Some(true), + timeout: Some(200), + disable_breaks: None, + repl_mode: None, + allow_unsafe_eval_blocked_by_csp: None, + unique_context_id: None, + }), + ) + .await + .ok()?; + let evaluate_response: cdp::EvaluateResponse = + serde_json::from_value(evaluate_response).ok()?; + + if evaluate_response.exception_details.is_some() { + None + } else { + Some(evaluate_response) + } +} diff --git a/cli/tools/mod.rs b/cli/tools/mod.rs index c4a8306ab9..13a37adddb 100644 --- a/cli/tools/mod.rs +++ b/cli/tools/mod.rs @@ -10,6 +10,7 @@ pub mod fmt; pub mod info; pub mod init; pub mod installer; +pub mod jupyter; pub mod lint; pub mod repl; pub mod run; diff --git a/cli/tools/repl/mod.rs b/cli/tools/repl/mod.rs index fb0891fa62..a1e741dfdd 100644 --- a/cli/tools/repl/mod.rs +++ b/cli/tools/repl/mod.rs @@ -13,7 +13,7 @@ use deno_runtime::permissions::Permissions; use deno_runtime::permissions::PermissionsContainer; use rustyline::error::ReadlineError; -mod cdp; +pub(crate) mod cdp; mod channel; mod editor; mod session; @@ -24,8 +24,9 @@ use channel::RustylineSyncMessageHandler; use channel::RustylineSyncResponse; use editor::EditorHelper; use editor::ReplEditor; -use session::EvaluationOutput; -use session::ReplSession; +pub use session::EvaluationOutput; +pub use session::ReplSession; +pub use session::REPL_INTERNALS_NAME; #[allow(clippy::await_holding_refcell_ref)] async fn read_line_and_poll( diff --git a/cli/tools/repl/session.rs b/cli/tools/repl/session.rs index d89cc95c3b..a1b602b4b5 100644 --- a/cli/tools/repl/session.rs +++ b/cli/tools/repl/session.rs @@ -116,9 +116,10 @@ pub fn result_to_evaluation_output( } } -struct TsEvaluateResponse { - ts_code: String, - value: cdp::EvaluateResponse, +#[derive(Debug)] +pub struct TsEvaluateResponse { + pub ts_code: String, + pub value: cdp::EvaluateResponse, } pub struct ReplSession { @@ -305,7 +306,7 @@ impl ReplSession { result_to_evaluation_output(result) } - async fn evaluate_line_with_object_wrapping( + pub async fn evaluate_line_with_object_wrapping( &mut self, line: &str, ) -> Result { @@ -395,29 +396,24 @@ impl ReplSession { Ok(()) } - pub async fn get_eval_value( + pub async fn call_function_on_args( &mut self, - evaluate_result: &cdp::RemoteObject, - ) -> Result { - // TODO(caspervonb) we should investigate using previews here but to keep things - // consistent with the previous implementation we just get the preview result from - // Deno.inspectArgs. + function_declaration: String, + args: &[cdp::RemoteObject], + ) -> Result { + let arguments: Option> = if args.is_empty() { + None + } else { + Some(args.iter().map(|a| a.into()).collect()) + }; + let inspect_response = self .post_message_with_event_loop( "Runtime.callFunctionOn", Some(cdp::CallFunctionOnArgs { - function_declaration: format!( - r#"function (object) {{ - try {{ - return {0}.inspectArgs(["%o", object], {{ colors: !{0}.noColor }}); - }} catch (err) {{ - return {0}.inspectArgs(["%o", err]); - }} - }}"#, - *REPL_INTERNALS_NAME - ), + function_declaration, object_id: None, - arguments: Some(vec![evaluate_result.into()]), + arguments, silent: None, return_by_value: None, generate_preview: None, @@ -432,6 +428,31 @@ impl ReplSession { let response: cdp::CallFunctionOnResponse = serde_json::from_value(inspect_response)?; + Ok(response) + } + + pub async fn get_eval_value( + &mut self, + evaluate_result: &cdp::RemoteObject, + ) -> Result { + // TODO(caspervonb) we should investigate using previews here but to keep things + // consistent with the previous implementation we just get the preview result from + // Deno.inspectArgs. + let response = self + .call_function_on_args( + format!( + r#"function (object) {{ + try {{ + return {0}.inspectArgs(["%o", object], {{ colors: !{0}.noColor }}); + }} catch (err) {{ + return {0}.inspectArgs(["%o", err]); + }} + }}"#, + *REPL_INTERNALS_NAME + ), + &[evaluate_result.clone()], + ) + .await?; let value = response.result.value.unwrap(); let s = value.as_str().unwrap(); diff --git a/ext/node/Cargo.toml b/ext/node/Cargo.toml index 44b56978e5..35d8a2de5c 100644 --- a/ext/node/Cargo.toml +++ b/ext/node/Cargo.toml @@ -19,7 +19,7 @@ aes.workspace = true brotli.workspace = true bytes.workspace = true cbc.workspace = true -data-encoding = "2.3.3" +data-encoding.workspace = true deno_core.workspace = true deno_fetch.workspace = true deno_fs.workspace = true