Make deno_cli installable via crates.io (#2946)

- Fixes cargo publish on deno_typescript, deno_cli_snapshots, and
  deno_cli.
- Combines cli_snapshots and js into one directory.
- Extracts TS version at compile time rather than runtime
- Bumps version awkwardly - it was necessary to test end-to-end
  publishing. Sorry.
- Adds git submodule deno_typescript/typescript

.gitmodules

@@ -7,3 +7,6 @@
 [submodule "js/deps/https/deno.land/x/std"]
 	path = js/deps/https/deno.land/std
 	url = https://github.com/denoland/deno_std.git
+[submodule "deno_typescript/typescript"]
+	path = deno_typescript/typescript
+	url = https://github.com/microsoft/TypeScript.git

Cargo.lock

@@ -266,14 +266,14 @@ dependencies = [
 
 [[package]]
 name = "deno_cli"
-version = "0.18.0"
+version = "0.18.3"
 dependencies = [
  "ansi_term 0.12.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "atty 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)",
  "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "deno 0.18.0",
- "deno_cli_snapshots 0.18.0",
- "deno_typescript 0.18.0",
+ "deno_cli_snapshots 0.18.3",
+ "deno_typescript 0.18.3",
  "dirs 2.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)",
  "fwdansi 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -312,15 +312,15 @@ dependencies = [
 
 [[package]]
 name = "deno_cli_snapshots"
-version = "0.18.0"
+version = "0.18.3"
 dependencies = [
  "deno 0.18.0",
- "deno_typescript 0.18.0",
+ "deno_typescript 0.18.3",
 ]
 
 [[package]]
 name = "deno_typescript"
-version = "0.18.0"
+version = "0.18.3"
 dependencies = [
  "deno 0.18.0",
  "serde 1.0.100 (registry+https://github.com/rust-lang/crates.io-index)",

Cargo.toml

@@ -4,5 +4,5 @@ members = [
   "core",
   "tools/hyper_hello",
   "deno_typescript",
-  "cli_snapshots",
+  "js",
 ]

cli/Cargo.toml

@@ -5,12 +5,18 @@ path = "main.rs"
 
 [package]
 name = "deno_cli"
-version = "0.18.0"
+version = "0.18.3"
+license = "MIT"
+authors = ["the Deno authors"]
 edition = "2018"
+description = "Provides the deno executable"
+repository = "https://github.com/denoland/deno"
 default-run = "deno"
 
 [dependencies]
-deno = { path = "../core" }
+deno = { path = "../core", version = "0.18.0" }
+deno_cli_snapshots = { path = "../js", version = "0.18.3" }
+deno_typescript = { path = "../deno_typescript", version = "0.18.3" }
 
 ansi_term = "0.12.1"
 atty = "0.2.13"
@@ -45,8 +51,6 @@ tokio-rustls = "0.10.0"
 tokio-threadpool = "0.1.15"
 url = "1.7.2"
 utime = "0.2.1"
-deno_cli_snapshots = { path = "../cli_snapshots" }
-deno_typescript = { path = "../deno_typescript" }
 
 [target.'cfg(windows)'.dependencies]
 winapi = "0.3.8"

cli/assets.rs

@@ -1,8 +0,0 @@
-static DENO_RUNTIME: &str = include_str!("../js/lib.deno_runtime.d.ts");
-
-pub fn get_source_code(name: &str) -> Option<&'static str> {
-  match name {
-    "lib.deno_runtime.d.ts" => Some(DENO_RUNTIME),
-    _ => deno_typescript::get_asset(name),
-  }
-}

cli/main.rs

@@ -9,7 +9,7 @@ extern crate futures;
 extern crate serde_json;
 extern crate clap;
 extern crate deno;
-extern crate deno_typescript;
+extern crate deno_cli_snapshots;
 extern crate indexmap;
 #[cfg(unix)]
 extern crate nix;
@@ -21,7 +21,6 @@ extern crate url;
 #[cfg(test)]
 mod integration_tests;
 
-mod assets;
 mod colors;
 pub mod compilers;
 pub mod deno_dir;
@@ -133,7 +132,7 @@ fn create_worker_and_state(
 }
 
 fn types_command() {
-  let content = assets::get_source_code("lib.deno_runtime.d.ts").unwrap();
+  let content = deno_cli_snapshots::get_asset("lib.deno_runtime.d.ts").unwrap();
   println!("{}", content);
 }
 
@@ -405,7 +404,7 @@ fn run_script(flags: DenoFlags, argv: Vec<String>) {
 fn version_command() {
   println!("deno: {}", version::DENO);
   println!("v8: {}", version::v8());
-  println!("typescript: {}", version::typescript());
+  println!("typescript: {}", version::TYPESCRIPT);
 }
 
 fn main() {

cli/ops/compiler.rs

@@ -1,6 +1,5 @@
 // Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
 use super::dispatch_json::{Deserialize, JsonOp, Value};
-use crate::assets;
 use crate::state::ThreadSafeState;
 use crate::tokio_util;
 use deno::*;
@@ -89,7 +88,7 @@ pub fn op_fetch_asset(
   _zero_copy: Option<PinnedBuf>,
 ) -> Result<JsonOp, ErrBox> {
   let args: FetchAssetArgs = serde_json::from_value(args)?;
-  if let Some(source_code) = assets::get_source_code(&args.name) {
+  if let Some(source_code) = deno_cli_snapshots::get_asset(&args.name) {
     Ok(JsonOp::Sync(json!(source_code)))
   } else {
     panic!("op_fetch_asset bad asset {}", args.name)

cli/ops/os.rs

@@ -38,7 +38,7 @@ pub fn op_start(
     "versionFlag": state.flags.version,
     "v8Version": version::v8(),
     "denoVersion": version::DENO,
-    "tsVersion": version::typescript(),
+    "tsVersion": version::TYPESCRIPT,
     "noColor": !colors::use_color(),
     "xevalDelim": state.flags.xeval_delim.clone(),
     "os": BUILD_OS,

cli/version.rs

@@ -1,17 +1,7 @@
 // Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
-use serde_json;
 pub const DENO: &str = env!("CARGO_PKG_VERSION");
+pub const TYPESCRIPT: &str = deno_cli_snapshots::TS_VERSION;
 
 pub fn v8() -> &'static str {
   deno::v8_version()
 }
-
-pub fn typescript() -> String {
-  // TODO: By using include_str! we are including the package.json into
-  // the deno binary using serde to decode it at runtime. This is suboptimal
-  // in space and time. We need to extract the TypeScript version at compile
-  // time instead. This will be easier after #2608.
-  let data = include_str!("../node_modules/typescript/package.json");
-  let pkg: serde_json::Value = serde_json::from_str(data).unwrap();
-  pkg["version"].as_str().unwrap().to_string()
-}

cli_snapshots/Cargo.toml

@@ -1,18 +0,0 @@
-[package]
-name = "deno_cli_snapshots"
-version = "0.18.0"
-license = "MIT"
-authors = ["Ryan Dahl <ry@tinyclouds.org>"]
-edition = "2018"
-description = "Provides snapshots for the deno CLI"
-repository = "https://github.com/ry/deno_typescript"
-
-[lib]
-path = "lib.rs"
-
-[dev-dependencies]
-deno = { path = "../core" }
-
-[build-dependencies]
-deno_typescript = { path = "../deno_typescript" }
-

core/Cargo.toml

@@ -5,7 +5,7 @@ name = "deno"
 version = "0.18.0"
 edition = "2018"
 description = "A secure JavaScript/TypeScript runtime built with V8, Rust, and Tokio"
-authors = ["The deno authors <bertbelder@nodejs.org>"]
+authors = ["the Deno authors"]
 license = "MIT"
 readme = "README.md"
 repository = "https://github.com/denoland/deno"

deno_typescript/Cargo.toml

@@ -1,16 +1,24 @@
 [package]
 name = "deno_typescript"
-version = "0.18.0"
+version = "0.18.3"
 license = "MIT"
 description = "To compile TypeScript to a snapshot during build.rs"
 repository = "https://github.com/ry/deno_typescript"
-authors = ["Ryan Dahl <ry@tinyclouds.org>"]
+authors = ["the Deno authors"]
 edition = "2018"
 
+exclude = [
+  "typescript/tests/*",
+  "typescript/src/*",
+  "typescript/scripts/*",
+  "typescript/doc/*",
+  "typescript/lib/*/*.json",
+]
+
 [lib]
 path = "lib.rs"
 
 [dependencies]
-deno = { path = "../core" }
+deno = { path = "../core", version = "0.18.0" }
 serde_json = "1.0.40"
 serde = { version = "1.0.100", features = ["derive"] }

deno_typescript/amd_runtime.js

@@ -11,6 +11,7 @@ let require;
 /**
  * @type {(name: string, deps: ReadonlyArray<string>, factory: (...deps: any[]) => void) => void}
  */
+// eslint-disable-next-line @typescript-eslint/no-unused-vars
 let define;
 
 (function() {

deno_typescript/compiler_main.js

@@ -44,7 +44,10 @@ function main(configText, rootNames) {
   const emitResult = program.emit();
   handleDiagnostics(host, emitResult.diagnostics);
 
-  dispatch("setEmitResult", emitResult);
+  dispatch(
+    "setEmitResult",
+    Object.assign(emitResult, { tsVersion: ts.version })
+  );
 }
 
 /**

deno_typescript/globals.d.ts

@@ -1,13 +0,0 @@
-// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
-
-// This scopes the `ts` namespace globally, which is where it exists at runtime
-// when building Deno, but the `typescript/lib/typescript.d.ts` is defined as a
-// module
-
-import * as _ts from "typescript";
-
-declare global {
-  namespace ts {
-    export = _ts;
-  }
-}

deno_typescript/jsconfig.json

@@ -1,6 +0,0 @@
-{
-  "compilerOptions": {
-    "strict": true,
-    "target": "esnext"
-  }
-}

deno_typescript/lib.deno_core.d.ts

@@ -21,7 +21,7 @@ interface EvalErrorInfo {
 }
 
 declare interface DenoCore {
-  print(s: string, is_err?: boolean);
+  print(s: string, isErr?: boolean);
   dispatch(
     opId: number,
     control: Uint8Array,
@@ -45,8 +45,6 @@ declare interface DenoCore {
     data?: ArrayBufferView
   ): null | Uint8Array;
 
-  print(x: string, isErr?: boolean): void;
-
   shared: SharedArrayBuffer;
 
   /** Evaluate provided code in the current context.
@@ -63,4 +61,4 @@ declare interface DenoCore {
 declare interface DenoInterface {
   core: DenoCore;
 }
-declare var Deno: DenoInterface;
+declare let Deno: DenoInterface;

deno_typescript/lib.rs

@@ -18,11 +18,16 @@ use std::path::PathBuf;
 use std::sync::Arc;
 use std::sync::Mutex;
 
-static TYPESCRIPT_CODE: &str =
-  include_str!("../third_party/node_modules/typescript/lib/typescript.js");
+static TYPESCRIPT_CODE: &str = include_str!("typescript/lib/typescript.js");
 static COMPILER_CODE: &str = include_str!("compiler_main.js");
 static AMD_RUNTIME_CODE: &str = include_str!("amd_runtime.js");
 
+pub fn ts_version() -> String {
+  let data = include_str!("typescript/package.json");
+  let pkg: serde_json::Value = serde_json::from_str(data).unwrap();
+  pkg["version"].as_str().unwrap().to_string()
+}
+
 #[derive(Debug)]
 pub struct TSState {
   bundle: bool,
@@ -196,15 +201,6 @@ fn write_snapshot(
   Ok(())
 }
 
-macro_rules! inc {
-  ($e:expr) => {
-    Some(include_str!(concat!(
-      "../third_party/node_modules/typescript/lib/",
-      $e
-    )))
-  };
-}
-
 /// Same as get_asset() but returns NotFound intead of None.
 pub fn get_asset2(name: &str) -> Result<&'static str, ErrBox> {
   match get_asset(name) {
@@ -217,8 +213,14 @@ pub fn get_asset2(name: &str) -> Result<&'static str, ErrBox> {
 }
 
 pub fn get_asset(name: &str) -> Option<&'static str> {
+  macro_rules! inc {
+    ($e:expr) => {
+      Some(include_str!(concat!("typescript/lib/", $e)))
+    };
+  }
   match name {
     "lib.deno_core.d.ts" => Some(include_str!("lib.deno_core.d.ts")),
+    "typescript.d.ts" => inc!("typescript.d.ts"),
     "lib.esnext.d.ts" => inc!("lib.esnext.d.ts"),
     "lib.es2019.d.ts" => inc!("lib.es2019.d.ts"),
     "lib.es2018.d.ts" => inc!("lib.es2018.d.ts"),

deno_typescript/ops.rs

@@ -108,8 +108,12 @@ fn resolve_module_names(_s: &mut TSState, v: Value) -> Result<Value, ErrBox> {
   let mut resolved = Vec::<String>::new();
   let referrer = ModuleSpecifier::resolve_url_or_path(&v.containing_file)?;
   for specifier in v.module_names {
-    let ms = ModuleSpecifier::resolve_import(&specifier, referrer.as_str())?;
-    resolved.push(ms.as_str().to_string());
+    if specifier.starts_with("$asset$/") {
+      resolved.push(specifier.clone());
+    } else {
+      let ms = ModuleSpecifier::resolve_import(&specifier, referrer.as_str())?;
+      resolved.push(ms.as_str().to_string());
+    }
   }
   Ok(json!(resolved))
 }

deno_typescript/typescript

@@ -0,0 +1 @@
+Subproject commit cf7b2d4ae91c4f27ba9ae7137ddf9a407815e590

js/Cargo.toml

@@ -0,0 +1,25 @@
+[package]
+name = "deno_cli_snapshots"
+version = "0.18.3"
+license = "MIT"
+authors = ["the Deno authors"]
+edition = "2018"
+description = "Provides snapshots for the deno CLI"
+repository = "https://github.com/denoland/deno"
+exclude = [
+  "deps/https/deno.land/std/fs/testdata/0-link.ts",
+  "deps/https/deno.land/std/fs/testdata/copy_dir_link_file/0.txt",
+]
+
+[lib]
+path = "lib.rs"
+
+[dependencies]
+deno_typescript = { path = "../deno_typescript", version = "0.18.3" }
+
+[dev-dependencies]
+deno = { path = "../core", version = "0.18.0" }
+
+[build-dependencies]
+deno_typescript = { path = "../deno_typescript", version = "0.18.3" }
+

js/README.md

@@ -1,9 +1,10 @@
+# Crate: `deno_cli_snapshots`
+
+## AKA `cli_snapshots` AKA `//js`
+
 This is a small crate which exports just a few static blobs. It contains a
 build.rs file which compiles Deno's internal JavaScript and TypeScript code
 first into a single AMD bundle, and then into a binary V8 Snapshot.
 
 The main Deno executable crate ("cli") depends on this crate and has access to
 all the runtime code.
-
-The //js/ directory should be moved as a sub-directory of this crate, to denote
-the dependency structure. However, that is left to future work.

js/build.rs

@@ -6,17 +6,21 @@ fn main() {
   // To debug snapshot issues uncomment:
   // deno_typescript::trace_serializer();
 
+  println!(
+    "cargo:rustc-env=TS_VERSION={}",
+    deno_typescript::ts_version()
+  );
+
   let c = PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").unwrap());
   let o = PathBuf::from(env::var_os("OUT_DIR").unwrap());
-  let js_dir = c.join("../js");
 
-  let root_names = vec![js_dir.join("main.ts")];
+  let root_names = vec![c.join("main.ts")];
   let bundle = o.join("CLI_SNAPSHOT.js");
   let state = deno_typescript::compile_bundle(&bundle, root_names).unwrap();
   assert!(bundle.exists());
   deno_typescript::mksnapshot_bundle(&bundle, state).unwrap();
 
-  let root_names = vec![js_dir.join("compiler.ts")];
+  let root_names = vec![c.join("compiler.ts")];
   let bundle = o.join("COMPILER_SNAPSHOT.js");
   let state = deno_typescript::compile_bundle(&bundle, root_names).unwrap();
   assert!(bundle.exists());

js/lib.rs

@@ -1,3 +1,5 @@
+pub const TS_VERSION: &str = env!("TS_VERSION");
+
 pub static CLI_SNAPSHOT: &[u8] =
   include_bytes!(concat!(env!("OUT_DIR"), "/CLI_SNAPSHOT.bin"));
 pub static CLI_SNAPSHOT_MAP: &[u8] =
@@ -12,6 +14,16 @@ pub static COMPILER_SNAPSHOT_MAP: &[u8] =
 pub static COMPILER_SNAPSHOT_DTS: &[u8] =
   include_bytes!(concat!(env!("OUT_DIR"), "/COMPILER_SNAPSHOT.d.ts"));
 
+static DENO_RUNTIME: &str = include_str!("lib.deno_runtime.d.ts");
+
+/// Same as deno_typescript::get_asset but also has lib.deno_runtime.d.ts
+pub fn get_asset(name: &str) -> Option<&'static str> {
+  match name {
+    "lib.deno_runtime.d.ts" => Some(DENO_RUNTIME),
+    _ => deno_typescript::get_asset(name),
+  }
+}
+
 #[test]
 fn cli_snapshot() {
   let mut isolate =

js/ts_global.d.ts

@@ -4,8 +4,13 @@
 // when building Deno, but the `typescript/lib/typescript.d.ts` is defined as a
 // module.
 
+// Warning! This is a magical import. We don't want to have multiple copies of
+// typescript.d.ts around the repo, there's already one in
+// deno_typescript/typescript/lib/typescript.d.ts. Ideally we could simply point
+// to that in this import specifier, but "cargo package" is very strict and
+// requires all files to be present in a crate's subtree.
 // eslint-disable-next-line @typescript-eslint/no-unused-vars
-import * as ts_ from "../node_modules/typescript/lib/typescript.d.ts";
+import * as ts_ from "$asset$/typescript.d.ts";
 
 declare global {
   namespace ts {

js/version_test.ts

@@ -1,7 +1,7 @@
 import { test, assert } from "./test_util.ts";
 
 test(function version(): void {
-  const pattern = /^\d+\.\d+\.\d+$/;
+  const pattern = /^\d+\.\d+\.\d+/;
   assert(pattern.test(Deno.version.deno));
   assert(pattern.test(Deno.version.v8));
   assert(pattern.test(Deno.version.typescript));

tools/cargo_publish_others.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python
+# Publishes 'deno_cli', 'deno_cli_snapshots', and 'deno_typescript' crates.
+# DOES NOT PUBLISH 'deno' crate see tools/cargo_package.py for that.
+
+import os
+import sys
+import argparse
+from util import run, root_path
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--dry-run", action="store_true")
+    args = parser.parse_args()
+
+    cargo_publish = ["cargo", "publish"]
+    if args.dry_run:
+        cargo_publish += ["--dry-run"]
+
+    # Publish the deno_typescript crate.
+    os.chdir(os.path.join(root_path, "deno_typescript"))
+    run(cargo_publish)
+
+    # Publish the deno_cli_snapshots crate.
+    os.chdir(os.path.join(root_path, "js"))
+    run(cargo_publish)
+
+    # Publish the deno_cli crate.
+    os.chdir(os.path.join(root_path, "cli"))
+    run(cargo_publish)
+
+
+if __name__ == '__main__':
+    sys.exit(main())

website/index.html

@@ -100,10 +100,12 @@ href="https://deno.land/x/install/install.sh">https://deno.land/x/install/instal
       <p>Or using PowerShell:</p>
       <pre>iwr <a
 href="https://deno.land/x/install/install.ps1">https://deno.land/x/install/install.ps1</a> -useb | iex</pre>
-      <p>Using <a href="https://brew.sh/">Homebrew</a> (mac):</p>
+      <p>Using <a href="https://formulae.brew.sh/formula/deno">Homebrew</a> (mac):</p>
       <pre>brew install deno</pre>
       <p>Using <a href="https://scoop.sh/">Scoop</a> (windows):
       <pre>scoop install deno</pre>
+      <p>Using <a href="https://crates.io/crates/deno_cli">Cargo</a>:
+      <pre>cargo install deno_cli</pre>
       <p>See <a href="https://github.com/denoland/deno_install">deno_install</a> for more installation options.</p>
 
       <h2 id="example">Example <a href="#example">#</a></h2>

website/manual.md

@@ -120,7 +120,14 @@ Using [Homebrew](https://brew.sh/) (mac):
 brew install deno
 ```
 
-Deno can also be installed manually, by downloading a tarball or zip file at
+To install from source:
+
+```shell
+cargo install deno_cli
+```
+
+Deno binaries can also be installed manually, by downloading a tarball or zip
+file at
 [github.com/denoland/deno/releases](https://github.com/denoland/deno/releases).
 These packages contain just a single executable file. You will have to set the
 executable bit on Mac and Linux.