build: add support for rust proc-macro crates

BUILD.gn

@@ -13,8 +13,8 @@ group("default") {
 
 rust_executable("hyper_hello") {
   source_root = "tools/hyper_hello.rs"
-  extern = [
+  extern_rlib = [
-    "$rust_build:hyper",
+    "hyper",
-    "$rust_build:ring",
+    "ring",
   ]
 }

build_extra/rust/run.py

@@ -7,10 +7,23 @@ import sys
 import os
 import re
 
+if sys.platform == 'win32':
+    # On Windows, when gn is setting up the build toolchain, it produces a set
+    # of environment variables that are required to invoke the right build
+    # toolchain. We need to load those environment variables here too in order
+    # for rustc to be able to successfully invoke the linker tool.
+    # The file is in 'windows environment block' format, which contains
+    # multiple 'key=value' pairs, separated by '\0' bytes, and terminated by
+    # two '\0' bytes at the end.
+    env_pairs = open("environment.x64").read()[:-2].split('\0')
+    env = dict([pair.split('=', 1) for pair in env_pairs])
+else:
+    env = os.environ.copy()
+
 # This is for src/msg.rs to know where to find msg_generated.rs.
 # When building with Cargo this variable is set by build.rs.
-os.environ["GN_OUT_DIR"] = os.path.abspath(".")
+env["GN_OUT_DIR"] = os.path.abspath(".")
-assert os.path.isdir(os.environ["GN_OUT_DIR"])
+assert os.path.isdir(env["GN_OUT_DIR"])
 
 # Set the CARGO_PKG_VERSION env variable if provided as an argument
 # When building with Cargo this variable is set automatically
@@ -18,8 +31,8 @@ args = sys.argv[1:]
 for i, arg in enumerate(args):
     match = re.search('--cargo-pkg-version="?([^"]*)"?', arg)
     if match:
-        os.environ["CARGO_PKG_VERSION"] = match.group(1)
+        env["CARGO_PKG_VERSION"] = match.group(1)
         del args[i]
         break
 
-sys.exit(subprocess.call(args))
+sys.exit(subprocess.call(args, env=env))

build_extra/rust/rust.gni

@@ -11,10 +11,20 @@ declare_args() {
   rust_treat_warnings_as_errors = true
 }
 
-if (is_win) {
+if (is_linux) {
-  executable_suffix = ".exe"
-} else {
   executable_suffix = ""
+  shared_lib_prefix = "lib"
+  shared_lib_suffix = ".so"
+} else if (is_mac) {
+  executable_suffix = ""
+  shared_lib_prefix = "lib"
+  shared_lib_suffix = ".dylib"
+} else if (is_win) {
+  executable_suffix = ".exe"
+  shared_lib_prefix = ""
+  shared_lib_suffix = ".dll"
+} else {
+  assert(false, "Unsupported platform")
 }
 
 # To simplify transitive dependency management with gn, we build all rust
@@ -40,7 +50,7 @@ out_dir = "$root_out_dir/rust_crates"
 # the arguments it passes to the system linker. That's what dummy.rs is for.
 _rustc_info = exec_script("get_rustc_info.py", [], "json")
 
-template("rust_crate") {
+template("_rust_crate") {
   config_name = "${target_name}_config"
   action_name = "${target_name}_rustc"
 
@@ -103,24 +113,47 @@ template("rust_crate") {
     crate_suffix = ""
   }
 
-  # Derive filenames for 'extern' and 'extern_version' linked rust libraries.
+  if (crate_type == "bin") {
-  extern_rlibs = []
+    out_file = "$crate_name$crate_suffix.o"
+    emit_type = "obj"
+  } else if (crate_type == "proc-macro") {
+    out_file = "$shared_lib_prefix$crate_name$crate_suffix$shared_lib_suffix"
+    emit_type = "link"
+  } else if (crate_type == "rlib") {
+    out_file = "lib$crate_name$crate_suffix.rlib"
+    emit_type = "link"
+  }
+  out_path = "$out_dir/$out_file"
+
+  # Merge `invoker.extern` and `invoker.extern_rlib` into a single list.
+  extern = []
   if (defined(invoker.extern)) {
-    foreach(label, invoker.extern) {
+    extern += invoker.extern
-      extern_rlibs += [
+  }
+  if (defined(invoker.extern_rlib)) {
+    foreach(extern_crate_name, invoker.extern_rlib) {
+      extern += [
         {
-          label = label
+          label = "$rust_build:$extern_crate_name"
-          crate_name = get_label_info(label, "name")
+          crate_name = extern_crate_name
-          rlib = "$out_dir/lib$crate_name.rlib"
+          crate_type = "rlib"
         },
       ]
     }
   }
-  if (defined(invoker.extern_version)) {
+
-    foreach(info, invoker.extern_version) {
+  # Add output file info to every entry in the 'extern' list.
-      extern_rlibs += [
+  extern_outputs = []
-        {
+  foreach(info, extern) {
-          crate_name = info.crate_name
+    extern_outputs += [
+      {
+        label = info.label
+        crate_name = info.crate_name
+        crate_type = info.crate_type
+
+        if (!defined(info.crate_version)) {
+          crate_suffix = ""
+        } else {
           crate_version = info.crate_version
           crate_suffix = exec_script("//tools/sha256sum.py",
                                      [
@@ -128,33 +161,33 @@ template("rust_crate") {
                                        "--format=-%.8s",
                                      ],
                                      "trim string")
-          label = ":$crate_name-$crate_version"
+        }
-          rlib = "$out_dir/lib$crate_name$crate_suffix.rlib"
+
-        },
+        if (crate_type == "rlib") {
-      ]
+          out_file = "lib$crate_name$crate_suffix.rlib"
-    }
+        } else if (info.crate_type == "proc_macro") {
+          out_file =
+              "$shared_lib_prefix$crate_name$crate_suffix$shared_lib_suffix"
+        }
+        out_path = "$out_dir/$out_file"
+      },
+    ]
   }
 
   config(config_name) {
-    foreach(extern, extern_rlibs) {
+    foreach(info, extern_outputs) {
-      libs += [ extern.rlib ]
+      if (info.crate_type == "rlib") {
+        libs += [ info.out_path ]
+      }
     }
     lib_dirs = [ out_dir ]
   }
 
-  if (crate_type == "bin") {
-    rustc_output = "$out_dir/$crate_name$crate_suffix.o"
-    emit_type = "obj"
-  } else if (crate_type == "rlib") {
-    rustc_output = "$out_dir/lib$crate_name$crate_suffix.rlib"
-    emit_type = "link"
-  }
-
   source_set(target_name) {
     public_deps = [
       ":$action_name",
     ]
-    libs += [ rustc_output ]
+    libs += [ out_path ]
     all_dependent_configs = [ ":$config_name" ]
   }
 
@@ -164,7 +197,7 @@ template("rust_crate") {
       source_root,
     ]
     outputs = [
-      rustc_output,
+      out_path,
     ]
     depfile = "$out_dir/$crate_name$crate_suffix.d"
 
@@ -205,6 +238,20 @@ template("rust_crate") {
       ]
     }
 
+    if (is_win) {
+      # Proc-macro crates need to be linked by rustc itself, because rustc
+      # doesn't expose all the information necessary to produce the correct
+      # linker invocation ourselves. However gn's setup creates an environment
+      # where link.exe doesn't always work, so we direct rustc to use lld-link,
+      # and explicitly load the proper environment that makes it work in run.py.
+      args += [
+        "-Clinker-flavor=lld-link",
+        "-Clinker=" + rebase_path(
+                "//third_party/llvm-build/Release+Asserts/bin/lld-link.exe",
+                root_build_dir),
+      ]
+    }
+
     if (is_debug) {
       args += [ "-g" ]
     }
@@ -242,23 +289,37 @@ template("rust_crate") {
       }
     }
 
-    # Build the list of '--extern' arguments from the 'extern_rlibs' array.
+    # Build the list of '--extern' arguments from the 'extern_outputs' array.
-    foreach(extern, extern_rlibs) {
+    foreach(info, extern_outputs) {
       args += [
         "--extern",
-        extern.crate_name + "=" + rebase_path(extern.rlib, root_build_dir),
+        info.crate_name + "=" + rebase_path(info.out_path, root_build_dir),
       ]
-      sources += [ extern.rlib ]
+      sources += [ info.out_path ]
-      deps += [ extern.label ]
+      deps += [ info.label ]
     }
   }
 }
 
+template("rust_rlib") {
+  _rust_crate(target_name) {
+    forward_variables_from(invoker, "*")
+    crate_type = "rlib"
+  }
+}
+
+template("rust_proc_macro") {
+  _rust_crate(target_name) {
+    forward_variables_from(invoker, "*")
+    crate_type = "proc-macro"
+  }
+}
+
 template("rust_executable") {
   bin_name = target_name + "_bin"
   bin_label = ":" + bin_name
 
-  rust_crate(bin_name) {
+  _rust_crate(bin_name) {
     crate_type = "bin"
     forward_variables_from(invoker, "*")
   }
@@ -279,11 +340,16 @@ template("rust_executable") {
     deps += [ bin_label ]
 
     if (defined(extern)) {
-      deps += extern
+      foreach(info, extern) {
+        if (info.crate_type == "rlib") {
+          deps += [ info.label ]
+        }
+      }
     }
-    if (defined(extern_version)) {
+
-      foreach(info, extern_version) {
+    if (defined(extern_rlib)) {
-        deps += [ info.label ]
+      foreach(extern_crate_name, extern_rlib) {
+        deps += [ "$rust_build:$extern_crate_name" ]
       }
     }
   }

cli/BUILD.gn

@@ -8,43 +8,48 @@ import("//third_party/v8/snapshot_toolchain.gni")
 import("../deno.gni")
 
 main_extern = [
-  "../core:deno",
+  {
-
+    label = "../core:deno"
-  "$rust_build:ansi_term",
+    crate_name = "deno"
-  "$rust_build:atty",
+    crate_type = "rlib"
-  "$rust_build:clap",
+  },
-  "$rust_build:dirs",
+]
-  "$rust_build:flatbuffers",
+main_extern_rlib = [
-  "$rust_build:futures",
+  "ansi_term",
-  "$rust_build:http",
+  "atty",
-  "$rust_build:hyper",
+  "clap",
-  "$rust_build:hyper_rustls",
+  "dirs",
-  "$rust_build:lazy_static",
+  "flatbuffers",
-  "$rust_build:libc",
+  "futures",
-  "$rust_build:log",
+  "http",
-  "$rust_build:rand",
+  "hyper",
-  "$rust_build:regex",
+  "hyper_rustls",
-  "$rust_build:remove_dir_all",
+  "lazy_static",
-  "$rust_build:ring",
+  "libc",
-  "$rust_build:rustyline",
+  "log",
-  "$rust_build:serde_json",
+  "rand",
-  "$rust_build:source_map_mappings",
+  "regex",
-  "$rust_build:tempfile",
+  "remove_dir_all",
-  "$rust_build:tokio",
+  "ring",
-  "$rust_build:tokio_executor",
+  "rustyline",
-  "$rust_build:tokio_fs",
+  "serde_json",
-  "$rust_build:tokio_io",
+  "source_map_mappings",
-  "$rust_build:tokio_process",
+  "tempfile",
-  "$rust_build:tokio_rustls",
+  "tokio",
-  "$rust_build:tokio_threadpool",
+  "tokio_executor",
-  "$rust_build:url",
+  "tokio_fs",
-  "$rust_build:utime",
+  "tokio_io",
+  "tokio_process",
+  "tokio_rustls",
+  "tokio_threadpool",
+  "url",
+  "utime",
 ]
 if (is_win) {
-  main_extern += [ "$rust_build:winapi" ]
+  main_extern_rlib += [ "winapi" ]
 }
 if (is_posix) {
-  main_extern += [ "$rust_build:nix" ]
+  main_extern_rlib += [ "nix" ]
 }
 
 ts_sources = [
@@ -154,6 +159,7 @@ deno_cargo_info = exec_script("../build_extra/rust/get_cargo_info.py",
 rust_executable("deno") {
   source_root = "main.rs"
   extern = main_extern
+  extern_rlib = main_extern_rlib
   deps = [
     ":deno_deps",
   ]
@@ -170,6 +176,7 @@ rust_executable("deno") {
 rust_test("cli_test") {
   source_root = "main.rs"
   extern = main_extern
+  extern_rlib = main_extern_rlib
   deps = [
     ":deno_deps",
   ]

core/BUILD.gn

@@ -18,19 +18,19 @@ group("deno_core_deps") {
 }
 
 # deno does not depend on flatbuffers nor tokio.
-main_extern = [
+main_extern_rlib = [
-  "$rust_build:futures",
+  "futures",
-  "$rust_build:libc",
+  "libc",
-  "$rust_build:serde_json",
+  "serde_json",
-  "$rust_build:log",
+  "log",
 ]
 
-rust_crate("deno") {
+rust_rlib("deno") {
   source_root = "lib.rs"
   deps = [
     ":deno_core_deps",
   ]
-  extern = main_extern
+  extern_rlib = main_extern_rlib
 }
 
 rust_test("deno_core_test") {
@@ -38,19 +38,25 @@ rust_test("deno_core_test") {
   deps = [
     ":deno_core_deps",
   ]
-  extern = main_extern
+  extern_rlib = main_extern_rlib
 }
 
 http_bench_extern = [
-  "$rust_build:futures",
+  {
-  "$rust_build:lazy_static",
+    label = ":deno"
-  "$rust_build:libc",
+    crate_name = "deno"
-  "$rust_build:log",
+    crate_type = "rlib"
-  "$rust_build:tokio",
+  },
-  ":deno",
+]
+http_bench_extern_rlib = [
+  "futures",
+  "lazy_static",
+  "libc",
+  "log",
+  "tokio",
 ]
 if (is_win) {
-  http_bench_extern += [ "$rust_build:winapi" ]
+  http_bench_extern_rlib += [ "winapi" ]
 }
 
 rust_executable("deno_core_http_bench") {
@@ -59,6 +65,7 @@ rust_executable("deno_core_http_bench") {
     ":deno_core_deps",
   ]
   extern = http_bench_extern
+  extern_rlib = http_bench_extern_rlib
 }
 
 rust_test("deno_core_http_bench_test") {
@@ -67,4 +74,5 @@ rust_test("deno_core_http_bench_test") {
     ":deno_core_deps",
   ]
   extern = http_bench_extern
+  extern_rlib = http_bench_extern_rlib
 }