feat: rust bindgen and publish flow (#1507)

2024-11-21 15:04:33 -05:00 · 2024-06-26 19:49:06 -07:00 · 2024-06-26 19:49:06 -07:00 · 0b440db772
commit 0b440db772
parent b8d49899d3
10 changed files with 297 additions and 62 deletions
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@ -202,7 +202,7 @@ jobs:
        if: startsWith(matrix.config.os, 'ubuntu')
        run: |
          sudo apt-get install -y clang-format
-          clang-format --Werror --dry-run src/binding.cc
+          clang-format --verbose --Werror --dry-run src/*.cc src/*.hpp

      - name: Test (ASAN)
        env:
@ -231,11 +231,13 @@ jobs:

      - name: Prepare binary publish
        if: matrix.config.variant == 'debug' || matrix.config.variant == 'release'
-        run: gzip -9c
-          target/${{ matrix.config.target }}/${{ matrix.config.variant }}/gn_out/obj/${{ env.LIB_NAME }}.${{ env.LIB_EXT }} >
-          target/${{ env.LIB_NAME }}_${{ matrix.config.variant }}_${{ matrix.config.target }}.${{ env.LIB_EXT }}.gz &&
+        run: |
+          gzip -9c target/${{ matrix.config.target }}/${{ matrix.config.variant }}/gn_out/obj/${{ env.LIB_NAME }}.${{ env.LIB_EXT }} > target/${{ env.LIB_NAME }}_${{ matrix.config.variant }}_${{ matrix.config.target }}.${{ env.LIB_EXT }}.gz
          ls -l target/${{ env.LIB_NAME }}_${{ matrix.config.variant }}_${{ matrix.config.target }}.${{ env.LIB_EXT }}.gz

+          cp target/${{ matrix.config.target }}/${{ matrix.config.variant}}/gn_out/src_binding.rs target/src_binding_${{ matrix.config.variant }}_${{ matrix.config.target }}.rs
+          ls -l target/src_binding_${{ matrix.config.variant }}_${{ matrix.config.target }}.rs
+
      - name: Binary publish
        uses: softprops/action-gh-release@v0.1.15
        if: >-
@ -247,25 +249,49 @@ jobs:
        with:
          files: target/${{ env.LIB_NAME }}_${{ matrix.config.variant }}_${{ matrix.config.target }}.${{ env.LIB_EXT }}.gz

-      # TODO: add clang-format and maybe cpplint.
+      - name: Upload CI artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: src_binding_${{ matrix.config.variant }}_${{ matrix.config.target }}.rs
+          path: target/src_binding_${{ matrix.config.variant }}_${{ matrix.config.target }}.rs

-      - name: Cargo Package
-        if: >-
-          github.repository == 'denoland/rusty_v8' &&
-          startsWith(matrix.config.target , 'x86_64') &&
-          matrix.config.variant == 'debug' &&
-          runner.os == 'Linux'
-        run: cargo package -vv --locked
+  publish:
+    needs: build
+    runs-on: ${{ github.repository == 'denoland/rusty_v8' && 'ubuntu-22.04-xl' || 'ubuntu-22.04' }}
+    steps:
+      - name: Configure git
+        run: git config --global core.symlinks true
+
+      - name: Clone repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 10
+          submodules: recursive
+
+      - name: Install rust
+        uses: dsherret/rust-toolchain-file@v1
+
+      - name: Install python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.11.x
+          architecture: x64
+
+      - name: Download CI artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: gen
+          pattern: src_binding_*.rs
+          merge-multiple: true

      - name: Publish
-        # Only publish on x64 linux when there's a git tag:
-        if: >-
-          startsWith(github.ref, 'refs/tags/') &&
-          github.repository == 'denoland/rusty_v8' &&
-          startsWith(matrix.config.target , 'x86_64') &&
-          !endsWith(matrix.config.target , 'android') &&
-          matrix.config.variant == 'debug' &&
-          runner.os == 'Linux'
+        if: github.repository == 'denoland/rusty_v8'
        env:
          CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
-        run: cargo publish -vv
+          DRY_RUN: ${{ startsWith(github.ref, 'refs/tags/') == false }}
+        run: |
+          args="-vv --locked --allow-dirty"
+          if [ "$DRY_RUN" = "true" ]; then
+            args="$args --dry-run"
+          fi
+          cargo publish $args
--- a/Cargo.lock
+++ b/Cargo.lock
@ -80,6 +80,29 @@ version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f1fdabc7756949593fe60f30ec81974b613357de856987752631dea1e3394c80"

+[[package]]
+name = "bindgen"
+version = "0.69.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0"
+dependencies = [
+ "bitflags 2.5.0",
+ "cexpr",
+ "clang-sys",
+ "itertools",
+ "lazy_static",
+ "lazycell",
+ "log",
+ "prettyplease",
+ "proc-macro2",
+ "quote",
+ "regex",
+ "rustc-hash",
+ "shlex",
+ "syn 2.0.60",
+ "which 4.4.2",
+]
+
 [[package]]
 name = "bit-set"
 version = "0.5.3"
@ -153,6 +176,15 @@ version = "1.0.95"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d32a725bc159af97c3e629873bb9f88fb8cf8a4867175f76dc987815ea07c83b"

+[[package]]
+name = "cexpr"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766"
+dependencies = [
+ "nom",
+]
+
 [[package]]
 name = "cfg-if"
 version = "0.1.10"
@ -171,6 +203,17 @@ version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e"

+[[package]]
+name = "clang-sys"
+version = "1.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4"
+dependencies = [
+ "glob",
+ "libc",
+ "libloading 0.8.3",
+]
+
 [[package]]
 name = "cocoa"
 version = "0.24.1"
@ -604,6 +647,15 @@ dependencies = [
 "web-sys",
 ]

+[[package]]
+name = "itertools"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569"
+dependencies = [
+ "either",
+]
+
 [[package]]
 name = "itoa"
 version = "1.0.11"
@ -641,6 +693,12 @@ version = "1.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"

+[[package]]
+name = "lazycell"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
+
 [[package]]
 name = "libc"
 version = "0.2.153"
@ -736,6 +794,12 @@ dependencies = [
 "objc",
 ]

+[[package]]
+name = "minimal-lexical"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
+
 [[package]]
 name = "miniz_oxide"
 version = "0.7.3"
@ -866,6 +930,16 @@ dependencies = [
 "memoffset",
 ]

+[[package]]
+name = "nom"
+version = "7.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
+dependencies = [
+ "memchr",
+ "minimal-lexical",
+]
+
 [[package]]
 name = "num-traits"
 version = "0.2.18"
@ -978,6 +1052,16 @@ version = "0.2.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5da3b0203fd7ee5720aa0b5e790b591aa5d3f41c3ed2c34a3a393382198af2f7"

+[[package]]
+name = "prettyplease"
+version = "0.2.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e"
+dependencies = [
+ "proc-macro2",
+ "syn 2.0.60",
+]
+
 [[package]]
 name = "proc-macro-crate"
 version = "1.3.1"
@ -1081,6 +1165,12 @@ version = "0.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f1382d1f0a252c4bf97dc20d979a2fdd05b024acd7c2ed0f7595d7817666a157"

+[[package]]
+name = "rustc-hash"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
+
 [[package]]
 name = "rustix"
 version = "0.38.33"
@ -1167,6 +1257,12 @@ dependencies = [
 "serde",
 ]

+[[package]]
+name = "shlex"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
+
 [[package]]
 name = "slotmap"
 version = "1.0.7"
@ -1353,6 +1449,7 @@ name = "v8"
 version = "0.94.0"
 dependencies = [
 "align-data",
+ "bindgen",
 "bitflags 2.5.0",
 "bytes",
 "fslock",
@ -1362,7 +1459,7 @@ dependencies = [
 "once_cell",
 "rustversion",
 "trybuild",
- "which",
+ "which 6.0.1",
 ]

 [[package]]
@ -1615,6 +1712,18 @@ dependencies = [
 "bitflags 1.3.2",
 ]

+[[package]]
+name = "which"
+version = "4.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7"
+dependencies = [
+ "either",
+ "home",
+ "once_cell",
+ "rustix",
+]
+
 [[package]]
 name = "which"
 version = "6.0.1"
--- a/Cargo.toml
+++ b/Cargo.toml
@ -96,6 +96,7 @@ gzip-header = "1.0.0"
 fslock = "0.2"
 which = "6"
 home = "0"
+bindgen = "0.69"

 [dev-dependencies]
 miniz_oxide = "0.7.3"
@ -107,6 +108,7 @@ trybuild = "1.0.96"
 which = "6"
 home = "0"
 rustversion = "1"
+bindgen = "0.69"

 [[example]]
 name = "hello_world"
--- a/build.rs
+++ b/build.rs
@ -72,6 +72,7 @@ fn main() {

  // Early exit
  if is_cargo_doc || is_rls {
+    print_prebuilt_src_binding_path();
    return;
  }

@ -79,6 +80,10 @@ fn main() {

  // Don't attempt rebuild but link
  if is_trybuild {
+    println!(
+      "cargo:rustc-env=RUSTY_V8_SRC_BINDING_PATH={}",
+      env::var("RUSTY_V8_SRC_BINDING_PATH").unwrap()
+    );
    return;
  }

@ -92,14 +97,22 @@ fn main() {
  };

  // Build from source
-  if env::var_os("V8_FROM_SOURCE").is_some() {
+  if env_bool("V8_FROM_SOURCE") {
    if is_asan && std::env::var_os("OPT_LEVEL").unwrap_or_default() == "0" {
      panic!("v8 crate cannot be compiled with OPT_LEVEL=0 and ASAN.\nTry `[profile.dev.package.v8] opt-level = 1`.\nAborting before miscompilations cause issues.");
    }

-    return build_v8(is_asan);
+    // cargo publish doesn't like pyc files.
+    env::set_var("PYTHONDONTWRITEBYTECODE", "1");
+
+    build_v8(is_asan);
+    build_binding();
+
+    return;
  }

+  print_prebuilt_src_binding_path();
+
  // utilize a lockfile to prevent linking of
  // only partially downloaded static library.
  let root = env::current_dir().unwrap();
@ -119,12 +132,38 @@ fn main() {
  lockfile.unlock().expect("Couldn't unlock lockfile");
 }

+fn build_binding() {
+  let output = Command::new(python())
+    .arg("./tools/get_bindgen_args.py")
+    .arg("--gn-out")
+    .arg(build_dir().join("gn_out"))
+    .output()
+    .unwrap();
+  let args = String::from_utf8(output.stdout).unwrap();
+  let args = args.split('\0').collect::<Vec<_>>();
+
+  let bindings = bindgen::Builder::default()
+    .header("src/binding.hpp")
+    .parse_callbacks(Box::new(bindgen::CargoCallbacks::new()))
+    .clang_args(["-x", "c++", "-std=c++20", "-Iv8/include"])
+    .clang_args(args)
+    .allowlist_item("RUST_.*")
+    .generate()
+    .expect("Unable to generate bindings");
+
+  let out_path = build_dir().join("gn_out").join("src_binding.rs");
+  println!(
+    "cargo:rustc-env=RUSTY_V8_SRC_BINDING_PATH={}",
+    out_path.display()
+  );
+  bindings
+    .write_to_file(out_path)
+    .expect("Couldn't write bindings!");
+}
+
 fn build_v8(is_asan: bool) {
  env::set_var("DEPOT_TOOLS_WIN_TOOLCHAIN", "0");

-  // cargo publish doesn't like pyc files.
-  env::set_var("PYTHONDONTWRITEBYTECODE", "1");
-
  // git submodule update --init --recursive
  let libcxx_src = PathBuf::from("buildtools/third_party/libc++/trunk/src");
  if !libcxx_src.is_dir() {
@ -369,6 +408,28 @@ fn download_ninja_gn_binaries() {
  env::set_var("NINJA", ninja);
 }

+fn prebuilt_profile() -> &'static str {
+  let target_os = env::var("CARGO_CFG_TARGET_OS").unwrap();
+  // Note: we always use the release build on windows.
+  if target_os == "windows" {
+    return "release";
+  }
+  // Use v8 in release mode unless $V8_FORCE_DEBUG=true
+  match env_bool("V8_FORCE_DEBUG") {
+    true => "debug",
+    _ => "release",
+  }
+}
+
+fn static_lib_name(suffix: &str) -> String {
+  let target_os = env::var("CARGO_CFG_TARGET_OS").unwrap();
+  if target_os == "windows" {
+    format!("rusty_v8{suffix}.lib")
+  } else {
+    format!("librusty_v8{suffix}.a")
+  }
+}
+
 fn static_lib_url() -> String {
  if let Ok(custom_archive) = env::var("RUSTY_V8_ARCHIVE") {
    return custom_archive;
@ -378,40 +439,17 @@ fn static_lib_url() -> String {
    env::var("RUSTY_V8_MIRROR").unwrap_or_else(|_| default_base.into());
  let version = env::var("CARGO_PKG_VERSION").unwrap();
  let target = env::var("TARGET").unwrap();
-  let target_os = env::var("CARGO_CFG_TARGET_OS").unwrap();
-  // Note: we always use the release build on windows.
-  if target_os == "windows" {
-    return format!("{}/v{}/rusty_v8_release_{}.lib.gz", base, version, target);
-  }
-  // Use v8 in release mode unless $V8_FORCE_DEBUG=true
-  let profile = match env_bool("V8_FORCE_DEBUG") {
-    true => "debug",
-    _ => "release",
-  };
+  let profile = prebuilt_profile();
  format!(
-    "{}/v{}/librusty_v8_{}_{}.a.gz",
-    base, version, profile, target
+    "{}/v{}/{}.gz",
+    base,
+    version,
+    static_lib_name(&format!("_{}_{}", profile, target)),
  )
 }

-fn env_bool(key: &str) -> bool {
-  matches!(
-    env::var(key).unwrap_or_default().as_str(),
-    "true" | "1" | "yes"
-  )
-}
-
-fn static_lib_name() -> &'static str {
-  let target_os = env::var("CARGO_CFG_TARGET_OS").unwrap();
-  if target_os == "windows" {
-    "rusty_v8.lib"
-  } else {
-    "librusty_v8.a"
-  }
-}
-
 fn static_lib_path() -> PathBuf {
-  static_lib_dir().join(static_lib_name())
+  static_lib_dir().join(static_lib_name(""))
 }

 fn static_checksum_path() -> PathBuf {
@ -666,6 +704,19 @@ fn print_link_flags() {
  }
 }

+fn print_prebuilt_src_binding_path() {
+  let target = env::var("TARGET").unwrap();
+  let profile = prebuilt_profile();
+  let src_binding_path = get_dirs(None)
+    .root
+    .join("gen")
+    .join(format!("src_binding_{}_{}.rs", profile, target));
+  println!(
+    "cargo:rustc-env=RUSTY_V8_SRC_BINDING_PATH={}",
+    src_binding_path.display()
+  );
+}
+
 // Chromium depot_tools contains helpers
 // which delegate to the "relevant" `buildtools`
 // directory when invoked, so they don't count.
@ -875,6 +926,7 @@ pub fn maybe_gen(manifest_dir: &str, gn_args: GnArgs) -> PathBuf {
      .arg(format!("--script-executable={}", python()))
      .arg("gen")
      .arg(&gn_out_dir)
+      .arg("--ide=json")
      .arg("--args=".to_owned() + &args)
      .stdout(Stdio::inherit())
      .stderr(Stdio::inherit())
@ -914,8 +966,9 @@ fn rerun_if_changed(out_dir: &Path, maybe_env: Option<NinjaEnv>, target: &str) {
  let deps = ninja_get_deps(out_dir, maybe_env, target);
  for d in deps {
    let p = out_dir.join(d);
-    assert!(p.exists(), "Path doesn't exist: {:?}", p);
-    println!("cargo:rerun-if-changed={}", p.display());
+    if p.exists() {
+      println!("cargo:rerun-if-changed={}", p.display());
+    }
  }
 }

@ -975,6 +1028,13 @@ pub fn parse_ninja_graph(s: &str) -> HashSet<String> {
  out
 }

+fn env_bool(key: &str) -> bool {
+  matches!(
+    env::var(key).unwrap_or_default().as_str(),
+    "true" | "1" | "yes"
+  )
+}
+
 #[cfg(test)]
 mod test {
  use super::*;
--- a/src/binding.cc
+++ b/src/binding.cc
@ -44,9 +44,6 @@ static_assert(sizeof(two_pointers_t) ==
                  sizeof(std::shared_ptr<v8::BackingStore>),
              "std::shared_ptr<v8::BackingStore> size mismatch");

-static_assert(sizeof(v8::ScriptOrigin) <= sizeof(size_t) * 8,
-              "ScriptOrigin size mismatch");
-
 static_assert(sizeof(v8::HandleScope) == sizeof(size_t) * 3,
              "HandleScope size mismatch");

--- a/src/binding.hpp
+++ b/src/binding.hpp
@ -0,0 +1,12 @@
+#include <v8-message.h>
+
+/**
+ * Types defined here will be compiled with bindgen
+ * and made available in `crate::binding` in rust.
+ */
+
+// TODO: In the immediate term, cppgc definitions will go here.
+// In the future we should migrate over the rest of our SIZE definitions,
+// and eventually entire structs and functions.
+
+static size_t RUST_v8__ScriptOrigin_SIZE = sizeof(v8::ScriptOrigin);
--- a/src/binding.rs
+++ b/src/binding.rs
@ -0,0 +1,4 @@
+#![allow(non_upper_case_globals)]
+#![allow(non_camel_case_types)]
+#![allow(non_snake_case)]
+include!(env!("RUSTY_V8_SRC_BINDING_PATH"));
--- a/src/lib.rs
+++ b/src/lib.rs
@ -30,6 +30,7 @@ extern crate bitflags;
 mod array_buffer;
 mod array_buffer_view;
 mod bigint;
+mod binding;
 mod context;
 pub mod cppgc;
 mod data;
--- a/src/script.rs
+++ b/src/script.rs
@ -13,7 +13,10 @@ use crate::Value;
 /// The origin, within a file, of a script.
 #[repr(C)]
 #[derive(Debug)]
-pub struct ScriptOrigin<'s>([usize; 8], PhantomData<&'s ()>);
+pub struct ScriptOrigin<'s>(
+  [u8; crate::binding::RUST_v8__ScriptOrigin_SIZE],
+  PhantomData<&'s ()>,
+);

 extern "C" {
  fn v8__Script__Compile(
--- a/tools/get_bindgen_args.py
+++ b/tools/get_bindgen_args.py
@ -0,0 +1,21 @@
+import argparse
+import json
+import os
+
+parser = argparse.ArgumentParser(description='Generate args for bindgen')
+parser.add_argument('--gn-out', help='GN out directory')
+args = parser.parse_args()
+
+with open(os.path.join(args.gn_out, 'project.json')) as project_json:
+    project = json.load(project_json)
+
+target = project['targets']['//v8:v8_headers']
+
+assert '//v8:cppgc_headers' in target['deps']
+
+args = []
+
+for define in target['defines']:
+    args.append(f'-D{define}')
+
+print('\0'.join(args), end="")