download ninja/gn from cipd (#1605)

build.rs

@@ -356,36 +356,11 @@ fn maybe_install_sysroot(arch: &str) {
   }
 }
 
-fn host_platform() -> String {
-  let os = if cfg!(target_os = "linux") {
-    "linux"
-  } else if cfg!(target_os = "macos") {
-    "mac"
-  } else if cfg!(target_os = "windows") {
-    "windows"
-  } else {
-    "unknown"
-  };
-
-  let arch = if cfg!(target_arch = "x86_64") {
-    "amd64"
-  } else if cfg!(target_arch = "aarch64") {
-    "arm64"
-  } else if cfg!(target_arch = "arm") {
-    "arm"
-  } else {
-    "unknown"
-  };
-  format!("{os}-{arch}")
-}
-
 fn download_ninja_gn_binaries() {
-  let target_dir = build_dir();
-  let bin_dir = target_dir
-    .join("ninja_gn_binaries-20221218")
-    .join(host_platform());
-  let gn = bin_dir.join("gn");
-  let ninja = bin_dir.join("ninja");
+  let target_dir = build_dir().join("ninja_gn_binaries");
+
+  let gn = target_dir.join("gn").join("gn");
+  let ninja = target_dir.join("ninja").join("ninja");
   #[cfg(windows)]
   let gn = gn.with_extension("exe");
   #[cfg(windows)]
@@ -990,8 +965,7 @@ pub fn build(target: &str, maybe_env: Option<NinjaEnv>) {
 fn rerun_if_changed(out_dir: &Path, maybe_env: Option<NinjaEnv>, target: &str) {
   let deps = ninja_get_deps(out_dir, maybe_env, target);
   for d in deps {
-    let p = out_dir.join(d);
-    if p.exists() {
+    if let Ok(p) = out_dir.join(d).canonicalize() {
       println!("cargo:rerun-if-changed={}", p.display());
     }
   }

tools/ninja_gn_binaries.py

@@ -4,22 +4,44 @@
 # found in the LICENSE file.
 """This script is used to download prebuilt gn/ninja binaries."""
 
-from __future__ import print_function
+import platform
+import json
 import argparse
 import os
 import sys
-import tarfile
+import zipfile
 import tempfile
 import time
+import http.client
+from v8_deps import Var
+from urllib.error import HTTPError, URLError
+from stat import ST_MODE, S_IXOTH, S_IXGRP, S_IXUSR
+from urllib.request import urlopen
+from urllib.parse import urlparse
 
-try:
-    from urllib2 import HTTPError, URLError, urlopen
-except ImportError:  # For Py3 compatibility
-    from urllib.error import HTTPError, URLError
-    from urllib.request import urlopen
 
-URL = "https://github.com/denoland/ninja_gn_binaries/archive/20221218.tar.gz"
-DIR = None
+def get_platform():
+    system = platform.system().lower()
+    if system == 'darwin':
+        system = 'mac'
+    machine = platform.machine().lower()
+    if machine == 'x86_64':
+        machine = 'amd64'
+
+    return f'{system}-{machine}'
+
+
+PLATFORM = get_platform()
+is_windows = PLATFORM.startswith('windows')
+
+RESOLVE_URL = 'https://chrome-infra-packages.appspot.com/_ah/api/repo/v1/instance/resolve?package_name={}&version={}'
+INSTANCE_URL = 'https://chrome-infra-packages.appspot.com/_ah/api/repo/v1/instance?package_name={}&instance_id={}'
+
+NINJA_VERSION = Var('ninja_version')
+GN_VERSION = Var('gn_version')
+
+NINJA_PACKAGE = f'infra/3pp/tools/ninja/{PLATFORM}'
+GN_PACKAGE = f'gn/gn/{PLATFORM}'
 
 
 def DownloadUrl(url, output_file):
@@ -67,18 +89,35 @@ def DownloadAndUnpack(url, output_dir):
         DownloadUrl(url, f)
         f.seek(0)
         EnsureDirExists(output_dir)
-        t = tarfile.open(mode='r:gz', fileobj=f)
-        t.extractall(path=output_dir)
+        with zipfile.ZipFile(f, 'r') as z:
+            z.extractall(path=output_dir)
+            if not is_windows:
+                for info in z.infolist():
+                    if info.is_dir():
+                        continue
+                    file = os.path.join(output_dir, info.filename)
+                    hi = info.external_attr >> 16
+                    if hi:
+                        mode = os.stat(file)[ST_MODE]
+                        mode |= hi
+                        os.chmod(file, mode)
 
 
-def Update():
-    try:
-        DownloadAndUnpack(URL, DIR)
-    except URLError:
-        print('Failed to download ninja/gn binaries.')
-        sys.exit(1)
+def DownloadCIPD(package, tag, output_dir):
+    def get(url):
+        parsed = urlparse(url)
+        conn = http.client.HTTPSConnection(parsed.netloc)
+        conn.request("GET", parsed.path + (f'?{parsed.query}' if parsed.query else ''), headers={"Host": parsed.netloc})
+        response = conn.getresponse()
+        if response.status != 200:
+            raise Exception(f'GET {url} returned {response.status} {response.reason}')
+        data = response.read().decode()
+        return json.loads(data)
 
-    return 0
+    resolved = get(RESOLVE_URL.format(package, tag))
+    instance_id = resolved['instance_id']
+    instance = get(INSTANCE_URL.format(package, instance_id))
+    DownloadAndUnpack(instance['fetch_url'], output_dir)
 
 
 def main():
@@ -86,11 +125,11 @@ def main():
     parser.add_argument('--dir', help='Where to extract the package.')
     args = parser.parse_args()
 
-    if args.dir:
-        global DIR
-        DIR = os.path.abspath(args.dir)
+    output_dir = os.path.abspath(args.dir)
+
+    DownloadCIPD(GN_PACKAGE, GN_VERSION, os.path.join(output_dir, 'gn'))
+    DownloadCIPD(NINJA_PACKAGE, NINJA_VERSION, os.path.join(output_dir, 'ninja'))
 
-    return Update()
 
 
 if __name__ == '__main__':

tools/setup_rbe.py

@@ -14,13 +14,7 @@ NINJA=autoninja
 ```
 """
 
-Str = str
-def Var(name):
-    if name == 'rbe_instance':
-        return 'projects/rbe-chromium-untrusted/instances/default_instance'
-    return vars[name]
-with open('./v8/DEPS') as f:
-    exec(f.read())
+from v8_deps import Var, hooks
 
 import subprocess
 import os

tools/update_deps.py

@@ -1,9 +1,4 @@
-Str = str
-def Var(name):
-    return vars[name]
-with open('./v8/DEPS') as f:
-    exec(f.read())
-
+from v8_deps import deps
 import subprocess
 
 def process(name, dep):

tools/v8_deps.py

@@ -0,0 +1,8 @@
+Str = str
+def Var(name):
+    if name == 'rbe_instance':
+        return 'projects/rbe-chromium-untrusted/instances/default_instance'
+    return vars[name]
+with open('./v8/DEPS') as f:
+    exec(f.read())
+