tools: clean up third_party.py, and merge prebuilt.py into it (#2950)

* Remove reference to removed dir 'third_party/rust_crates'.
* Remove reference to unused environment variable 'DENO_NINJA_PATH'.
* Remove helper functions 'root()' and 'tp()'.
* Move definition of 'third_party_path' to build.py.
* Move definition of 'gn_exe()' to setup.py.
* Move 'download_sccache()' and 'download_hyperfine()' from prebuilt.py
  to third_party.py, and delete prebuilt.py.
* Add helper function 'get_platform_dir_name()' to locate the
  platform-specific 'v8/buildtools/<platform>' and
  'prebuilt/<platform>' directories.
* Add helper function 'get_prebuilt_tool_path()' that returns the full
  path to a platform-specific executable in //prebuilt.
* Cosmetic improvements.

tools/benchmark.py

@@ -15,7 +15,7 @@ import tempfile
 import subprocess
 from util import find_exts, root_path, run, run_output
 from util import build_path, executable_suffix
-import prebuilt
+import third_party
 from http_benchmark import http_benchmark
 import throughput_benchmark
 import http_server
@@ -178,10 +178,11 @@ def run_max_mem_benchmark(deno_exe):
 
 
 def run_exec_time(deno_exe, build_dir):
+    third_party.download_hyperfine()
+    hyperfine_exe = third_party.get_prebuilt_tool_path("hyperfine")
     benchmark_file = os.path.join(build_dir, "hyperfine_results.json")
-    hyperfine = prebuilt.load_hyperfine()
     run([
-        hyperfine, "--ignore-failure", "--export-json", benchmark_file,
+        hyperfine_exe, "--ignore-failure", "--export-json", benchmark_file,
         "--warmup", "3"
     ] + [
         deno_exe + " run " + " ".join(args)

tools/format.py

@@ -5,8 +5,7 @@ import os
 import sys
 import argparse
 from third_party import google_env, python_env
-from third_party import clang_format_path, third_party_path
+from util import root_path, run, find_exts, platform, third_party_path
-from util import root_path, run, find_exts, platform
 
 parser = argparse.ArgumentParser()
 parser.add_argument("--js", help="only run prettier", action="store_true")
@@ -15,6 +14,8 @@ parser.add_argument("--py", help="only run yapf", action="store_true")
 parser.add_argument("--gn", help="only run gn format", action="store_true")
 parser.add_argument("--cc", help="only run clang format", action="store_true")
 
+clang_format_path = os.path.join(third_party_path, "depot_tools",
+                                 "clang-format")
 prettier_path = os.path.join(third_party_path, "node_modules", "prettier",
                              "bin-prettier.js")
 tools_path = os.path.join(root_path, "tools")

tools/lint.py

@@ -4,12 +4,11 @@
 
 import os
 import sys
-from util import enable_ansi_colors, find_exts, root_path, run
+from util import enable_ansi_colors, find_exts, root_path, run, third_party_path
 from third_party import python_env
 
 enable_ansi_colors()
 
-third_party_path = os.path.join(root_path, "third_party")
 cpplint = os.path.join(third_party_path, "cpplint", "cpplint.py")
 eslint = os.path.join(third_party_path, "node_modules", "eslint", "bin",
                       "eslint")

tools/prebuilt.py

@@ -1,39 +0,0 @@
-# Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
-import sys
-import os
-from util import run, root_path
-from third_party import tp, google_env
-
-
-def download_prebuilt(sha1_file):
-    run([
-        sys.executable,
-        tp('depot_tools/download_from_google_storage.py'),
-        '--platform=' + sys.platform,
-        '--no_auth',
-        '--bucket=denoland',
-        '--sha1_file',
-        sha1_file,
-    ],
-        env=google_env())
-
-
-def get_platform_path(tool):
-    if sys.platform == 'win32':
-        return "prebuilt/win/" + tool + ".exe"
-    elif sys.platform.startswith('linux'):
-        return "prebuilt/linux64/" + tool
-    elif sys.platform == 'darwin':
-        return "prebuilt/mac/" + tool
-
-
-def load_sccache():
-    p = get_platform_path("sccache")
-    download_prebuilt(p + ".sha1")
-    return os.path.join(root_path, p)
-
-
-def load_hyperfine():
-    p = get_platform_path("hyperfine")
-    download_prebuilt(p + ".sha1")
-    return os.path.join(root_path, p)

tools/setup.py

@@ -5,10 +5,9 @@ import re
 import sys
 from distutils.spawn import find_executable
 import argparse
-import prebuilt
 import third_party
-from util import (build_mode, build_path, enable_ansi_colors, root_path, run,
+from util import build_mode, build_path, enable_ansi_colors, run, shell_quote
-                  shell_quote)
+from util import root_path, third_party_path
 
 parser = argparse.ArgumentParser()
 parser.add_argument(
@@ -30,8 +29,8 @@ def main():
         third_party.download_gn()
         third_party.download_clang_format()
         third_party.download_clang()
+        third_party.download_sccache()
         third_party.maybe_download_sysroot()
-        prebuilt.load_sccache()
 
     write_lastchange()
 
@@ -126,7 +125,7 @@ def generate_gn_args(mode):
     if "DENO_BUILD_ARGS" in os.environ:
         out += os.environ["DENO_BUILD_ARGS"].split()
 
-    cacher = os.path.join(root_path, prebuilt.get_platform_path("sccache"))
+    cacher = third_party.get_prebuilt_tool_path("sccache")
     if not os.path.exists(cacher):
         cacher = find_executable("sccache") or find_executable("ccache")
 
@@ -143,6 +142,13 @@ def generate_gn_args(mode):
     return out
 
 
+def gn_exe():
+    if "DENO_GN_PATH" in os.environ:
+        return os.environ["DENO_GN_PATH"]
+    else:
+        return os.path.join(third_party_path, "depot_tools", "gn")
+
+
 # gn gen.
 def gn_gen(mode):
     os.environ["DENO_BUILD_MODE"] = mode
@@ -169,8 +175,7 @@ def gn_gen(mode):
     for line in gn_args:
         print "  " + line
 
-    run([third_party.gn_path, "gen", build_path()],
+    run([gn_exe(), "gen", build_path()], env=third_party.google_env())
-        env=third_party.google_env())
 
 
 if __name__ == '__main__':

tools/third_party.py

@@ -3,40 +3,17 @@
 # This script contains helper functions to work with the third_party subrepo.
 
 import os
-from os import path
 import re
 import site
 import sys
 from tempfile import mkdtemp
-from util import add_env_path, find_exts, make_env, rmtree, root_path, run
+from util import add_env_path, executable_suffix, make_env, rmtree, root_path
+from util import run, third_party_path
 
-
+chromium_build_path = os.path.join(root_path, "build")
-# Helper function that returns the full path to a subpath of the repo root.
+depot_tools_path = os.path.join(third_party_path, "depot_tools")
-def root(*subpath_parts):
+prebuilt_path = os.path.join(root_path, "prebuilt")
-    return path.normpath(path.join(root_path, *subpath_parts))
+python_packages_path = os.path.join(third_party_path, "python_packages")
-
-
-# Helper function that returns the full path to a file/dir in third_party.
-def tp(*subpath_parts):
-    return root("third_party", *subpath_parts)
-
-
-build_path = root("build")
-
-third_party_path = tp()
-depot_tools_path = tp("depot_tools")
-rust_crates_path = tp("rust_crates")
-python_packages_path = tp("python_packages")
-clang_format_path = tp(depot_tools_path, "clang-format")
-
-if "DENO_GN_PATH" in os.environ:
-    gn_path = os.environ["DENO_GN_PATH"]
-else:
-    gn_path = tp(depot_tools_path, "gn")
-if "DENO_NINJA_PATH" in os.environ:
-    ninja_path = os.environ["DENO_NINJA_PATH"]
-else:
-    ninja_path = tp(depot_tools_path, "ninja")
 
 python_site_env = None
 
@@ -55,7 +32,7 @@ def python_env(env=None, merge_env=None):
         python_site_env = {}
         temp = os.environ["PATH"], sys.path
         os.environ["PATH"], sys.path = "", []
-        site.addsitedir(build_path)  # Modifies PATH and sys.path.
+        site.addsitedir(chromium_build_path)  # Modifies PATH and sys.path.
         site.addsitedir(python_packages_path)  # Modifies PATH and sys.path.
         python_site_env = {"PATH": os.environ["PATH"], "PYTHONPATH": sys.path}
         os.environ["PATH"], sys.path = temp
@@ -137,6 +114,7 @@ def run_pip():
     ],
         cwd=third_party_path,
         merge_env=pip_env)
+
     # Remove the temporary pip installation.
     rmtree(temp_python_home)
 
@@ -152,7 +130,7 @@ def run_gclient_sync():
     # Since depot_tools is listed in .gclient_entries, gclient will install a
     # fresh copy in `third_party/depot_tools`.
     # If it all works out, we remove the depot_tools_temp directory afterwards.
-    depot_tools_temp_path = root("depot_tools_temp")
+    depot_tools_temp_path = os.path.join(root_path, "depot_tools_temp")
 
     # Rename depot_tools to depot_tools_temp.
     try:
@@ -163,7 +141,7 @@ def run_gclient_sync():
         # failed half-way, before we got the chance to remove the temp dir.
         # We'll use whatever is in the temp dir that was already there.
         # If not, the user can recover by removing the temp directory manually.
-        if path.isdir(depot_tools_temp_path):
+        if os.path.isdir(depot_tools_temp_path):
             pass
         else:
             raise
@@ -172,8 +150,8 @@ def run_gclient_sync():
         "gclient", "sync", "--reset", "--shallow", "--no-history", "--nohooks"
     ]
     envs = {
-        'DEPOT_TOOLS_UPDATE': "0",
+        "DEPOT_TOOLS_UPDATE": "0",
-        'GCLIENT_FILE': root("gclient_config.py")
+        "GCLIENT_FILE": os.path.join(root_path, "gclient_config.py")
     }
     env = google_env(depot_tools_path_=depot_tools_temp_path, merge_env=envs)
     run(args, cwd=third_party_path, env=env)
@@ -182,89 +160,107 @@ def run_gclient_sync():
     # gclient did indeed install a fresh copy.
     # Also check that `{depot_tools_temp_path}/gclient.py` exists, so a typo in
     # this script won't accidentally blow out someone's home dir.
-    if (path.isdir(path.join(depot_tools_path, ".git"))
+    if (os.path.isdir(os.path.join(depot_tools_path, ".git"))
-            and path.isfile(path.join(depot_tools_path, "gclient.py"))
+            and os.path.isfile(os.path.join(depot_tools_path, "gclient.py"))
-            and path.isfile(path.join(depot_tools_temp_path, "gclient.py"))):
+            and os.path.isfile(
+                os.path.join(depot_tools_temp_path, "gclient.py"))):
         rmtree(depot_tools_temp_path)
 
 
-# Download the given item from Google storage.
+def get_platform_dir_name():
-def download_from_google_storage(item, bucket):
+    if sys.platform == "win32":
-    if sys.platform == 'win32':
+        return "win"
-        sha1_file = "v8/buildtools/win/%s.exe.sha1" % item
+    elif sys.platform == "darwin":
-    elif sys.platform == 'darwin':
+        return "mac"
-        sha1_file = "v8/buildtools/mac/%s.sha1" % item
+    elif sys.platform.startswith("linux"):
-    elif sys.platform.startswith('linux'):
+        return "linux64"
-        sha1_file = "v8/buildtools/linux64/%s.sha1" % item
 
+
+def get_prebuilt_tool_path(tool):
+    return os.path.join(prebuilt_path, get_platform_dir_name(),
+                        tool + executable_suffix)
+
+
+# Download the given item from Google storage.
+def download_from_google_storage(item, bucket, base_dir):
+    download_script = os.path.join(depot_tools_path,
+                                   "download_from_google_storage.py")
+    sha1_file = os.path.join(base_dir, get_platform_dir_name(),
+                             item + executable_suffix + ".sha1")
     run([
         sys.executable,
-        tp('depot_tools/download_from_google_storage.py'),
+        download_script,
-        '--platform=' + sys.platform,
+        "--platform=" + sys.platform,
-        '--no_auth',
+        "--no_auth",
-        '--bucket=%s' % bucket,
+        "--bucket=%s" % bucket,
-        '--sha1_file',
+        "--sha1_file",
-        tp(sha1_file),
+        sha1_file,
     ],
         env=google_env())
 
 
 # Download the given item from Chrome Infrastructure Package Deployment.
 def download_from_cipd(item, version):
-    if sys.platform == 'win32':
+    cipd_exe = os.path.join(depot_tools_path, "cipd")
-        root_dir = "v8/buildtools/win"
+    download_dir = os.path.join(third_party_path, "v8", "buildtools",
+                                get_platform_dir_name())
+
+    if sys.platform == "win32":
         item += "windows-amd64"
-    elif sys.platform == 'darwin':
+    elif sys.platform == "darwin":
-        root_dir = "v8/buildtools/mac"
         item += "mac-amd64"
-    elif sys.platform.startswith('linux'):
+    elif sys.platform.startswith("linux"):
-        root_dir = "v8/buildtools/linux64"
         item += "linux-amd64"
 
-    # init cipd if necessary
+    # Init cipd if necessary.
-    if not os.path.exists(path.join(tp(root_dir), ".cipd")):
+    if not os.path.exists(os.path.join(download_dir, ".cipd")):
         run([
-            tp('depot_tools/cipd'),
+            cipd_exe,
-            'init',
+            "init",
-            tp(root_dir),
+            download_dir,
-            '-force',
+            "-force",
-        ],
+        ], env=google_env())
-            env=google_env())
 
     run([
-        tp('depot_tools/cipd'),
+        cipd_exe,
-        'install',
+        "install",
         item,
-        'git_revision:' + version,
+        "git_revision:" + version,
-        '-root',
+        "-root",
-        tp(root_dir),
+        download_dir,
     ],
         env=google_env())
 
 
 # Download gn from Google storage.
 def download_gn():
-    download_from_cipd('gn/gn/', '152c5144ceed9592c20f0c8fd55769646077569b')
+    download_from_cipd("gn/gn/", "152c5144ceed9592c20f0c8fd55769646077569b")
 
 
 # Download clang-format from Google storage.
 def download_clang_format():
-    download_from_google_storage('clang-format', 'chromium-clang-format')
+    download_from_google_storage(
+        "clang-format", "chromium-clang-format",
+        os.path.join(third_party_path, "v8", "buildtools"))
+
+
+def download_sccache():
+    download_from_google_storage("sccache", "denoland", prebuilt_path)
+
+
+def download_hyperfine():
+    download_from_google_storage("hyperfine", "denoland", prebuilt_path)
 
 
 # Download clang by calling the clang update script.
 def download_clang():
-    run([sys.executable,
+    update_script = os.path.join(third_party_path, "v8", "tools", "clang",
-         tp('v8/tools/clang/scripts/update.py')],
+                                 "scripts", "update.py")
-        env=google_env())
+    run([sys.executable, update_script], env=google_env())
 
 
 def maybe_download_sysroot():
-    if sys.platform.startswith('linux'):
+    if sys.platform.startswith("linux"):
-        run([
+        install_script = os.path.join(chromium_build_path, "linux",
-            sys.executable,
+                                      "sysroot_scripts", "install-sysroot.py")
-            os.path.join(root_path,
+        run([sys.executable, install_script, "--arch=amd64"], env=google_env())
-                         'build/linux/sysroot_scripts/install-sysroot.py'),
-            '--arch=amd64'
-        ],
-            env=google_env())

tools/util.py

@@ -18,8 +18,10 @@ else:
     FG_GREEN = "\x1b[32m"
 
 executable_suffix = ".exe" if os.name == "nt" else ""
+
 root_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 tests_path = os.path.join(root_path, "tests")
+third_party_path = os.path.join(root_path, "third_party")
 
 
 def make_env(merge_env=None, env=None):