diff --git a/tools/build.py b/tools/build.py index fb061eae30..8b33cef898 100755 --- a/tools/build.py +++ b/tools/build.py @@ -4,7 +4,8 @@ import argparse import os import sys from os.path import join -from util import run +from third_party import depot_tools_path, third_party_path, fix_symlinks, google_env +from util import root_path, run import distutils.spawn parser = argparse.ArgumentParser(description='') @@ -16,19 +17,13 @@ parser.add_argument( '--mode', default='debug', help='Build configuration: debug, release.') options, targets = parser.parse_known_args() -root_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) -third_party_path = join(root_path, "third_party") -depot_tools_path = join(third_party_path, "depot_tools") -gn_path = join(depot_tools_path, "gn") -ninja_path = join(depot_tools_path, "ninja") - -# Add third_party/depot_tools to PATH because some google tools (e.g. -# tool_wrapper, download_from_google_storage) use some google specific python -# wrapper. -os.environ["PATH"] = depot_tools_path + os.pathsep + os.environ["PATH"] +fix_symlinks() os.chdir(root_path) +gn_path = join(depot_tools_path, "gn") +ninja_path = join(depot_tools_path, "ninja") + if options.build_path: build_path = options.build_path else: @@ -63,7 +58,7 @@ if not os.path.exists(args_filename) or options.args: with open(args_filename, "w+") as f: f.write("\n".join(gn_args) + "\n") -run([gn_path, "gen", build_path]) +run([gn_path, "gen", build_path], env=google_env()) target = " ".join(targets) if targets else ":all" -run([ninja_path, "-C", build_path, target]) +run([ninja_path, "-C", build_path, target], env=google_env()) diff --git a/tools/format.py b/tools/format.py index b994535f7c..2bbc95bed4 100755 --- a/tools/format.py +++ b/tools/format.py @@ -1,9 +1,10 @@ #!/usr/bin/env python import os -from util import run, find_exts +from third_party import third_party_path, fix_symlinks, google_env +from util import root_path, run, find_exts + +fix_symlinks() -root_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) -third_party_path = os.path.join(root_path, "third_party") prettier = os.path.join(third_party_path, "node_modules", "prettier", "bin-prettier.js") tools_path = os.path.join(root_path, "tools") @@ -15,7 +16,7 @@ os.chdir(root_path) run(["clang-format", "-i", "-style", "Google"] + find_exts("src", ".cc", ".h")) for fn in ["BUILD.gn", ".gn"] + find_exts("build_extra", ".gn", ".gni"): - run(["third_party/depot_tools/gn", "format", fn]) + run(["third_party/depot_tools/gn", "format", fn], env=google_env()) # TODO(ry) Install yapf in third_party. run(["yapf", "-i"] + find_exts("tools/", ".py") + diff --git a/tools/run_hooks.py b/tools/run_hooks.py index 58178a69c7..59161f5f2e 100755 --- a/tools/run_hooks.py +++ b/tools/run_hooks.py @@ -1,29 +1,7 @@ #!/usr/bin/env python -import os -import sys -from util import run +import third_party -root_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) -third_party_path = os.path.join(root_path, "third_party") -depot_tools_path = os.path.join(third_party_path, "depot_tools") -os.chdir(root_path) +third_party.fix_symlinks() - -def download(filename): - run([ - "python", - os.path.join(depot_tools_path + '/download_from_google_storage.py'), - '--platform=' + sys.platform, '--no_auth', '--bucket=chromium-gn', - '--sha1_file', - os.path.join(root_path, filename) - ]) - - -if sys.platform == 'win32': - download("third_party/v8/buildtools/win/gn.exe.sha1") -elif sys.platform == 'darwin': - download("third_party/v8/buildtools/mac/gn.sha1") -elif sys.platform.startswith('linux'): - download("third_party/v8/buildtools/linux64/gn.sha1") - -run(['python', 'third_party/v8/tools/clang/scripts/update.py', '--if-needed']) +third_party.download_gn() +third_party.download_clang() diff --git a/tools/sync_third_party.py b/tools/sync_third_party.py index f32d669b2a..cf0de30f3a 100755 --- a/tools/sync_third_party.py +++ b/tools/sync_third_party.py @@ -1,35 +1,13 @@ #!/usr/bin/env python -# Only run this script if you are changing Deno's dependencies. - -import os -from os.path import join -from util import run, remove_and_symlink - -root_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) -third_party_path = join(root_path, "third_party") - -try: - os.makedirs(third_party_path) -except: - pass -os.chdir(third_party_path) - -# Run yarn to install JavaScript dependencies. -remove_and_symlink("../package.json", "package.json") -remove_and_symlink("../yarn.lock", "yarn.lock") -run(["yarn"]) -# Run cargo to install Rust dependencies. -run(["cargo", "fetch", "--manifest-path=" + root_path + "/Cargo.toml"], - envs={'CARGO_HOME': third_party_path + '/rust_crates'}) -# Run gclient to install other dependencies. -run(["gclient", "sync", "--reset", "--shallow", "--no-history", "--nohooks"], - envs={'GCLIENT_FILE': root_path + "/gclient_config.py"}) -# TODO(ry) Is it possible to remove these symlinks? -remove_and_symlink("v8/third_party/googletest", "googletest", True) -remove_and_symlink("v8/third_party/jinja2", "jinja2", True) -remove_and_symlink("v8/third_party/llvm-build", "llvm-build", True) -remove_and_symlink("v8/third_party/markupsafe", "markupsafe", True) - +# Run this script if you are changing Deno's dependencies. # To update the deno_third_party git repo after running this, try the following: # cd third_party # find . -type f | grep -v "\.git" | xargs -I% git add -f --no-warn-embedded-repo "%" + +import third_party + +third_party.fix_symlinks() + +third_party.run_yarn() +third_party.run_cargo() +third_party.run_gclient_sync() diff --git a/tools/third_party.py b/tools/third_party.py new file mode 100644 index 0000000000..ef8ed466f3 --- /dev/null +++ b/tools/third_party.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python +# This script contains helper functions to work with the third_party subrepo. + +import os +import sys +from os import path +from util import find_exts, make_env, remove_and_symlink, rmtree, root_path, run + + +# Helper function that returns the full path to a subpath of the repo root. +def root(*subpath_parts): + return path.normpath(path.join(root_path, *subpath_parts)) + + +# Helper function that returns the full path to a file/dir in third_party. +def tp(*subpath_parts): + return root("third_party", *subpath_parts) + + +third_party_path = tp() +depot_tools_path = tp("depot_tools") +rust_crates_path = tp("rust_crates") + + +# This function creates or modifies an environment so that it matches the +# expectations of various google tools (gn, gclient, etc). +def google_env(env=None, merge_env={}, depot_tools_path=depot_tools_path): + env = make_env(env=env, merge_env=merge_env) + # Depot_tools to be in the PATH, before Python. + path_prefix = depot_tools_path + os.path.pathsep + if not env['PATH'].startswith(path_prefix): + env['PATH'] = path_prefix + env['PATH'] + # We're not using Google's internal infrastructure. + if os.name == 'nt' and not 'DEPOT_TOOLS_WIN_TOOLCHAIN' in env: + env['DEPOT_TOOLS_WIN_TOOLCHAIN'] = "0" + return env + + +def fix_symlinks(): + # Ensure the third_party directory exists. + try: + os.makedirs(third_party_path) + except: + pass + + # Make symlinks to Yarn metadata living in the root repo. + remove_and_symlink("../package.json", tp("package.json")) + remove_and_symlink("../yarn.lock", tp("yarn.lock")) + + # TODO(ry) Is it possible to remove these symlinks? + remove_and_symlink("v8/third_party/googletest", tp("googletest"), True) + remove_and_symlink("v8/third_party/jinja2", tp("jinja2"), True) + remove_and_symlink("v8/third_party/llvm-build", tp("llvm-build"), True) + remove_and_symlink("v8/third_party/markupsafe", tp("markupsafe"), True) + + # On Windows, git doesn't create the right type of symlink if the symlink + # and it's target are in different repos. Here we fix the symlinks that exist + # in the root repo while their target is in the third_party repo. + remove_and_symlink("third_party/node_modules", root("node_modules"), True) + remove_and_symlink("third_party/v8/build", root("build"), True) + remove_and_symlink("third_party/v8/buildtools", root("buildtools"), True) + remove_and_symlink("third_party/v8/build_overrides", + root("build_overrides"), True) + remove_and_symlink("third_party/v8/testing", root("testing"), True) + + +# Run Yarn to install JavaScript dependencies. +def run_yarn(): + run(["yarn"], cwd=third_party_path) + + +# Run Cargo to install Rust dependencies. +def run_cargo(): + # Deletes the cargo index lockfile; it appears that cargo itself doesn't do it. + # If the lockfile ends up in the git repo, it'll make cargo hang for everyone + # else who tries to run sync_third_party. + def delete_lockfile(): + lockfiles = find_exts( + path.join(rust_crates_path, "registry/index"), '.cargo-index-lock') + for lockfile in lockfiles: + os.remove(lockfile) + + # Delete the index lockfile in case someone accidentally checked it in. + delete_lockfile() + + run(["cargo", "fetch", "--manifest-path=" + root("Cargo.toml")], + cwd=third_party_path, + merge_env={'CARGO_HOME': rust_crates_path}) + + # Delete the lockfile again so it doesn't end up in the git repo. + delete_lockfile() + + +# Run gclient to install other dependencies. +def run_gclient_sync(): + # Depot_tools will normally try to self-update, which will fail because + # it's not checked out from it's own git repository; gclient will then try + # to fix things up and not succeed, and and we'll end up with a huge mess. + # To work around this, we rename the `depot_tools` directory to + # `{root_path}/depot_tools_temp` first, and we set DEPOT_TOOLS_UPDATE=0 in + # the environment so depot_tools doesn't attempt to self-update. + # Since depot_tools is listed in .gclient_entries, gclient will install a + # fresh copy in `third_party/depot_tools`. + # If it all works out, we remove the depot_tools_temp directory afterwards. + depot_tools_temp_path = root("depot_tools_temp") + + # Rename depot_tools to depot_tools_temp. + try: + os.rename(depot_tools_path, depot_tools_temp_path) + except: + # If renaming failed, and the depot_tools_temp directory already exists, + # assume that it's still there because a prior run_gclient_sync() call + # failed half-way, before we got the chance to remove the temp dir. + # We'll use whatever is in the temp dir that was already there. + # If not, the user can recover by removing the temp directory manually. + if path.isdir(depot_tools_temp_path): + pass + else: + raise + + args = [ + "gclient", "sync", "--reset", "--shallow", "--no-history", "--nohooks" + ] + envs = { + 'DEPOT_TOOLS_UPDATE': "0", + 'GCLIENT_FILE': root("gclient_config.py") + } + env = google_env(depot_tools_path=depot_tools_temp_path, merge_env=envs) + run(args, cwd=third_party_path, env=env) + + # Delete the depot_tools_temp directory, but not before verifying that + # gclient did indeed install a fresh copy. + # Also check that `{depot_tools_temp_path}/gclient.py` exists, so a typo in + # this script won't accidentally blow out someone's home dir. + if (path.isdir(path.join(depot_tools_path, ".git")) + and path.isfile(path.join(depot_tools_path, "gclient.py")) + and path.isfile(path.join(depot_tools_temp_path, "gclient.py"))): + rmtree(depot_tools_temp_path) + + +# Download gn from Google storage. +def download_gn(): + if sys.platform == 'win32': + sha1_file = "v8/buildtools/win/gn.exe.sha1" + elif sys.platform == 'darwin': + sha1_file = "v8/buildtools/mac/gn.sha1" + elif sys.platform.startswith('linux'): + sha1_file = "v8/buildtools/linux64/gn.sha1" + + run([ + "python", + tp('depot_tools/download_from_google_storage.py'), + '--platform=' + sys.platform, + '--no_auth', + '--bucket=chromium-gn', + '--sha1_file', + tp(sha1_file), + ], + env=google_env()) + + +# Download clang by calling the clang update script. +def download_clang(): + run(['python', + tp('v8/tools/clang/scripts/update.py'), '--if-needed'], + env=google_env()) diff --git a/tools/util.py b/tools/util.py index 4938c51579..436630401b 100644 --- a/tools/util.py +++ b/tools/util.py @@ -1,20 +1,30 @@ # Copyright 2018 Ryan Dahl # All rights reserved. MIT License. import os +import shutil +import stat import subprocess executable_suffix = ".exe" if os.name == "nt" else "" +root_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) -def run(args, quiet=False, envs={}): +def make_env(merge_env={}, env=None): + if env is None: + env = os.environ + env = env.copy() + for key in merge_env.keys(): + env[key] = merge_env[key] + return env + + +def run(args, quiet=False, cwd=None, env=None, merge_env={}): + args[0] = os.path.normpath(args[0]) if not quiet: print " ".join(args) - env = os.environ.copy() - for key in envs.keys(): - env[key] = envs[key] - args[0] = os.path.normpath(args[0]) + env = make_env(env=env, merge_env=merge_env) shell = os.name == "nt" # Run through shell to make .bat/.cmd files work. - subprocess.check_call(args, env=env, shell=shell) + subprocess.check_call(args, cwd=cwd, env=env, shell=shell) def remove_and_symlink(target, name, target_is_dir=False): @@ -68,3 +78,14 @@ def find_exts(directory, *extensions): matches.append(os.path.join(root, filename)) break return matches + + +# The Python equivalent of `rm -rf`. +def rmtree(directory): + # On Windows, shutil.rmtree() won't delete files that have a readonly bit. + # Git creates some files that do. The 'onerror' callback deals with those. + def rm_readonly(func, path, _): + os.chmod(path, stat.S_IWRITE) + func(path) + + shutil.rmtree(directory, onerror=rm_readonly)