mirror of
https://github.com/denoland/deno.git
synced 2024-12-23 15:49:44 -05:00
Clean up and fix tools
* Make sync_third_party work in general * Un-break build.py and run_hooks.py on windows * Partially fix format.py on windows * Reduce code duplication between run_hooks and sync_third_party
This commit is contained in:
parent
0875411267
commit
4d08bb85a4
6 changed files with 219 additions and 80 deletions
|
@ -4,7 +4,8 @@ import argparse
|
|||
import os
|
||||
import sys
|
||||
from os.path import join
|
||||
from util import run
|
||||
from third_party import depot_tools_path, third_party_path, fix_symlinks, google_env
|
||||
from util import root_path, run
|
||||
import distutils.spawn
|
||||
|
||||
parser = argparse.ArgumentParser(description='')
|
||||
|
@ -16,19 +17,13 @@ parser.add_argument(
|
|||
'--mode', default='debug', help='Build configuration: debug, release.')
|
||||
options, targets = parser.parse_known_args()
|
||||
|
||||
root_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
|
||||
third_party_path = join(root_path, "third_party")
|
||||
depot_tools_path = join(third_party_path, "depot_tools")
|
||||
gn_path = join(depot_tools_path, "gn")
|
||||
ninja_path = join(depot_tools_path, "ninja")
|
||||
|
||||
# Add third_party/depot_tools to PATH because some google tools (e.g.
|
||||
# tool_wrapper, download_from_google_storage) use some google specific python
|
||||
# wrapper.
|
||||
os.environ["PATH"] = depot_tools_path + os.pathsep + os.environ["PATH"]
|
||||
fix_symlinks()
|
||||
|
||||
os.chdir(root_path)
|
||||
|
||||
gn_path = join(depot_tools_path, "gn")
|
||||
ninja_path = join(depot_tools_path, "ninja")
|
||||
|
||||
if options.build_path:
|
||||
build_path = options.build_path
|
||||
else:
|
||||
|
@ -63,7 +58,7 @@ if not os.path.exists(args_filename) or options.args:
|
|||
with open(args_filename, "w+") as f:
|
||||
f.write("\n".join(gn_args) + "\n")
|
||||
|
||||
run([gn_path, "gen", build_path])
|
||||
run([gn_path, "gen", build_path], env=google_env())
|
||||
|
||||
target = " ".join(targets) if targets else ":all"
|
||||
run([ninja_path, "-C", build_path, target])
|
||||
run([ninja_path, "-C", build_path, target], env=google_env())
|
||||
|
|
|
@ -1,9 +1,10 @@
|
|||
#!/usr/bin/env python
|
||||
import os
|
||||
from util import run, find_exts
|
||||
from third_party import third_party_path, fix_symlinks, google_env
|
||||
from util import root_path, run, find_exts
|
||||
|
||||
fix_symlinks()
|
||||
|
||||
root_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
|
||||
third_party_path = os.path.join(root_path, "third_party")
|
||||
prettier = os.path.join(third_party_path, "node_modules", "prettier",
|
||||
"bin-prettier.js")
|
||||
tools_path = os.path.join(root_path, "tools")
|
||||
|
@ -15,7 +16,7 @@ os.chdir(root_path)
|
|||
run(["clang-format", "-i", "-style", "Google"] + find_exts("src", ".cc", ".h"))
|
||||
|
||||
for fn in ["BUILD.gn", ".gn"] + find_exts("build_extra", ".gn", ".gni"):
|
||||
run(["third_party/depot_tools/gn", "format", fn])
|
||||
run(["third_party/depot_tools/gn", "format", fn], env=google_env())
|
||||
|
||||
# TODO(ry) Install yapf in third_party.
|
||||
run(["yapf", "-i"] + find_exts("tools/", ".py") +
|
||||
|
|
|
@ -1,29 +1,7 @@
|
|||
#!/usr/bin/env python
|
||||
import os
|
||||
import sys
|
||||
from util import run
|
||||
import third_party
|
||||
|
||||
root_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
|
||||
third_party_path = os.path.join(root_path, "third_party")
|
||||
depot_tools_path = os.path.join(third_party_path, "depot_tools")
|
||||
os.chdir(root_path)
|
||||
third_party.fix_symlinks()
|
||||
|
||||
|
||||
def download(filename):
|
||||
run([
|
||||
"python",
|
||||
os.path.join(depot_tools_path + '/download_from_google_storage.py'),
|
||||
'--platform=' + sys.platform, '--no_auth', '--bucket=chromium-gn',
|
||||
'--sha1_file',
|
||||
os.path.join(root_path, filename)
|
||||
])
|
||||
|
||||
|
||||
if sys.platform == 'win32':
|
||||
download("third_party/v8/buildtools/win/gn.exe.sha1")
|
||||
elif sys.platform == 'darwin':
|
||||
download("third_party/v8/buildtools/mac/gn.sha1")
|
||||
elif sys.platform.startswith('linux'):
|
||||
download("third_party/v8/buildtools/linux64/gn.sha1")
|
||||
|
||||
run(['python', 'third_party/v8/tools/clang/scripts/update.py', '--if-needed'])
|
||||
third_party.download_gn()
|
||||
third_party.download_clang()
|
||||
|
|
|
@ -1,35 +1,13 @@
|
|||
#!/usr/bin/env python
|
||||
# Only run this script if you are changing Deno's dependencies.
|
||||
|
||||
import os
|
||||
from os.path import join
|
||||
from util import run, remove_and_symlink
|
||||
|
||||
root_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
|
||||
third_party_path = join(root_path, "third_party")
|
||||
|
||||
try:
|
||||
os.makedirs(third_party_path)
|
||||
except:
|
||||
pass
|
||||
os.chdir(third_party_path)
|
||||
|
||||
# Run yarn to install JavaScript dependencies.
|
||||
remove_and_symlink("../package.json", "package.json")
|
||||
remove_and_symlink("../yarn.lock", "yarn.lock")
|
||||
run(["yarn"])
|
||||
# Run cargo to install Rust dependencies.
|
||||
run(["cargo", "fetch", "--manifest-path=" + root_path + "/Cargo.toml"],
|
||||
envs={'CARGO_HOME': third_party_path + '/rust_crates'})
|
||||
# Run gclient to install other dependencies.
|
||||
run(["gclient", "sync", "--reset", "--shallow", "--no-history", "--nohooks"],
|
||||
envs={'GCLIENT_FILE': root_path + "/gclient_config.py"})
|
||||
# TODO(ry) Is it possible to remove these symlinks?
|
||||
remove_and_symlink("v8/third_party/googletest", "googletest", True)
|
||||
remove_and_symlink("v8/third_party/jinja2", "jinja2", True)
|
||||
remove_and_symlink("v8/third_party/llvm-build", "llvm-build", True)
|
||||
remove_and_symlink("v8/third_party/markupsafe", "markupsafe", True)
|
||||
|
||||
# Run this script if you are changing Deno's dependencies.
|
||||
# To update the deno_third_party git repo after running this, try the following:
|
||||
# cd third_party
|
||||
# find . -type f | grep -v "\.git" | xargs -I% git add -f --no-warn-embedded-repo "%"
|
||||
|
||||
import third_party
|
||||
|
||||
third_party.fix_symlinks()
|
||||
|
||||
third_party.run_yarn()
|
||||
third_party.run_cargo()
|
||||
third_party.run_gclient_sync()
|
||||
|
|
166
tools/third_party.py
Normal file
166
tools/third_party.py
Normal file
|
@ -0,0 +1,166 @@
|
|||
#!/usr/bin/env python
|
||||
# This script contains helper functions to work with the third_party subrepo.
|
||||
|
||||
import os
|
||||
import sys
|
||||
from os import path
|
||||
from util import find_exts, make_env, remove_and_symlink, rmtree, root_path, run
|
||||
|
||||
|
||||
# Helper function that returns the full path to a subpath of the repo root.
|
||||
def root(*subpath_parts):
|
||||
return path.normpath(path.join(root_path, *subpath_parts))
|
||||
|
||||
|
||||
# Helper function that returns the full path to a file/dir in third_party.
|
||||
def tp(*subpath_parts):
|
||||
return root("third_party", *subpath_parts)
|
||||
|
||||
|
||||
third_party_path = tp()
|
||||
depot_tools_path = tp("depot_tools")
|
||||
rust_crates_path = tp("rust_crates")
|
||||
|
||||
|
||||
# This function creates or modifies an environment so that it matches the
|
||||
# expectations of various google tools (gn, gclient, etc).
|
||||
def google_env(env=None, merge_env={}, depot_tools_path=depot_tools_path):
|
||||
env = make_env(env=env, merge_env=merge_env)
|
||||
# Depot_tools to be in the PATH, before Python.
|
||||
path_prefix = depot_tools_path + os.path.pathsep
|
||||
if not env['PATH'].startswith(path_prefix):
|
||||
env['PATH'] = path_prefix + env['PATH']
|
||||
# We're not using Google's internal infrastructure.
|
||||
if os.name == 'nt' and not 'DEPOT_TOOLS_WIN_TOOLCHAIN' in env:
|
||||
env['DEPOT_TOOLS_WIN_TOOLCHAIN'] = "0"
|
||||
return env
|
||||
|
||||
|
||||
def fix_symlinks():
|
||||
# Ensure the third_party directory exists.
|
||||
try:
|
||||
os.makedirs(third_party_path)
|
||||
except:
|
||||
pass
|
||||
|
||||
# Make symlinks to Yarn metadata living in the root repo.
|
||||
remove_and_symlink("../package.json", tp("package.json"))
|
||||
remove_and_symlink("../yarn.lock", tp("yarn.lock"))
|
||||
|
||||
# TODO(ry) Is it possible to remove these symlinks?
|
||||
remove_and_symlink("v8/third_party/googletest", tp("googletest"), True)
|
||||
remove_and_symlink("v8/third_party/jinja2", tp("jinja2"), True)
|
||||
remove_and_symlink("v8/third_party/llvm-build", tp("llvm-build"), True)
|
||||
remove_and_symlink("v8/third_party/markupsafe", tp("markupsafe"), True)
|
||||
|
||||
# On Windows, git doesn't create the right type of symlink if the symlink
|
||||
# and it's target are in different repos. Here we fix the symlinks that exist
|
||||
# in the root repo while their target is in the third_party repo.
|
||||
remove_and_symlink("third_party/node_modules", root("node_modules"), True)
|
||||
remove_and_symlink("third_party/v8/build", root("build"), True)
|
||||
remove_and_symlink("third_party/v8/buildtools", root("buildtools"), True)
|
||||
remove_and_symlink("third_party/v8/build_overrides",
|
||||
root("build_overrides"), True)
|
||||
remove_and_symlink("third_party/v8/testing", root("testing"), True)
|
||||
|
||||
|
||||
# Run Yarn to install JavaScript dependencies.
|
||||
def run_yarn():
|
||||
run(["yarn"], cwd=third_party_path)
|
||||
|
||||
|
||||
# Run Cargo to install Rust dependencies.
|
||||
def run_cargo():
|
||||
# Deletes the cargo index lockfile; it appears that cargo itself doesn't do it.
|
||||
# If the lockfile ends up in the git repo, it'll make cargo hang for everyone
|
||||
# else who tries to run sync_third_party.
|
||||
def delete_lockfile():
|
||||
lockfiles = find_exts(
|
||||
path.join(rust_crates_path, "registry/index"), '.cargo-index-lock')
|
||||
for lockfile in lockfiles:
|
||||
os.remove(lockfile)
|
||||
|
||||
# Delete the index lockfile in case someone accidentally checked it in.
|
||||
delete_lockfile()
|
||||
|
||||
run(["cargo", "fetch", "--manifest-path=" + root("Cargo.toml")],
|
||||
cwd=third_party_path,
|
||||
merge_env={'CARGO_HOME': rust_crates_path})
|
||||
|
||||
# Delete the lockfile again so it doesn't end up in the git repo.
|
||||
delete_lockfile()
|
||||
|
||||
|
||||
# Run gclient to install other dependencies.
|
||||
def run_gclient_sync():
|
||||
# Depot_tools will normally try to self-update, which will fail because
|
||||
# it's not checked out from it's own git repository; gclient will then try
|
||||
# to fix things up and not succeed, and and we'll end up with a huge mess.
|
||||
# To work around this, we rename the `depot_tools` directory to
|
||||
# `{root_path}/depot_tools_temp` first, and we set DEPOT_TOOLS_UPDATE=0 in
|
||||
# the environment so depot_tools doesn't attempt to self-update.
|
||||
# Since depot_tools is listed in .gclient_entries, gclient will install a
|
||||
# fresh copy in `third_party/depot_tools`.
|
||||
# If it all works out, we remove the depot_tools_temp directory afterwards.
|
||||
depot_tools_temp_path = root("depot_tools_temp")
|
||||
|
||||
# Rename depot_tools to depot_tools_temp.
|
||||
try:
|
||||
os.rename(depot_tools_path, depot_tools_temp_path)
|
||||
except:
|
||||
# If renaming failed, and the depot_tools_temp directory already exists,
|
||||
# assume that it's still there because a prior run_gclient_sync() call
|
||||
# failed half-way, before we got the chance to remove the temp dir.
|
||||
# We'll use whatever is in the temp dir that was already there.
|
||||
# If not, the user can recover by removing the temp directory manually.
|
||||
if path.isdir(depot_tools_temp_path):
|
||||
pass
|
||||
else:
|
||||
raise
|
||||
|
||||
args = [
|
||||
"gclient", "sync", "--reset", "--shallow", "--no-history", "--nohooks"
|
||||
]
|
||||
envs = {
|
||||
'DEPOT_TOOLS_UPDATE': "0",
|
||||
'GCLIENT_FILE': root("gclient_config.py")
|
||||
}
|
||||
env = google_env(depot_tools_path=depot_tools_temp_path, merge_env=envs)
|
||||
run(args, cwd=third_party_path, env=env)
|
||||
|
||||
# Delete the depot_tools_temp directory, but not before verifying that
|
||||
# gclient did indeed install a fresh copy.
|
||||
# Also check that `{depot_tools_temp_path}/gclient.py` exists, so a typo in
|
||||
# this script won't accidentally blow out someone's home dir.
|
||||
if (path.isdir(path.join(depot_tools_path, ".git"))
|
||||
and path.isfile(path.join(depot_tools_path, "gclient.py"))
|
||||
and path.isfile(path.join(depot_tools_temp_path, "gclient.py"))):
|
||||
rmtree(depot_tools_temp_path)
|
||||
|
||||
|
||||
# Download gn from Google storage.
|
||||
def download_gn():
|
||||
if sys.platform == 'win32':
|
||||
sha1_file = "v8/buildtools/win/gn.exe.sha1"
|
||||
elif sys.platform == 'darwin':
|
||||
sha1_file = "v8/buildtools/mac/gn.sha1"
|
||||
elif sys.platform.startswith('linux'):
|
||||
sha1_file = "v8/buildtools/linux64/gn.sha1"
|
||||
|
||||
run([
|
||||
"python",
|
||||
tp('depot_tools/download_from_google_storage.py'),
|
||||
'--platform=' + sys.platform,
|
||||
'--no_auth',
|
||||
'--bucket=chromium-gn',
|
||||
'--sha1_file',
|
||||
tp(sha1_file),
|
||||
],
|
||||
env=google_env())
|
||||
|
||||
|
||||
# Download clang by calling the clang update script.
|
||||
def download_clang():
|
||||
run(['python',
|
||||
tp('v8/tools/clang/scripts/update.py'), '--if-needed'],
|
||||
env=google_env())
|
|
@ -1,20 +1,30 @@
|
|||
# Copyright 2018 Ryan Dahl <ry@tinyclouds.org>
|
||||
# All rights reserved. MIT License.
|
||||
import os
|
||||
import shutil
|
||||
import stat
|
||||
import subprocess
|
||||
|
||||
executable_suffix = ".exe" if os.name == "nt" else ""
|
||||
root_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
|
||||
|
||||
|
||||
def run(args, quiet=False, envs={}):
|
||||
def make_env(merge_env={}, env=None):
|
||||
if env is None:
|
||||
env = os.environ
|
||||
env = env.copy()
|
||||
for key in merge_env.keys():
|
||||
env[key] = merge_env[key]
|
||||
return env
|
||||
|
||||
|
||||
def run(args, quiet=False, cwd=None, env=None, merge_env={}):
|
||||
args[0] = os.path.normpath(args[0])
|
||||
if not quiet:
|
||||
print " ".join(args)
|
||||
env = os.environ.copy()
|
||||
for key in envs.keys():
|
||||
env[key] = envs[key]
|
||||
args[0] = os.path.normpath(args[0])
|
||||
env = make_env(env=env, merge_env=merge_env)
|
||||
shell = os.name == "nt" # Run through shell to make .bat/.cmd files work.
|
||||
subprocess.check_call(args, env=env, shell=shell)
|
||||
subprocess.check_call(args, cwd=cwd, env=env, shell=shell)
|
||||
|
||||
|
||||
def remove_and_symlink(target, name, target_is_dir=False):
|
||||
|
@ -68,3 +78,14 @@ def find_exts(directory, *extensions):
|
|||
matches.append(os.path.join(root, filename))
|
||||
break
|
||||
return matches
|
||||
|
||||
|
||||
# The Python equivalent of `rm -rf`.
|
||||
def rmtree(directory):
|
||||
# On Windows, shutil.rmtree() won't delete files that have a readonly bit.
|
||||
# Git creates some files that do. The 'onerror' callback deals with those.
|
||||
def rm_readonly(func, path, _):
|
||||
os.chmod(path, stat.S_IWRITE)
|
||||
func(path)
|
||||
|
||||
shutil.rmtree(directory, onerror=rm_readonly)
|
||||
|
|
Loading…
Reference in a new issue