Rename run_hooks.py to setup.py

Moves 'gn gen' into setup.py

Make tools/build.py more ergonomic.

.travis.yml

@@ -6,10 +6,16 @@ cache:
   ccache: true
   directories:
    - $DEPOT_TOOLS_PATH
-   - $BUILD_PATH
+   - $DENO_BUILD_PATH
 env:
   global:
-    - BUILD_PATH=$HOME/out/Default
+    # is_debug, use_allocator, and ccache are used to speed travis.
+    # use_custom_libcxx=false and use_sysroot=false seem to be required to build on
+    # Ubuntu 14.04
+    # Help: How do you wrap long lines here?
+    - DENO_BUILD_ARGS="is_debug=false use_allocator=\"none\" use_custom_libcxx=false use_sysroot=false"
+    - DENO_BUILD_PATH=$HOME/out/Default
+    - DENO_BUILD_MODE=debug
     - DEPOT_TOOLS_PATH=$HOME/depot_tools
 before_install: |
   if ! [ -x $DEPOT_TOOLS_PATH/gclient ]; then
@@ -25,19 +31,13 @@ install:
  - curl -sSf https://sh.rustup.rs | sh -s -- -y
  - export PATH=$HOME/.cargo/bin:$PATH
  - rustc --version
- # TODO(ry) Do not depend on run_hooks because it calls
- # //third_party/depot_tools/download_from_google_storage.py
- # Use git lfs and combine run_hooks with sync_third_party?
- - ./tools/run_hooks.py
  # ccache needs the custom LLVM to be in PATH and other variables.
  - export PATH=`pwd`/third_party/llvm-build/Release+Asserts/bin:$PATH
  - export CCACHE_CPP2=yes
  - export CCACHE_SLOPPINESS=time_macros
  - ccache -s
- # is_debug, use_allocator, and ccache are used to speed travis.
+ - ./tools/setup.py
- # use_custom_libcxx=false and use_sysroot=false seem to be required to build on
+ - ./tools/build.py
- # Ubuntu 14.04
- - ./tools/build.py --build_path=$BUILD_PATH --args='is_debug=false use_allocator="none" use_custom_libcxx=false use_sysroot=false'
 script:
  - ./tools/lint.py
- - ./tools/test.py $BUILD_PATH
+ - ./tools/test.py $DENO_BUILD_PATH

README.md

@@ -76,9 +76,9 @@ To build:
     # Fetch deps.
     git clone --recurse-submodules https://github.com/ry/deno.git
     cd deno
-    ./tools/run_hooks.py
+    ./tools/setup.py
 
-    # Build
+    # Build.
     ./tools/build.py
 
     # Run
@@ -89,7 +89,7 @@ Other useful commands:
     # Call ninja manually.
     ./third_party/depot_tools/ninja -C out/debug :all
     # Build a release binary.
-    ./tools/build.py --mode=release :deno
+    DENO_BUILD_MODE=release ./tools/build.py :deno
     # List executable targets.
     ./third_party/depot_tools/gn ls out/debug //:* --as=output --type=executable
     # List build configuation.
@@ -100,3 +100,4 @@ Other useful commands:
     ./third_party/depot_tools/gn desc out/debug/ :deno
     ./third_party/depot_tools/gn help
 
+Env vars: `DENO_BUILD_MODE`, `DENO_BUILD_PATH`, `DENO_BUILD_ARGS`.

tools/build.py

@@ -1,64 +1,41 @@
 #!/usr/bin/env python
 # Copyright 2018 the Deno authors. All rights reserved. MIT license.
-import argparse
 import os
 import sys
 from os.path import join
-from third_party import depot_tools_path, third_party_path, fix_symlinks, google_env
+import third_party
-from util import root_path, run
+from util import root_path, run, run_output, build_path
-import distutils.spawn
 
-parser = argparse.ArgumentParser(description='')
+third_party.fix_symlinks()
-parser.add_argument(
-    '--build_path', default='', help='Directory to build into.')
-parser.add_argument(
-    '--args', default='', help='Specifies build arguments overrides.')
-parser.add_argument(
-    '--mode', default='debug', help='Build configuration: debug, release.')
-options, targets = parser.parse_known_args()
 
-fix_symlinks()
+print "DENO_BUILD_PATH:", build_path()
-
+if not os.path.isdir(build_path()):
-os.chdir(root_path)
+    print "DENO_BUILD_PATH does not exist. Run tools/setup.py"
-
-gn_path = join(depot_tools_path, "gn")
-ninja_path = join(depot_tools_path, "ninja")
-
-if options.build_path:
-    build_path = options.build_path
-else:
-    build_path = join(root_path, "out", options.mode)
-
-gn_args = []
-if options.args:
-    gn_args += options.args.split()
-
-if options.mode == "release":
-    gn_args += ["is_official_build=true"]
-elif options.mode == "debug":
-    pass
-else:
-    print "Bad mode {}. Use 'release' or 'debug' (default)" % options.mode
     sys.exit(1)
+os.chdir(build_path())
 
-# Check if ccache is in the path, and if so we cc_wrapper.
-ccache_path = distutils.spawn.find_executable("ccache")
-if ccache_path:
-    gn_args += [r'cc_wrapper="%s"' % ccache_path]
 
-# mkdir $build_path. We do this so we can write args.gn before running gn gen.
+def maybe_add_default_target(args):
-if not os.path.isdir(build_path):
+    lines = run_output(
-    os.makedirs(build_path)
+        [third_party.ninja_path, "-t", "targets"],
+        env=third_party.google_env(),
+        quiet=True).split("\n")
+    targets = [l.rsplit(":", 1)[0] for l in lines]
+    deno_targets = [target for target in targets if target.startswith(":")]
+    deno_targets += [target.lstrip(":") for target in deno_targets]
 
-# Rather than using gn gen --args we manually write the args.gn override file.
+    target_specified = False
-# This is to avoid quoting/escaping complications when passing overrides as
+    for a in args:
-# command-line arguments.
+        if a in deno_targets:
-args_filename = join(build_path, "args.gn")
+            target_specified = True
-if not os.path.exists(args_filename) or options.args:
+            break
-    with open(args_filename, "w+") as f:
+    if not target_specified:
-        f.write("\n".join(gn_args) + "\n")
+        args += [":all"]
+    return args
 
-run([gn_path, "gen", build_path], env=google_env())
 
-target = " ".join(targets) if targets else ":all"
+ninja_args = maybe_add_default_target(sys.argv[1:])
-run([ninja_path, "-C", build_path, target], env=google_env())
+
+run([third_party.ninja_path] + ninja_args,
+    env=third_party.google_env(),
+    quiet=True)

tools/run_hooks.py

@@ -1,7 +0,0 @@
-#!/usr/bin/env python
-import third_party
-
-third_party.fix_symlinks()
-
-third_party.download_gn()
-third_party.download_clang()

tools/setup.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+import third_party
+from util import run, build_path, build_mode
+import os
+import distutils.spawn
+
+third_party.fix_symlinks()
+third_party.download_gn()
+third_party.download_clang()
+
+
+def get_gn_args():
+    out = []
+    if build_mode() == "release":
+        out += ["is_official_build=true"]
+    elif build_mode() == "debug":
+        pass
+    else:
+        print "Bad mode {}. Use 'release' or 'debug' (default)" % build_mode()
+        sys.exit(1)
+    if "DENO_BUILD_ARGS" in os.environ:
+        out += os.environ["DENO_BUILD_ARGS"].split()
+
+    # Check if ccache is in the path, and if so we cc_wrapper.
+    ccache_path = distutils.spawn.find_executable("ccache")
+    if ccache_path:
+        out += [r'cc_wrapper="%s"' % ccache_path]
+
+    print "DENO_BUILD_ARGS:", out
+
+    return out
+
+
+# gn gen.
+for mode in ["release", "debug"]:
+    os.environ["DENO_BUILD_MODE"] = mode
+
+    gn_args = get_gn_args()
+
+    # mkdir $build_path(). We do this so we can write args.gn before running gn gen.
+    if not os.path.isdir(build_path()):
+        os.makedirs(build_path())
+
+    # Rather than using gn gen --args we manually write the args.gn override file.
+    # This is to avoid quoting/escaping complications when passing overrides as
+    # command-line arguments.
+    args_filename = os.path.join(build_path(), "args.gn")
+    if not os.path.exists(args_filename) or gn_args:
+        with open(args_filename, "w+") as f:
+            f.write("\n".join(gn_args) + "\n")
+
+    run([third_party.gn_path, "gen", build_path()],
+        env=third_party.google_env())

tools/third_party.py

@@ -20,6 +20,8 @@ def tp(*subpath_parts):
 third_party_path = tp()
 depot_tools_path = tp("depot_tools")
 rust_crates_path = tp("rust_crates")
+gn_path = tp(depot_tools_path, "gn")
+ninja_path = tp(depot_tools_path, "ninja")
 
 
 # This function creates or modifies an environment so that it matches the

tools/util.py

@@ -30,6 +30,15 @@ def run(args, quiet=False, cwd=None, env=None, merge_env={}):
         sys.exit(rc)
 
 
+def run_output(args, quiet=False, cwd=None, env=None, merge_env={}):
+    args[0] = os.path.normpath(args[0])
+    if not quiet:
+        print " ".join(args)
+    env = make_env(env=env, merge_env=merge_env)
+    shell = os.name == "nt"  # Run through shell to make .bat/.cmd files work.
+    return subprocess.check_output(args, cwd=cwd, env=env, shell=shell)
+
+
 def remove_and_symlink(target, name, target_is_dir=False):
     try:
         # On Windows, directory symlink can only be removed with rmdir().
@@ -92,3 +101,18 @@ def rmtree(directory):
         func(path)
 
     shutil.rmtree(directory, onerror=rm_readonly)
+
+
+def build_mode():
+    if "DENO_BUILD_MODE" in os.environ:
+        return os.environ["DENO_BUILD_MODE"]
+    else:
+        return "debug"
+
+
+# E.G. "out/debug"
+def build_path():
+    if "DENO_BUILD_PATH" in os.environ:
+        return os.environ["DENO_BUILD_PATH"]
+    else:
+        return os.path.join(root_path, "out", build_mode())