travis: Set CARGO_HOME to third_party/rust_crates (#1370)

Adds prebuilt hyperfine and sccache binaries 

Only build test_cc for LSAN

Add Cargo.lock

Only run benchmarks in master

.appveyor.yml

@@ -250,10 +250,8 @@ init:
   - git config --global core.symlinks true
 
 install:
-  # Download and install sccache
+  # Make sure the PATH includes the prebuilt files (downloaded during setup.py)
-  - appveyor DownloadFile https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2018-04-26-sccache-x86_64-pc-windows-msvc
+  - set PATH=%PATH%;%CD%\prebuilt\win\
-  - mv 2018-04-26-sccache-x86_64-pc-windows-msvc sccache.exe
-  - set PATH=%PATH%;%CD%
 
   # Clone the third_party submodule.
   - ps: |-

.gitignore

@@ -4,7 +4,6 @@
 /target/
 *.pyc
 gclient_config.py_entries
-Cargo.lock
 yarn.lock
 # npm deps
 node_modules

.travis.yml

@@ -8,7 +8,7 @@ matrix:
   - os: osx
 env:
   global:
-  - CARGO_HOME=$HOME/.cargo/
+  - CARGO_HOME=$TRAVIS_BUILD_DIR/third_party/rust_crates/
   - RUSTUP_HOME=$HOME/.rustup/
   - RUST_BACKTRACE=1
   - CARGO_TARGET_DIR=$HOME/target
@@ -20,8 +20,10 @@ env:
   - secure: "hlQoMNDWgsdFPDAIwE4VCr21w6ftD8BGLNE8w23+9wCqMs8TXCGzdzpTYzSePmToUfF1WjGQTF3advifzg7pdrUp41ThWzRkEjiRoXEORkKFuZENgLdi4ZZsvOYIfX5dfTcw7wkLEuGkywy1D9s/4uJydiVqYYM3fWI7U2ax8+wpenRH59moUeHqGGzPk8Hy4wcD7SoP+fvuOYJ+KjU2urbddwrVus9jnmXbx0Srq/L/ayBr0aLpZXcYZ3nSMnXpc9mo3YuVZQR42+ARHI5R7V1wrCa/025/O4kpSry/bipyN7Oyry/WY+9Hu3EDdlIUosJsWVHhv5s8ZuZWJG/0VITUP+MnDexGNkbbbDzzwk/7dDhTo1DRpT7fmSFVzr6J9bw4JEU9k//znLuCOnRLoWagysQUVUZQXDh7TTfMmGP0wdsUggfr8azrkZF3hjg+g+aOlz0ZpVtYQ8hdhvLQRp4pmxyQlsKygf0fFXWJvAp40KXIkrKXJ6hXIGdfWirZ3dgc8rCgVdOKi6Hy39EKOiIdP3j2J2tv3NTPkUi7z+BaDc5cPCaarR8ZvLLkv1lvYTFap/fk+TXpM3AS0hs1o91fyJ0l/P0KKTp7V8MoNC7iXhzCK/InNgpcs0rK7w7QOlEHihRlY3Ya4m3KvSRbL/P55TneX5xwr6yP719ARs4="
 cache:
   directories:
-  - "$CARGO_HOME"
   - "$RUSTUP_HOME"
+  - $CARGO_HOME/registry/index/
+  - $CARGO_HOME/registry/cache/
+  - prebuilt/
   - third_party/v8/build/linux/debian_sid_amd64-sysroot/
   - third_party/v8/third_party/llvm-build/
 install:
@@ -39,31 +41,14 @@ install:
   rustc --version
   cargo --version
 - |-
-  # Install and start sccache.
+  if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
-  mkdir -p $HOME/bin
+    export PATH="`pwd`/prebuilt/mac:$PATH"
-  export PATH="$HOME/bin:$PATH"
+  else
-  if [ ! $(which sccache) ]; then
+    export PATH="`pwd`/prebuilt/linux64:$PATH"
-    if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
-      curl -fo $HOME/bin/sccache https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2018-04-02-sccache-x86_64-apple-darwin
-    else
-      curl -fo $HOME/bin/sccache https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2018-04-02-sccache-x86_64-unknown-linux-musl
-    fi
-    chmod +x $HOME/bin/sccache
-  fi
-  sccache --version
-  sccache --start-server
-- |-
-  # Install hyperfine (if benchmarking is enabled for this build).
-  if [ $BENCHMARK ]; then
-    if [ ! $(which hyperfine) ]; then
-      cargo install hyperfine
-    fi
-    hyperfine --version
   fi
 - |-
   # Remove unnnecessary cargo and rustup directories.
   # This keeps the Travis CI cache small and fast.
-  rm -rf "$CARGO_HOME"registry
   rm -rf "$RUSTUP_HOME"downloads
   rm -rf "$RUSTUP_HOME"tmp
   rm -rf "$RUSTUP_HOME"toolchains/*/etc
@@ -85,7 +70,7 @@ script:
   # We want to detect leaks during the build process as well as when executing
   # the tests. So set the ASAN_OPTIONS env var before build.py is run.
   export ASAN_OPTIONS=detect_leaks=1
-  ./tools/build.py -C target/debug -j2
+  ./tools/build.py -C target/debug -j2 test_cc
   ./target/debug/test_cc
 
 - |-
@@ -95,15 +80,8 @@ script:
 
 - |-
   # Cargo check
-  cargo check -j2 --release
+  cargo check -j2 --release --locked
 
-after_success:
-- |
-  # Run benchmarks and publish the result to github pages.
-  if [ $BENCHMARK ]; then
-    ./tools/benchmark.py target/release &&
-    cp -r website/* gh-pages/
-  fi
 before_deploy:
 - gzip -c target/release/deno > target/release/deno_${TRAVIS_OS_NAME}_x64.gz
 deploy:
@@ -115,7 +93,9 @@ deploy:
     tags: true
     repo: denoland/deno
   skip-cleanup: true
+# Run benchmarks and publish the result to github pages.
 - provider: pages
+  script: ./tools/benchmark.py target/release && cp -r website/* gh-pages/
   github-token: *github-token
   keep-history: true
   local-dir: gh-pages

Docs.md

@@ -387,6 +387,15 @@ We use Flatbuffers to define common structs and enums between TypeScript and
 Rust. These common data structures are defined in
 https://github.com/denoland/deno/blob/master/src/msg.fbs
 
+### Internal: Updating prebuilt binaries
+
+```
+./third_party/depot_tools/upload_to_google_storage.py -b denoland  \
+  -e ~/.config/gcloud/legacy_credentials/ry@tinyclouds.org/.boto `which sccache`
+mv `which sccache`.sha1 prebuilt/linux64/
+gsutil acl ch -u AllUsers:R gs://denoland/608be47bf01004aa11d4ed06955414e93934516e
+```
+
 ## Contributing
 
 See

prebuilt/.gitignore

@@ -0,0 +1,5 @@
+*.a
+*.lib
+*.exe
+sccache
+hyperfine

prebuilt/linux64/hyperfine.sha1

@@ -0,0 +1 @@
+c01e9c3d9169065911a1c74e935e0fdf15094e36

prebuilt/linux64/sccache.sha1

@@ -0,0 +1 @@
+be5e7f887af952dbe190d31290b84830da859f72

prebuilt/mac/hyperfine.sha1

@@ -0,0 +1 @@
+fe12887d9fcce693fa333992cb46c6ab7ada181e

prebuilt/mac/sccache.sha1

@@ -0,0 +1 @@
+962934c8a014c9342cee3b3e82dc0cd93d32875a

prebuilt/win/hyperfine.exe.sha1

@@ -0,0 +1 @@
+ad87698076d05b3d7d2cdf0705302ccf9444e3f2

prebuilt/win/sccache.exe.sha1

@@ -0,0 +1 @@
+6e470d96751254953ac54d518baafec013440fe0

tools/benchmark.py

@@ -16,6 +16,7 @@ import tempfile
 import http_server
 import throughput_benchmark
 from http_benchmark import http_benchmark
+import prebuilt
 
 # The list of the tuples of the benchmark name and arguments
 exec_time_benchmarks = [
@@ -156,7 +157,9 @@ def main(argv):
 
     os.chdir(root_path)
     import_data_from_gh_pages()
-    # TODO: Use hyperfine in //third_party
+
+    prebuilt.load_hyperfine()
+
     run([
         "hyperfine", "--ignore-failure", "--export-json", benchmark_file,
         "--warmup", "3"

tools/prebuilt.py

@@ -0,0 +1,37 @@
+import sys
+import os
+from util import run, root_path
+from third_party import tp, google_env
+
+
+def download_prebuilt(sha1_file):
+    run([
+        "python",
+        tp('depot_tools/download_from_google_storage.py'),
+        '--platform=' + sys.platform,
+        '--no_auth',
+        '--bucket=denoland',
+        '--sha1_file',
+        sha1_file,
+    ],
+        env=google_env())
+
+
+def load_sccache():
+    if sys.platform == 'win32':
+        p = "prebuilt/win/sccache.exe"
+    elif sys.platform.startswith('linux'):
+        p = "prebuilt/linux64/sccache"
+    elif sys.platform == 'darwin':
+        p = "prebuilt/mac/sccache"
+    download_prebuilt(p + ".sha1")
+    return os.path.join(root_path, p)
+
+
+def load_hyperfine():
+    if sys.platform == 'win32':
+        download_prebuilt("prebuilt/win/hyperfine.exe.sha1")
+    elif sys.platform.startswith('linux'):
+        download_prebuilt("prebuilt/linux64/hyperfine.sha1")
+    elif sys.platform == 'darwin':
+        download_prebuilt("prebuilt/mac/hyperfine.sha1")

tools/setup.py

@@ -7,6 +7,7 @@ import os
 import re
 import sys
 from distutils.spawn import find_executable
+import prebuilt
 
 
 def main():
@@ -19,7 +20,6 @@ def main():
     third_party.download_clang_format()
     third_party.download_clang()
     third_party.maybe_download_sysroot()
-
     write_lastchange()
 
     mode = build_mode(default=None)
@@ -113,8 +113,12 @@ def generate_gn_args(mode):
     if "DENO_BUILD_ARGS" in os.environ:
         out += os.environ["DENO_BUILD_ARGS"].split()
 
+    cacher = prebuilt.load_sccache()
+    if not os.path.exists(cacher):
+        cacher = find_executable("sccache") or find_executable("ccache")
+
     # Check if ccache or sccache are in the path, and if so we set cc_wrapper.
-    cc_wrapper = find_executable("sccache") or find_executable("ccache")
+    cc_wrapper = cacher
     if cc_wrapper:
         # The gn toolchain does not shell escape cc_wrapper, so do it here.
         out += ['cc_wrapper=%s' % gn_string(shell_quote(cc_wrapper))]
@@ -124,7 +128,7 @@ def generate_gn_args(mode):
             out += ["treat_warnings_as_errors=false"]
 
     # Look for sccache; if found, set rustc_wrapper.
-    rustc_wrapper = find_executable("sccache")
+    rustc_wrapper = cacher
     if rustc_wrapper:
         out += ['rustc_wrapper=%s' % gn_string(rustc_wrapper)]