Bundle pywin32 in third_party

.appveyor.yml

@@ -223,8 +223,6 @@ for:
       APPVEYOR_CACHE_SKIP_SAVE: true
 
 cache:
-  # Python packages installed with `pip --user`.
-  - $(APPDATA)\Python
   # Rust stuff.
   - $(CARGO_HOME)
   - $(RUSTUP_HOME)
@@ -277,23 +275,11 @@ install:
   # Make sure the right Python version is in PATH, and others are not.
   - ps: |-
       # Remove the wrong Python version(s) from PATH.
-      $p = $env:PATH -split ";" | where {
-             -not (Test-Path "$_\python.exe") -and
-             -not (Test-Path "$_\pip.exe")
-           }
-      # Add binary dir for `pip --user` packages.
-      $p += "$env:APPDATA\Python\Scripts"
+      $p = $env:PATH -split ";" | where { -not (Test-Path "$_\python.exe") }
       # Add python27-x64.
       $p += "C:\Python27-x64"
-      $p += "C:\Python27-x64\Scripts"
       $env:PATH = $p -join ";"
 
-  # Pip on Appveyor is too old. Install a recent version in our user dir.
-  - python -m pip install --upgrade --user --no-cache-dir pip
-
-  # Install Python packages.
-  - pip install --upgrade --user --no-cache-dir pywin32 yapf
-
   # Add Rust/Cargo to PATH.
   - ps: $env:PATH += ";$env:CARGO_HOME\bin"
 
@@ -338,7 +324,6 @@ install:
         "print 'Python', version") -join "`n" | & python -
 
   # Log some more versions.
-  - pip --version
   - rustc --version
   - cargo --version
 

third_party

@@ -1 +1 @@
-Subproject commit 184a96ba38c8927b6622ade4efc1d2a58247c2f8
+Subproject commit 0da585d1d855ad720b7e5a5a0d0af380e66216f3

tools/sync_third_party.py

@@ -14,4 +14,5 @@ third_party.fix_symlinks()
 
 third_party.run_yarn()
 third_party.run_cargo()
+third_party.run_pip()
 third_party.run_gclient_sync()

tools/third_party.py

@@ -3,9 +3,12 @@
 # This script contains helper functions to work with the third_party subrepo.
 
 import os
+import site
 import sys
 from os import path
-from util import find_exts, make_env, remove_and_symlink, rmtree, root_path, run
+from util import add_env_path, find_exts, make_env, remove_and_symlink, rmtree
+from util import root_path, run
+from tempfile import mkdtemp
 
 
 # Helper function that returns the full path to a subpath of the repo root.
@@ -21,19 +24,47 @@ def tp(*subpath_parts):
 third_party_path = tp()
 depot_tools_path = tp("depot_tools")
 rust_crates_path = tp("rust_crates")
+python_packages_path = tp("python_packages")
 gn_path = tp(depot_tools_path, "gn")
 clang_format_path = tp(depot_tools_path, "clang-format")
 ninja_path = tp(depot_tools_path, "ninja")
 
+python_site_env = None
+
+
+# Creates/modifies an environment so python can find packages that are bundled
+# in the 'third_party' directory.
+def python_env(env=None, merge_env={}, depot_tools_path=depot_tools_path):
+    global python_site_env
+
+    # Use site.addsitedir() to determine which search paths would be considered
+    # if 'third_party/python_packages' was a site-packages directory.
+    # PATH is also updated, so windows can find the DLLs that ship with pywin32.
+    if python_site_env is None:
+        python_site_env = {}
+        temp = os.environ["PATH"], sys.path
+        os.environ["PATH"], sys.path = "", []
+        site.addsitedir(python_packages_path)  # Modifies PATH and sys.path.
+        python_site_env = {"PATH": os.environ["PATH"], "PYTHONPATH": sys.path}
+        os.environ["PATH"], sys.path = temp
+
+    # Make a new environment object.
+    env = make_env(env=env, merge_env=merge_env)
+    # Apply PATH and PYTHONPATH from the site-packages environment.
+    add_env_path(python_site_env["PATH"], env=env, key="PATH")
+    add_env_path(python_site_env["PYTHONPATH"], env=env, key="PYTHONPATH")
+
+    return env
+
 
 # This function creates or modifies an environment so that it matches the
 # expectations of various google tools (gn, gclient, etc).
 def google_env(env=None, merge_env={}, depot_tools_path=depot_tools_path):
-    env = make_env(env=env, merge_env=merge_env)
+    # Google tools need the python env too.
+    env = python_env(env=env, merge_env=merge_env)
+
     # Depot_tools to be in the PATH, before Python.
-    path_prefix = depot_tools_path + os.path.pathsep
-    if not env['PATH'].startswith(path_prefix):
-        env['PATH'] = path_prefix + env['PATH']
+    add_env_path(depot_tools_path, env=env, prepend=True)
 
     if os.name == "nt":  # Windows-only enviroment tweaks.
         # We're not using Google's internal infrastructure.
@@ -110,6 +141,29 @@ def run_cargo():
     delete_lockfile()
 
 
+# Install python packages with pip.
+def run_pip():
+    # Install an recent version of pip into a temporary directory. The version
+    # that is bundled with python is too old to support the next step.
+    temp_python_home = mkdtemp()
+    pip_env = {"PYTHONUSERBASE": temp_python_home}
+    run([sys.executable, "-m", "pip", "install", "--upgrade", "--user", "pip"],
+        cwd=third_party_path,
+        merge_env=pip_env)
+
+    # Install pywin32.
+    run([
+        sys.executable, "-m", "pip", "install", "--upgrade", "--target",
+        python_packages_path, "--platform=win_amd64", "--only-binary=:all:",
+        "pypiwin32"
+    ],
+        cwd=third_party_path,
+        merge_env=pip_env)
+
+    # Remove the temporary pip installation.
+    rmtree(temp_python_home)
+
+
 # Run gclient to install other dependencies.
 def run_gclient_sync():
     # Depot_tools will normally try to self-update, which will fail because

tools/util.py

@@ -19,6 +19,20 @@ def make_env(merge_env={}, env=None):
     return env
 
 
+def add_env_path(add, env, key="PATH", prepend=False):
+    dirs_left = env[key].split(os.pathsep) if key in env else []
+    dirs_right = add.split(os.pathsep) if type(add) is str else add
+
+    if prepend:
+        dirs_left, dirs_right = dirs_right, dirs_left
+
+    for dir in dirs_right:
+        if not dir in dirs_left:
+            dirs_left += [dir]
+
+    env[key] = os.pathsep.join(dirs_left)
+
+
 def run(args, quiet=False, cwd=None, env=None, merge_env={}):
     args[0] = os.path.normpath(args[0])
     if not quiet: