foster/denoland-deno
0
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2024-10-30 09:08:00 -04:00
Code Issues Wiki Activity

appveyor: automatically remove stale build outputs from cache

Browse source
This commit is contained in:
Bert Belder 2018-08-17 03:29:56 +02:00
parent d75010ddfe
commit 4a55724f81
No known key found for this signature in database
GPG key ID: 7A77887B2E2ED461
1 changed files with 140 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

144
.appveyor.yml
View file

@ -32,6 +32,19 @@ environment:
if ($LastExitCode -ne 0) { throw "Failure. Exit code: $LastExitCode" }
}
# Get-Tree lists all objects in a tree. It's different from Get-ChildItem
# in that the latter recurses through symlinks, which is problematic.
function Get-Tree([string[]] $Path, [switch] $Recurse, [switch] $Force) {
function Get-SubDirs([string[]] $Path) {
Get-ChildItem $Path -Force:$Force `
-Attributes Directory+!ReparsePoint |
foreach { $_.FullName } |
foreach { $_; Get-SubDirs $_ }
}
if ($Recurse) { $Path += Get-SubDirs $Path }
Get-ChildItem $Path -Force:$Force @args
}
# `Delete-Tree` is a simple wrapper around Remove-Item. It doesn't set
# an error status if one of the paths to be deleted doesn't exist.
function Delete-Tree([string[]] $Path) {
@ -46,9 +59,104 @@ environment:
}
}
# `$will_save_cache` equals true if the cache will be saved at the end.
$will_save_cache = -not $env:APPVEYOR_PULL_REQUEST_NUMBER -and
# We set file atimes to this date and see if it changes.
$FILE_NOT_NEEDED = Get-Date -Date "1984-04-11T00:00:00Z" # A good year.
# Traced files are stored a hash table, using their full path as key.
$not_needed = @{}
# Whether filesystem last access time tracking has been enabled yet.
$atime_enabled = $false
# Trace whether files are actually used, so we can find and remove files
# that unnecessary. We use this to avoid hoarding stale build outputs.
function Start-TraceFilesNeeded([string[]] $Path, [switch] $Recurse) {
# Don't enable if the cache won't be saved.
if (-not (Get-SaveCache)) { return }
# Identify (new) files to trace. A non-existing path is not an error.
$files = $Path |
where { Test-Path $_ } |
foreach { Get-Tree $_ -Recurse:$Recurse -File } |
where { -not $not_needed.ContainsKey($_.FullName) }
# Set newly traced files' last access time to very long ago.
$files | foreach { $_.LastAccessTime = $FILE_NOT_NEEDED }
# Add newly traced files to the hash table with unnecessary files.
$files | foreach { $not_needed.Add($_.FullName, $true) }
# Enable last access time tracking only if any files were found.
if ($files -and -not $atime_enabled) {
Exec { fsutil behavior set DisableLastAccess 0 }
Set-Variable -Name atime_enabled -Value $true -Scope 1
}
# Log statistics.
Write-Host "Tracing file access for $($files.Count) files."
}
# Marks files as needed.
# -Auto : Auto mark files if their access time has changed.
# -Path <path[]> : Explicitly mark file(s) as needed.
# -Recurse : Recurse into directories specified with -Path.
# -Reason <text> : Optional reason, written to the build log.
function Set-FilesNeeded([switch] $Auto, [string[]] $Path,
[switch] $Recurse, [string] $Reason) {
# Helper function.
function Mark([System.IO.FileSystemInfo[]] $Files, [string] $How) {
# Find matching files that are traced, then remove them.
$keys = $Files.FullName |
where { $_ -and $not_needed.ContainsKey($_) }
$keys | foreach { $not_needed.Remove($_) }
# Write log message.
if ($keys.Count -gt 0) {
Write-Host ("$Reason$(if ($Reason) { ': ' })" +
"$($keys.Count) files $How marked 'needed'.")
}
}
# Skip marking step if there are no files being traced.
if ($not_needed.Count -eq 0) { return }
# Auto mark files 'needed' when their last access time has changed.
if ($Auto) {
$files = $not_needed.Keys |
where { Test-Path $_ -PathType Leaf } |
foreach { Get-Item $_ -Force } |
where { $_.LastAccessTime -ne $FILE_NOT_NEEDED }
Mark -Files $files -How "automatically"
}
# Mark explicitly specified paths.
if ($Path) {
$files = $Path |
where { Test-Path $_ } |
foreach { Get-Tree $_ -Recurse:$Recurse -Force -File }
Mark -Files $files -How "explicitly"
}
}
# Clean up stale files and end file tracking.
function Stop-TraceFilesNeeded {
# Look for files that had their atime changed, and mark them needed.
Set-FilesNeeded -Auto
# Make a list of all files to delete, then delete them.
$files = $not_needed.Keys |
where { Test-Path $_ -PathType Leaf } |
foreach { Get-Item $_ -Force }
# Compute the total size of all deleted files.
$size_info = $files | measure -Property Length -Sum
$size_mb = "{0:N1}" -f ($size_info.Sum / (1024 * 1024))
# Delete files, as well as parent directories if they became empty.
$files | Remove-Item -Force
$dirs = $files | foreach {
try { while ($_ = $_.Directory) { $_.Delete(); $_ } } catch {}
}
# All unnecessary files are now gone.
$not_needed.Clear()
# Log about what what was cleaned up.
if ($files.Count -gt 0) {
Write-Host "Deleted $($files.Count) unnecessary files and",
"$($dirs.Count) directories ($size_mb MB)."
}
}
# Get-SaveCache returns $true if the cache will be saved at the end.
function Get-SaveCache {
-not $env:APPVEYOR_PULL_REQUEST_NUMBER -and
-not $env:APPVEYOR_CACHE_SKIP_SAVE -eq "true"
}
for:
# Do no save the build cache for feature branches. TODO: Once we have multiple
@ -96,14 +204,20 @@ install:
# Appveyor cache, it'll include only objects that were actually needed.
# This step is skipped if the cache is not going to be saved.
- ps: |-
if ($will_save_cache) {
if (Get-SaveCache) {
Push-Location $env:DENO_THIRD_PARTY_PATH
Exec { & git gc --prune=all }
Pop-Location
}
# Configure depot_tools and add it to the search path. This is necessary
# because, later in this script, we need to invoke ninja directly.
- ps: |-
$env:PATH = "$env:DENO_THIRD_PARTY_PATH\depot_tools;$env:PATH"
$env:DEPOT_TOOLS_WIN_TOOLCHAIN = "0"
# Install a recent Node.js version.
- ps: Install-Product node 10 x64
- ps: Install-Product -Product node -Version 10 -Platform x64
# Make sure the right Python version is in PATH, and others are not.
- ps: |-
@ -134,7 +248,7 @@ install:
# up because we removed the 'rust-docs' component.
# * The actual update is done by removing and reinstalling with rustup-init.
- ps: |-
if ($will_save_cache -and (Test-Path $env:CARGO_HOME)) {
if (Get-SaveCache -and (Test-Path $env:CARGO_HOME)) {
try {
Exec -NoNewLines { & rustup update stable-x86_64-pc-windows-msvc }
} catch {
@ -174,8 +288,23 @@ install:
- cargo --version
before_build:
# Mark all files in the build dir 'not needed' until proven otherwise.
# TODO: also track files in third_party that aren't checked into the repo.
- ps: Start-TraceFilesNeeded $env:DENO_BUILD_PATH -Recurse
# Download clang and gn, generate ninja files.
- python tools\setup.py
- ps: Set-FilesNeeded -Auto -Reason "Setup finished"
# Mark files that are produced during the build, and are known to ninja, as
# needed. We obtain this list by dry-running `ninja -t clean`.
# N.b.: we deliberately do *not* pass `:all` to the clean command. It misses
# some important files, e.g. .exe and .pdb files.
- ps: |-
$outputs = ninja -C $env:DENO_BUILD_PATH -n -t clean -g |
where { $_ -match "^Remove (.*)$" } |
foreach { "$env:DENO_BUILD_PATH\$($Matches[1])" }
Set-FilesNeeded -Auto -Path $outputs -Reason "Build dependency graph"
build_script:
# Attempt to work around multiple rustc instances messing with the same file
@ -183,7 +312,12 @@ build_script:
# TODO: fix this properly.
- ninja -C out\debug -j 1 build_extra/rust:winapi build_extra/rust:winapi-0.2
- python tools\build.py
- ps: Set-FilesNeeded -Auto -Reason "Build finished"
test_script:
- python tools\lint.py
- ps: Exec { & python tools\test.py $env:DENO_BUILD_PATH }
after_test:
# Remove stale files and empty dirs from the build directory.
- ps: Stop-TraceFilesNeeded