From 869ddd8d0019683c007956bae17fc07cd30f39f7 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Fri, 10 May 2019 10:17:52 +0200 Subject: [PATCH] ci: still use aws for caches --- .azure-pipelines/auto.fanout.yml | 4 ++-- .azure-pipelines/auto.yml | 5 ----- .azure-pipelines/steps/macos.yml | 10 +++++----- .azure-pipelines/steps/run-script.yml | 6 +++--- .azure-pipelines/steps/windows.yml | 6 +++--- src/ci/docker/run.sh | 23 +++-------------------- src/ci/docker/scripts/sccache.sh | 3 +-- 7 files changed, 17 insertions(+), 40 deletions(-) diff --git a/.azure-pipelines/auto.fanout.yml b/.azure-pipelines/auto.fanout.yml index e12836257d9..def19a87076 100644 --- a/.azure-pipelines/auto.fanout.yml +++ b/.azure-pipelines/auto.fanout.yml @@ -160,7 +160,7 @@ jobs: - script: | REM md sccache - REM powershell -Command "iwr -outf sccache\sccache.exe https://rustlangtools.blob.core.windows.net/public/stable-x86_64-pc-windows-msvc.sccache.exe" + REM powershell -Command "iwr -outf sccache\sccache.exe https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2018-04-26-sccache-x86_64-pc-windows-msvc" echo ##vso[task.prependpath]%CD%\sccache displayName: Download and install sccache @@ -392,7 +392,7 @@ jobs: mkdir -p $HOME/rustsrc echo "##vso[task.setvariable variable=PATH;]$PATH" - curl -fo /usr/local/bin/sccache https://rustlangtools.blob.core.windows.net/public/stable-x86_64-apple-darwin.sccache + curl -fo /usr/local/bin/sccache https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2018-04-02-sccache-x86_64-apple-darwin chmod +x /usr/local/bin/sccache curl -fo /usr/local/bin/stamp https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2017-03-17-stamp-x86_64-apple-darwin diff --git a/.azure-pipelines/auto.yml b/.azure-pipelines/auto.yml index 5f4c155ce98..189536cff5f 100644 --- a/.azure-pipelines/auto.yml +++ b/.azure-pipelines/auto.yml @@ -1,11 +1,6 @@ # # Azure Pipelines "auto" branch build for Rust on Linux, macOS, and Windows. # -# Notes: -# To get sccache working, I (John Erickson) had to build it with the Azure feature enabled -# and with openssl statically-linked for Linux. -# Here's the build (with a backpointer to source) of where the bits came from: -# https://dev.azure.com/johnterickson/rust-lang/_build/results?buildId=275 pr: none # trigger: diff --git a/.azure-pipelines/steps/macos.yml b/.azure-pipelines/steps/macos.yml index a217df6f308..e88b5613e46 100644 --- a/.azure-pipelines/steps/macos.yml +++ b/.azure-pipelines/steps/macos.yml @@ -3,21 +3,21 @@ steps: - bash: | export PATH=$PATH:$HOME/.local/bin:$HOME/Library/Python/2.7/bin/ - mkdir -p $HOME/rustsrc + mkdir -p $HOME/rustsrc echo "##vso[task.setvariable variable=PATH;]$PATH" - curl -fo /usr/local/bin/sccache https://rustlangtools.blob.core.windows.net/public/stable-x86_64-apple-darwin.sccache + curl -fo /usr/local/bin/sccache https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2018-04-02-sccache-x86_64-apple-darwin chmod +x /usr/local/bin/sccache curl -fo /usr/local/bin/stamp https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2017-03-17-stamp-x86_64-apple-darwin chmod +x /usr/local/bin/stamp - + export CC=`pwd`/clang+llvm-7.0.0-x86_64-apple-darwin/bin/clang echo "##vso[task.setvariable variable=CC]$CC" - + export CXX=`pwd`/clang+llvm-7.0.0-x86_64-apple-darwin/bin/clang++ echo "##vso[task.setvariable variable=CXX]$CXX" - + echo "##vso[task.setvariable variable=AR]ar" displayName: Prep diff --git a/.azure-pipelines/steps/run-script.yml b/.azure-pipelines/steps/run-script.yml index 4ab3e2e06d3..cb8e8a629a7 100644 --- a/.azure-pipelines/steps/run-script.yml +++ b/.azure-pipelines/steps/run-script.yml @@ -5,7 +5,7 @@ steps: date && (curl -fs --head https://google.com | grep ^Date: | sed 's/Date: //g' || true) displayName: Log time information (before) -- bash: | +- bash: | which sccache stamp sh -x -c "$RUN_SCRIPT" env: @@ -15,8 +15,8 @@ steps: # Explicitly decrypt secret variables # See https://docs.microsoft.com/azure/devops/pipelines/process/variables?view=azure-devops&tabs=yaml%2Cbatch#secret-variables - SCCACHE_AZURE_CONNECTION_STRING: $(SCCACHE_AZURE_CONNECTION_STRING_SECRET) - DOCKER_LAYER_CACHE_AZURE_STORAGE_ACCOUNT_KEY: $(DOCKER_LAYER_CACHE_AZURE_STORAGE_ACCOUNT_KEY_SECRET) + AWS_ACCESS_KEY_ID: $(SCCACHE_AWS_ACCESS_KEY_ID) + AWS_SECRET_ACCESS_KEY: $(SCCACHE_AWS_SECRET_ACCESS_KEY) displayName: Run script - bash: | diff --git a/.azure-pipelines/steps/windows.yml b/.azure-pipelines/steps/windows.yml index 5dcba664bee..e3c06d543a9 100644 --- a/.azure-pipelines/steps/windows.yml +++ b/.azure-pipelines/steps/windows.yml @@ -106,7 +106,7 @@ steps: - script: | md sccache - powershell -Command "iwr -outf sccache\sccache.exe https://rustlangtools.blob.core.windows.net/public/stable-x86_64-pc-windows-msvc.sccache.exe" + powershell -Command "iwr -outf sccache\sccache.exe https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2018-04-26-sccache-x86_64-pc-windows-msvc" echo ##vso[task.prependpath]%CD%\sccache displayName: Download and install sccache @@ -155,6 +155,6 @@ steps: # explicitly decrypt secret variables # see https://docs.microsoft.com/en-us/azure/devops/pipelines/process/variables?view=azure-devops&tabs=yaml%2Cbatch - SCCACHE_AZURE_CONNECTION_STRING: $(SCCACHE_AZURE_CONNECTION_STRING_SECRET) - DOCKER_LAYER_CACHE_AZURE_STORAGE_ACCOUNT_KEY: $(DOCKER_LAYER_CACHE_AZURE_STORAGE_ACCOUNT_KEY_SECRET) + AWS_ACCESS_KEY_ID: $(SCCACHE_AWS_ACCESS_KEY_ID) + AWS_SECRET_ACCESS_KEY: $(SCCACHE_AWS_SECRET_ACCESS_KEY) displayName: Run script diff --git a/src/ci/docker/run.sh b/src/ci/docker/run.sh index cd913e30d11..0e1485601e9 100755 --- a/src/ci/docker/run.sh +++ b/src/ci/docker/run.sh @@ -1,7 +1,5 @@ #!/usr/bin/env bash -# ignore-tidy-linelength - set -e export MSYS_NO_PATHCONV=1 @@ -43,21 +41,9 @@ if [ -f "$docker_dir/$image/Dockerfile" ]; then cksum=$(sha512sum $hash_key | \ awk '{print $1}') - if [ "$DOCKER_LAYER_CACHE_AZURE_STORAGE_ACCOUNT" != "" ]; then - # install azcopy - echo "deb [arch=amd64] https://packages.microsoft.com/repos/microsoft-ubuntu-xenial-prod/ xenial main" > azure.list - sudo cp ./azure.list /etc/apt/sources.list.d/ - sudo apt-key adv --keyserver packages.microsoft.com --recv-keys EB3E94ADBE1229CF - sudo apt-get update - sudo apt-get install azcopy - - url="https://$DOCKER_LAYER_CACHE_AZURE_STORAGE_ACCOUNT.blob.core.windows.net/$DOCKER_LAYER_CACHE_AZURE_STORAGE_CONTAINER/$cksum" - upload="azcopy --quiet --destination $url --dest-key $DOCKER_LAYER_CACHE_AZURE_STORAGE_ACCOUNT_KEY" - else - s3url="s3://$SCCACHE_BUCKET/docker/$cksum" - url="https://s3-us-west-1.amazonaws.com/$SCCACHE_BUCKET/docker/$cksum" - upload="aws s3 cp - $s3url" - fi + s3url="s3://$SCCACHE_BUCKET/docker/$cksum" + url="https://s3-us-west-1.amazonaws.com/$SCCACHE_BUCKET/docker/$cksum" + upload="aws s3 cp - $s3url" echo "Attempting to download $url" rm -f /tmp/rustci_docker_cache @@ -134,9 +120,6 @@ if [ "$SCCACHE_BUCKET" != "" ]; then args="$args --env SCCACHE_REGION" args="$args --env AWS_ACCESS_KEY_ID" args="$args --env AWS_SECRET_ACCESS_KEY" -elif [ "$SCCACHE_AZURE_CONNECTION_STRING" != "" ]; then - args="$args --env SCCACHE_AZURE_CONNECTION_STRING" - args="$args --env SCCACHE_AZURE_BLOB_CONTAINER" else mkdir -p $HOME/.cache/sccache args="$args --env SCCACHE_DIR=/sccache --volume $HOME/.cache/sccache:/sccache" diff --git a/src/ci/docker/scripts/sccache.sh b/src/ci/docker/scripts/sccache.sh index d6f2f4fadb2..4c03419894e 100644 --- a/src/ci/docker/scripts/sccache.sh +++ b/src/ci/docker/scripts/sccache.sh @@ -1,7 +1,6 @@ set -ex -echo Use sccache built with Azure Storage support curl -fo /usr/local/bin/sccache \ - https://rustlangtools.blob.core.windows.net/public/stable-x86_64-unknown-linux-musl.sccache + https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2018-04-02-sccache-x86_64-unknown-linux-musl chmod +x /usr/local/bin/sccache