From 9b4f6de7a47c87f78f88a70397cf222d8e359e35 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Mon, 15 Jul 2019 08:23:56 -0700 Subject: [PATCH] azure: Prepare configuration for 4-core machines This commit updates some of our assorted Azure/CI configuration to prepare for some 4-core machines coming online. We're still in the process of performance testing them to get final numbers, but some changes are worth landing ahead of this. The updates here are: * Use `C:/` instead of `D:/` for submodule checkout since it should have plenty of space and the 4-core machines won't have `D:/` * Update `lzma-sys` to 0.1.14 which has support for VS2019, where 0.1.10 doesn't. * Update `src/ci/docker/run.sh` to work when it itself is running inside of a docker container (see the comment in the file for more info) * Print step timings on the `try` branch in addition to the `auto` branch in. The logs there should be seen by similarly many humans (not many) and can be useful for performance analysis after a `try` build runs. * Install the WIX and InnoSetup tools manually on Windows instead of relying on pre-installed copies on the VM. This gives us more control over what's being used on the Azure cloud right now (we control the version) and in the 4-core machines these won't be pre-installed. Note that on AppVeyor we actually already were installing InnoSetup, we just didn't carry that over on Azure! --- .../steps/install-windows-build-deps.yml | 25 ++++++++++ .azure-pipelines/steps/run.yml | 4 +- Cargo.lock | 7 ++- src/ci/docker/run.sh | 46 ++++++++++++++++--- src/ci/run.sh | 2 +- 5 files changed, 71 insertions(+), 13 deletions(-) diff --git a/.azure-pipelines/steps/install-windows-build-deps.yml b/.azure-pipelines/steps/install-windows-build-deps.yml index ed06679464c..c42c2311b49 100644 --- a/.azure-pipelines/steps/install-windows-build-deps.yml +++ b/.azure-pipelines/steps/install-windows-build-deps.yml @@ -1,4 +1,29 @@ steps: +# We use the WIX toolset to create combined installers for Windows, and these +# binaries are downloaded from +# https://github.com/wixtoolset/wix3 originally +- bash: | + set -e + curl -O https://rust-lang-ci2.s3-us-west-1.amazonaws.com/rust-ci-mirror/wix311-binaries.zip + echo "##vso[task.setvariable variable=WIX]`pwd`/wix" + mkdir -p wix/bin + cd wix/bin + 7z x ../../wix311-binaries.zip + displayName: Install wix + condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) + +# We use InnoSetup and its `iscc` program to also create combined installers. +# Honestly at this point WIX above and `iscc` are just holdovers from +# oh-so-long-ago and are required for creating installers on Windows. I think +# one is MSI installers and one is EXE, but they're not used so frequently at +# this point anyway so perhaps it's a wash! +- script: | + powershell -Command "$ProgressPreference = 'SilentlyContinue'; iwr -outf is-install.exe https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror/2017-08-22-is.exe" + is-install.exe /VERYSILENT /SUPPRESSMSGBOXES /NORESTART /SP- + echo ##vso[task.prependpath]C:\Program Files (x86)\Inno Setup 5 + displayName: Install InnoSetup + condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) + # We've had issues with the default drive in use running out of space during a # build, and it looks like the `C:` drive has more space than the default `D:` # drive. We should probably confirm this with the azure pipelines team at some diff --git a/.azure-pipelines/steps/run.yml b/.azure-pipelines/steps/run.yml index 28a958eb145..795eb73507c 100644 --- a/.azure-pipelines/steps/run.yml +++ b/.azure-pipelines/steps/run.yml @@ -111,8 +111,8 @@ steps: condition: and(succeeded(), not(variables.SKIP_JOB), ne(variables['Agent.OS'], 'Windows_NT')) displayName: Check out submodules (Unix) - script: | - if not exist D:\cache\rustsrc\NUL mkdir D:\cache\rustsrc - sh src/ci/init_repo.sh . /d/cache/rustsrc + if not exist C:\cache\rustsrc\NUL mkdir C:\cache\rustsrc + sh src/ci/init_repo.sh . /c/cache/rustsrc condition: and(succeeded(), not(variables.SKIP_JOB), eq(variables['Agent.OS'], 'Windows_NT')) displayName: Check out submodules (Windows) diff --git a/Cargo.lock b/Cargo.lock index a8e9850e643..7b2ee720104 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1547,11 +1547,10 @@ dependencies = [ [[package]] name = "lzma-sys" -version = "0.1.10" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cc 1.0.35 (registry+https://github.com/rust-lang/crates.io-index)", - "filetime 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.54 (registry+https://github.com/rust-lang/crates.io-index)", "pkg-config 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -4282,7 +4281,7 @@ name = "xz2" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "lzma-sys 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "lzma-sys 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -4437,7 +4436,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum log_settings 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "19af41f0565d7c19b2058153ad0b42d4d5ce89ec4dbf06ed6741114a8b63e7cd" "checksum lsp-codec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "169d737ad89cf8ddd82d1804d9122f54568c49377665157277cc90d747b1d31a" "checksum lsp-types 0.57.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b62b77309737b1e262b3bbf37ff8faa740562c633b14702afe9be85dbcb6f88a" -"checksum lzma-sys 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "d1eaa027402541975218bb0eec67d6b0412f6233af96e0d096d31dbdfd22e614" +"checksum lzma-sys 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)" = "16b5c59c57cc4d39e7999f50431aa312ea78af7c93b23fbb0c3567bd672e7f35" "checksum mac 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" "checksum macro-utils 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f2c4deaccc2ead6a28c16c0ba82f07d52b6475397415ce40876e559b0b0ea510" "checksum maplit 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "08cbb6b4fef96b6d77bfc40ec491b1690c779e77b05cd9f07f787ed376fd4c43" diff --git a/src/ci/docker/run.sh b/src/ci/docker/run.sh index dea41bee6e0..7d3f013ea01 100755 --- a/src/ci/docker/run.sh +++ b/src/ci/docker/run.sh @@ -125,26 +125,60 @@ fi # goes ahead and sets it for all builders. args="$args --privileged" -exec docker \ +# Things get a little weird if this script is already running in a docker +# container. If we're already in a docker container then we assume it's set up +# to do docker-in-docker where we have access to a working `docker` command. +# +# If this is the case (we check via the presence of `/.dockerenv`) +# then we can't actually use the `--volume` argument. Typically we use +# `--volume` to efficiently share the build and source directory between this +# script and the container we're about to spawn. If we're inside docker already +# though the `--volume` argument maps the *host's* folder to the container we're +# about to spawn, when in fact we want the folder in this container itself. To +# work around this we use a recipe cribbed from +# https://circleci.com/docs/2.0/building-docker-images/#mounting-folders to +# create a temporary container with a volume. We then copy the entire source +# directory into this container, and then use that copy in the container we're +# about to spawn. Finally after the build finishes we re-extract the object +# directory. +# +# Note that none of this is necessary if we're *not* in a docker-in-docker +# scenario. If this script is run on a bare metal host then we share a bunch of +# data directories to share as much data as possible. Note that we also use +# `LOCAL_USER_ID` (recognized in `src/ci/run.sh`) to ensure that files are all +# read/written as the same user as the bare-metal user. +if [ -f /.dockerenv ]; then + docker create -v /checkout --name checkout alpine:3.4 /bin/true + docker cp . checkout:/checkout + args="$args --volumes-from checkout" +else + args="$args --volume $root_dir:/checkout:ro" + args="$args --volume $objdir:/checkout/obj" + args="$args --volume $HOME/.cargo:/cargo" + args="$args --volume $HOME/rustsrc:$HOME/rustsrc" + args="$args --env LOCAL_USER_ID=`id -u`" +fi + +docker \ run \ - --volume "$root_dir:/checkout:ro" \ - --volume "$objdir:/checkout/obj" \ --workdir /checkout/obj \ --env SRC=/checkout \ $args \ --env CARGO_HOME=/cargo \ --env DEPLOY \ --env DEPLOY_ALT \ - --env LOCAL_USER_ID=`id -u` \ --env CI \ --env TF_BUILD \ --env BUILD_SOURCEBRANCHNAME \ --env TOOLSTATE_REPO_ACCESS_TOKEN \ --env TOOLSTATE_REPO \ --env CI_JOB_NAME="${CI_JOB_NAME-$IMAGE}" \ - --volume "$HOME/.cargo:/cargo" \ - --volume "$HOME/rustsrc:$HOME/rustsrc" \ --init \ --rm \ rust-ci \ /checkout/src/ci/run.sh + +if [ -f /.dockerenv ]; then + rm -rf $objdir + docker cp checkout:/checkout/obj $objdir +fi diff --git a/src/ci/run.sh b/src/ci/run.sh index b40bef77665..1039343827d 100755 --- a/src/ci/run.sh +++ b/src/ci/run.sh @@ -25,7 +25,7 @@ source "$ci_dir/shared.sh" branch_name=$(getCIBranch) -if [ ! isCI ] || [ "$branch_name" = "auto" ]; then +if [ ! isCI ] || [ "$branch_name" = "auto" ] || [ "$branch_name" = "try" ]; then RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set build.print-step-timings --enable-verbose-tests" fi