Move `auto` jobs to calculate-job-matrix.py

.github/workflows/ci.yml

@@ -179,416 +179,6 @@ jobs:
           AWS_ACCESS_KEY_ID: "${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}"
           AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}"
         if: "success() && !env.SKIP_JOB && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')"
-  auto:
-    name: "auto - ${{ matrix.name }}"
-    env:
-      CI_JOB_NAME: "${{ matrix.name }}"
-      CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
-      HEAD_SHA: "${{ github.event.pull_request.head.sha || github.sha }}"
-      DOCKER_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
-      SCCACHE_BUCKET: rust-lang-ci-sccache2
-      DEPLOY_BUCKET: rust-lang-ci2
-      TOOLSTATE_REPO: "https://github.com/rust-lang-nursery/rust-toolstate"
-      TOOLSTATE_ISSUES_API_URL: "https://api.github.com/repos/rust-lang/rust/issues"
-      TOOLSTATE_PUBLISH: 1
-      CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZI5DHEBFL
-      ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZN24CBO55
-      AWS_REGION: us-west-1
-      CACHE_DOMAIN: ci-caches.rust-lang.org
-    if: "github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'"
-    strategy:
-      matrix:
-        include:
-          - name: aarch64-gnu
-            os:
-              - self-hosted
-              - ARM64
-              - linux
-          - name: arm-android
-            os: ubuntu-20.04-8core-32gb
-            env: {}
-          - name: armhf-gnu
-            os: ubuntu-20.04-8core-32gb
-            env: {}
-          - name: dist-aarch64-linux
-            env:
-              CODEGEN_BACKENDS: "llvm,cranelift"
-            os: ubuntu-20.04-8core-32gb
-          - name: dist-android
-            os: ubuntu-20.04-8core-32gb
-            env: {}
-          - name: dist-arm-linux
-            os: ubuntu-20.04-16core-64gb
-            env: {}
-          - name: dist-armhf-linux
-            os: ubuntu-20.04-8core-32gb
-            env: {}
-          - name: dist-armv7-linux
-            os: ubuntu-20.04-8core-32gb
-            env: {}
-          - name: dist-i586-gnu-i586-i686-musl
-            os: ubuntu-20.04-8core-32gb
-            env: {}
-          - name: dist-i686-linux
-            os: ubuntu-20.04-8core-32gb
-            env: {}
-          - name: dist-loongarch64-linux
-            os: ubuntu-20.04-8core-32gb
-            env: {}
-          - name: dist-ohos
-            os: ubuntu-20.04-8core-32gb
-            env: {}
-          - name: dist-powerpc-linux
-            os: ubuntu-20.04-8core-32gb
-            env: {}
-          - name: dist-powerpc64-linux
-            os: ubuntu-20.04-8core-32gb
-            env: {}
-          - name: dist-powerpc64le-linux
-            os: ubuntu-20.04-8core-32gb
-            env: {}
-          - name: dist-riscv64-linux
-            os: ubuntu-20.04-8core-32gb
-            env: {}
-          - name: dist-s390x-linux
-            os: ubuntu-20.04-8core-32gb
-            env: {}
-          - name: dist-various-1
-            os: ubuntu-20.04-8core-32gb
-            env: {}
-          - name: dist-various-2
-            os: ubuntu-20.04-8core-32gb
-            env: {}
-          - name: dist-x86_64-freebsd
-            os: ubuntu-20.04-8core-32gb
-            env: {}
-          - name: dist-x86_64-illumos
-            os: ubuntu-20.04-8core-32gb
-            env: {}
-          - name: dist-x86_64-linux
-            env:
-              CODEGEN_BACKENDS: "llvm,cranelift"
-            os: ubuntu-20.04-16core-64gb
-          - name: dist-x86_64-linux-alt
-            env:
-              IMAGE: dist-x86_64-linux
-              CODEGEN_BACKENDS: "llvm,cranelift"
-            os: ubuntu-20.04-16core-64gb
-          - name: dist-x86_64-musl
-            env:
-              CODEGEN_BACKENDS: "llvm,cranelift"
-            os: ubuntu-20.04-8core-32gb
-          - name: dist-x86_64-netbsd
-            os: ubuntu-20.04-8core-32gb
-            env: {}
-          - name: i686-gnu
-            os: ubuntu-20.04-8core-32gb
-            env: {}
-          - name: i686-gnu-nopt
-            os: ubuntu-20.04-8core-32gb
-            env: {}
-          - name: mingw-check
-            os: ubuntu-20.04-4core-16gb
-            env: {}
-          - name: test-various
-            os: ubuntu-20.04-8core-32gb
-            env: {}
-          - name: x86_64-gnu
-            os: ubuntu-20.04-4core-16gb
-            env: {}
-          - name: x86_64-gnu-stable
-            env:
-              IMAGE: x86_64-gnu
-              RUST_CI_OVERRIDE_RELEASE_CHANNEL: stable
-              CI_ONLY_WHEN_CHANNEL: nightly
-            os: ubuntu-20.04-4core-16gb
-          - name: x86_64-gnu-aux
-            os: ubuntu-20.04-4core-16gb
-            env: {}
-          - name: x86_64-gnu-integration
-            env:
-              CI_ONLY_WHEN_CHANNEL: nightly
-            os: ubuntu-20.04-8core-32gb
-          - name: x86_64-gnu-debug
-            os: ubuntu-20.04-8core-32gb
-            env: {}
-          - name: x86_64-gnu-distcheck
-            os: ubuntu-20.04-8core-32gb
-            env: {}
-          - name: x86_64-gnu-llvm-18
-            env:
-              RUST_BACKTRACE: 1
-            os: ubuntu-20.04-8core-32gb
-          - name: x86_64-gnu-llvm-17
-            env:
-              RUST_BACKTRACE: 1
-            os: ubuntu-20.04-8core-32gb
-          - name: x86_64-gnu-nopt
-            os: ubuntu-20.04-4core-16gb
-            env: {}
-          - name: x86_64-gnu-tools
-            env:
-              DEPLOY_TOOLSTATES_JSON: toolstates-linux.json
-            os: ubuntu-20.04-8core-32gb
-          - name: dist-x86_64-apple
-            env:
-              SCRIPT: "./x.py dist bootstrap --include-default-paths --host=x86_64-apple-darwin --target=x86_64-apple-darwin"
-              RUST_CONFIGURE_ARGS: "--enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc --set rust.lto=thin --set rust.codegen-units=1"
-              RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
-              MACOSX_DEPLOYMENT_TARGET: 10.12
-              SELECT_XCODE: /Applications/Xcode_14.3.1.app
-              NO_LLVM_ASSERTIONS: 1
-              NO_DEBUG_ASSERTIONS: 1
-              NO_OVERFLOW_CHECKS: 1
-              DIST_REQUIRE_ALL_TOOLS: 1
-              CODEGEN_BACKENDS: "llvm,cranelift"
-            os: macos-13
-          - name: dist-apple-various
-            env:
-              SCRIPT: "./x.py dist bootstrap --include-default-paths --host='' --target=aarch64-apple-ios,x86_64-apple-ios,aarch64-apple-ios-sim"
-              RUST_CONFIGURE_ARGS: "--enable-sanitizers --enable-profiler --set rust.jemalloc"
-              RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
-              MACOSX_DEPLOYMENT_TARGET: 10.12
-              SELECT_XCODE: /Applications/Xcode_14.3.1.app
-              NO_LLVM_ASSERTIONS: 1
-              NO_DEBUG_ASSERTIONS: 1
-              NO_OVERFLOW_CHECKS: 1
-            os: macos-13
-          - name: x86_64-apple-1
-            env:
-              SCRIPT: "./x.py --stage 2 test --skip tests/ui --skip tests/rustdoc --skip tests/run-make-fulldeps"
-              RUST_CONFIGURE_ARGS: "--build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc"
-              RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
-              MACOSX_DEPLOYMENT_TARGET: 10.12
-              MACOSX_STD_DEPLOYMENT_TARGET: 10.12
-              SELECT_XCODE: /Applications/Xcode_14.3.1.app
-              NO_LLVM_ASSERTIONS: 1
-              NO_DEBUG_ASSERTIONS: 1
-              NO_OVERFLOW_CHECKS: 1
-            os: macos-13
-          - name: x86_64-apple-2
-            env:
-              SCRIPT: "./x.py --stage 2 test tests/ui tests/rustdoc tests/run-make-fulldeps"
-              RUST_CONFIGURE_ARGS: "--build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc"
-              RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
-              MACOSX_DEPLOYMENT_TARGET: 10.12
-              MACOSX_STD_DEPLOYMENT_TARGET: 10.12
-              SELECT_XCODE: /Applications/Xcode_14.3.1.app
-              NO_LLVM_ASSERTIONS: 1
-              NO_DEBUG_ASSERTIONS: 1
-              NO_OVERFLOW_CHECKS: 1
-            os: macos-13
-          - name: dist-aarch64-apple
-            env:
-              SCRIPT: "./x.py dist bootstrap --include-default-paths --host=aarch64-apple-darwin --target=aarch64-apple-darwin"
-              RUST_CONFIGURE_ARGS: "--enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc --set llvm.ninja=false --set rust.lto=thin"
-              RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
-              SELECT_XCODE: /Applications/Xcode_14.3.1.app
-              USE_XCODE_CLANG: 1
-              MACOSX_DEPLOYMENT_TARGET: 11.0
-              MACOSX_STD_DEPLOYMENT_TARGET: 11.0
-              NO_LLVM_ASSERTIONS: 1
-              NO_DEBUG_ASSERTIONS: 1
-              NO_OVERFLOW_CHECKS: 1
-              DIST_REQUIRE_ALL_TOOLS: 1
-            os: macos-14
-          - name: aarch64-apple
-            env:
-              SCRIPT: "./x.py --stage 2 test --host=aarch64-apple-darwin --target=aarch64-apple-darwin"
-              RUST_CONFIGURE_ARGS: "--enable-sanitizers --enable-profiler --set rust.jemalloc"
-              RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
-              SELECT_XCODE: /Applications/Xcode_14.3.1.app
-              USE_XCODE_CLANG: 1
-              MACOSX_DEPLOYMENT_TARGET: 11.0
-              MACOSX_STD_DEPLOYMENT_TARGET: 11.0
-              NO_LLVM_ASSERTIONS: 1
-              NO_DEBUG_ASSERTIONS: 1
-              NO_OVERFLOW_CHECKS: 1
-            os: macos-14
-          - name: x86_64-msvc
-            env:
-              RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --enable-profiler"
-              SCRIPT: make ci-msvc
-            os: windows-2019-8core-32gb
-          - name: i686-msvc
-            env:
-              RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-msvc"
-              SCRIPT: make ci-msvc
-            os: windows-2019-8core-32gb
-          - name: x86_64-msvc-ext
-            env:
-              SCRIPT: python x.py --stage 2 test src/tools/cargotest src/tools/cargo && src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstate/toolstates.json windows
-              HOST_TARGET: x86_64-pc-windows-msvc
-              RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --enable-lld --save-toolstates=/tmp/toolstate/toolstates.json"
-              DEPLOY_TOOLSTATES_JSON: toolstates-windows.json
-            os: windows-2019-8core-32gb
-          - name: i686-mingw
-            env:
-              RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-gnu"
-              SCRIPT: make ci-mingw
-              NO_DOWNLOAD_CI_LLVM: 1
-              CUSTOM_MINGW: 1
-            os: windows-2019-8core-32gb
-          - name: x86_64-mingw
-            env:
-              SCRIPT: make ci-mingw
-              RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-gnu --enable-profiler"
-              NO_DOWNLOAD_CI_LLVM: 1
-              CUSTOM_MINGW: 1
-            os: windows-2019-8core-32gb
-          - name: dist-x86_64-msvc
-            env:
-              RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --host=x86_64-pc-windows-msvc --target=x86_64-pc-windows-msvc --enable-full-tools --enable-profiler --set rust.codegen-units=1"
-              SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist windows-ci -- python x.py dist bootstrap --include-default-paths
-              DIST_REQUIRE_ALL_TOOLS: 1
-            os: windows-2019-8core-32gb
-          - name: dist-i686-msvc
-            env:
-              RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-msvc --host=i686-pc-windows-msvc --target=i686-pc-windows-msvc,i586-pc-windows-msvc --enable-full-tools --enable-profiler"
-              SCRIPT: python x.py dist bootstrap --include-default-paths
-              DIST_REQUIRE_ALL_TOOLS: 1
-            os: windows-2019-8core-32gb
-          - name: dist-aarch64-msvc
-            env:
-              RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --host=aarch64-pc-windows-msvc --enable-full-tools --enable-profiler"
-              SCRIPT: python x.py dist bootstrap --include-default-paths
-              DIST_REQUIRE_ALL_TOOLS: 1
-            os: windows-2019-8core-32gb
-          - name: dist-i686-mingw
-            env:
-              RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-gnu --enable-full-tools --enable-profiler"
-              NO_DOWNLOAD_CI_LLVM: 1
-              SCRIPT: python x.py dist bootstrap --include-default-paths
-              CUSTOM_MINGW: 1
-              DIST_REQUIRE_ALL_TOOLS: 1
-            os: windows-2019-8core-32gb
-          - name: dist-x86_64-mingw
-            env:
-              SCRIPT: python x.py dist bootstrap --include-default-paths
-              RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-gnu --enable-full-tools --enable-profiler"
-              NO_DOWNLOAD_CI_LLVM: 1
-              CUSTOM_MINGW: 1
-              DIST_REQUIRE_ALL_TOOLS: 1
-            os: windows-2019-8core-32gb
-          - name: dist-x86_64-msvc-alt
-            env:
-              RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --enable-extended --enable-profiler"
-              SCRIPT: python x.py dist bootstrap --include-default-paths
-            os: windows-2019-8core-32gb
-    defaults:
-      run:
-        shell: "${{ contains(matrix.os, 'windows') && 'msys2 {0}' || 'bash' }}"
-    timeout-minutes: 600
-    runs-on: "${{ matrix.os }}"
-    steps:
-      - if: "contains(matrix.os, 'windows')"
-        uses: msys2/setup-msys2@v2.22.0
-        with:
-          msystem: "${{ contains(matrix.name, 'i686') && 'mingw32' || 'mingw64' }}"
-          update: false
-          release: true
-          path-type: inherit
-          install: "make dos2unix diffutils\n"
-      - name: disable git crlf conversion
-        run: git config --global core.autocrlf false
-      - name: checkout the source code
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 2
-      - name: configure the PR in which the error message will be posted
-        run: "echo \"[CI_PR_NUMBER=$num]\""
-        env:
-          num: "${{ github.event.number }}"
-        if: "success() && !env.SKIP_JOB && github.event_name == 'pull_request'"
-      - name: add extra environment variables
-        run: src/ci/scripts/setup-environment.sh
-        env:
-          EXTRA_VARIABLES: "${{ toJson(matrix.env) }}"
-        if: success() && !env.SKIP_JOB
-      - name: decide whether to skip this job
-        run: src/ci/scripts/should-skip-this.sh
-        if: success() && !env.SKIP_JOB
-      - name: ensure the channel matches the target branch
-        run: src/ci/scripts/verify-channel.sh
-        if: success() && !env.SKIP_JOB
-      - name: collect CPU statistics
-        run: src/ci/scripts/collect-cpu-stats.sh
-        if: success() && !env.SKIP_JOB
-      - name: show the current environment
-        run: src/ci/scripts/dump-environment.sh
-        if: success() && !env.SKIP_JOB
-      - name: install awscli
-        run: src/ci/scripts/install-awscli.sh
-        if: success() && !env.SKIP_JOB
-      - name: install sccache
-        run: src/ci/scripts/install-sccache.sh
-        if: success() && !env.SKIP_JOB
-      - name: select Xcode
-        run: src/ci/scripts/select-xcode.sh
-        if: success() && !env.SKIP_JOB
-      - name: install clang
-        run: src/ci/scripts/install-clang.sh
-        if: success() && !env.SKIP_JOB
-      - name: install tidy
-        run: src/ci/scripts/install-tidy.sh
-        if: success() && !env.SKIP_JOB
-      - name: install WIX
-        run: src/ci/scripts/install-wix.sh
-        if: success() && !env.SKIP_JOB
-      - name: disable git crlf conversion
-        run: src/ci/scripts/disable-git-crlf-conversion.sh
-        if: success() && !env.SKIP_JOB
-      - name: checkout submodules
-        run: src/ci/scripts/checkout-submodules.sh
-        if: success() && !env.SKIP_JOB
-      - name: install MSYS2
-        run: src/ci/scripts/install-msys2.sh
-        if: success() && !env.SKIP_JOB
-      - name: install MinGW
-        run: src/ci/scripts/install-mingw.sh
-        if: success() && !env.SKIP_JOB
-      - name: install ninja
-        run: src/ci/scripts/install-ninja.sh
-        if: success() && !env.SKIP_JOB
-      - name: enable ipv6 on Docker
-        run: src/ci/scripts/enable-docker-ipv6.sh
-        if: success() && !env.SKIP_JOB
-      - name: disable git crlf conversion
-        run: src/ci/scripts/disable-git-crlf-conversion.sh
-        if: success() && !env.SKIP_JOB
-      - name: ensure line endings are correct
-        run: src/ci/scripts/verify-line-endings.sh
-        if: success() && !env.SKIP_JOB
-      - name: ensure backported commits are in upstream branches
-        run: src/ci/scripts/verify-backported-commits.sh
-        if: success() && !env.SKIP_JOB
-      - name: ensure the stable version number is correct
-        run: src/ci/scripts/verify-stable-version-number.sh
-        if: success() && !env.SKIP_JOB
-      - name: run the build
-        run: src/ci/scripts/run-build-from-ci.sh 2>&1
-        env:
-          AWS_ACCESS_KEY_ID: "${{ env.CACHES_AWS_ACCESS_KEY_ID }}"
-          AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}"
-          TOOLSTATE_REPO_ACCESS_TOKEN: "${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}"
-        if: success() && !env.SKIP_JOB
-      - name: create github artifacts
-        run: src/ci/scripts/create-doc-artifacts.sh
-        if: success() && !env.SKIP_JOB
-      - name: upload artifacts to github
-        uses: actions/upload-artifact@v4
-        with:
-          name: "${{ env.DOC_ARTIFACT_NAME }}"
-          path: obj/artifacts/doc
-          if-no-files-found: ignore
-          retention-days: 5
-        if: success() && !env.SKIP_JOB
-      - name: upload artifacts to S3
-        run: src/ci/scripts/upload-artifacts.sh
-        env:
-          AWS_ACCESS_KEY_ID: "${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}"
-          AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}"
-        if: "success() && !env.SKIP_JOB && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')"
   master:
     name: master
     runs-on: ubuntu-latest
@@ -636,7 +226,7 @@ jobs:
     runs-on: ubuntu-latest
   auto-success:
     needs:
-      - auto
+      - job
     if: "success() && github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'"
     steps:
       - name: mark the job as a success
@@ -646,7 +236,7 @@ jobs:
     runs-on: ubuntu-latest
   auto-failure:
     needs:
-      - auto
+      - job
     if: "!success() && github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'"
     steps:
       - name: mark the job as a failure

src/ci/github-actions/calculate-job-matrix.py

@@ -6,8 +6,6 @@ be executed on CI.
 
 It reads job definitions from `src/ci/github-actions/jobs.yml`
 and filters them based on the event that happened on CI.
-
-Currently, it only supports PR and try builds.
 """
 import enum
 import json
@@ -22,14 +20,30 @@ JOBS_YAML_PATH = Path(__file__).absolute().parent / "jobs.yml"
 
 
 def name_jobs(jobs: List[Dict], prefix: str) -> List[Dict]:
+    """
+    Add a `name` attribute to each job, based on its image and the given `prefix`.
+    """
     for job in jobs:
         job["name"] = f"{prefix} - {job['image']}"
     return jobs
 
 
+def add_base_env(jobs: List[Dict], environment: Dict[str, str]) -> List[Dict]:
+    """
+    Prepends `environment` to the `env` attribute of each job.
+    The `env` of each job has higher precedence than `environment`.
+    """
+    for job in jobs:
+        env = environment.copy()
+        env.update(job.get("env", {}))
+        job["env"] = env
+    return jobs
+
+
 class JobType(enum.Enum):
     PR = enum.auto()
     Try = enum.auto()
+    Auto = enum.auto()
 
 
 def find_job_type(github_ctx: Dict[str, Any]) -> Optional[JobType]:
@@ -53,14 +67,19 @@ def find_job_type(github_ctx: Dict[str, Any]) -> Optional[JobType]:
         if try_build:
             return JobType.Try
 
+        if ref == "refs/heads/auto" and repository == "rust-lang-ci/rust":
+            return JobType.Auto
+
     return None
 
 
 def calculate_jobs(job_type: JobType, job_data: Dict[str, Any]) -> List[Dict[str, Any]]:
     if job_type == JobType.PR:
-        return name_jobs(job_data["pr"], "PR")
+        return add_base_env(name_jobs(job_data["pr"], "PR"), job_data["envs"]["pr"])
     elif job_type == JobType.Try:
-        return name_jobs(job_data["try"], "try")
+        return add_base_env(name_jobs(job_data["try"], "try"), job_data["envs"]["production"])
+    elif job_type == JobType.Auto:
+        return add_base_env(name_jobs(job_data["auto"], "auto"), job_data["envs"]["production"])
 
     return []
 
@@ -68,11 +87,11 @@ def calculate_jobs(job_type: JobType, job_data: Dict[str, Any]) -> List[Dict[str
 if __name__ == "__main__":
     logging.basicConfig(level=logging.INFO)
 
-    github_ctx = json.loads(os.environ["GITHUB_CTX"])
-
     with open(JOBS_YAML_PATH) as f:
         data = yaml.safe_load(f)
 
+    github_ctx = json.loads(os.environ["GITHUB_CTX"])
+
     job_type = find_job_type(github_ctx)
     logging.info(f"Job type: {job_type}")
 

src/ci/github-actions/ci.yml

@@ -342,6 +342,8 @@ concurrency:
 
 jobs:
   # The job matrix for `calculate_matrix` is defined in src/ci/github-actions/jobs.yml.
+  # It calculates which jobs should be executed, based on the data of the ${{ github }} context.
+  # If you want to modify CI jobs, take a look at src/ci/github-actions/jobs.yml.
   calculate_matrix:
     name: Calculate job matrix
     runs-on: ubuntu-latest
@@ -355,7 +357,6 @@ jobs:
           GITHUB_CTX: ${{ toJSON(github) }}
         run: python3 src/ci/github-actions/calculate-job-matrix.py >> $GITHUB_OUTPUT
         id: jobs
-  # This is either a PR or a try job, decided by calculate-job-matrix.py
   job:
     <<: *base-ci-job
     name: ${{ matrix.name }}
@@ -375,390 +376,6 @@ jobs:
         # Check the `calculate_matrix` job to see how is the matrix defined.
         include: ${{ fromJSON(needs.calculate_matrix.outputs.jobs) }}
 
-  auto:
-    <<: *base-ci-job
-    name: auto - ${{ matrix.name }}
-    env:
-      <<: [*shared-ci-variables, *prod-variables]
-    if: github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'
-    strategy:
-      matrix:
-        include:
-          #############################
-          #   Linux/Docker builders   #
-          #############################
-
-          - name: aarch64-gnu
-            <<: *job-aarch64-linux
-
-          - name: arm-android
-            <<: *job-linux-8c
-
-          - name: armhf-gnu
-            <<: *job-linux-8c
-
-          - name: dist-aarch64-linux
-            env:
-              CODEGEN_BACKENDS: llvm,cranelift
-            <<: *job-linux-8c
-
-          - name: dist-android
-            <<: *job-linux-8c
-
-          - name: dist-arm-linux
-            <<: *job-linux-16c
-
-          - name: dist-armhf-linux
-            <<: *job-linux-8c
-
-          - name: dist-armv7-linux
-            <<: *job-linux-8c
-
-          - name: dist-i586-gnu-i586-i686-musl
-            <<: *job-linux-8c
-
-          - name: dist-i686-linux
-            <<: *job-linux-8c
-
-          - name: dist-loongarch64-linux
-            <<: *job-linux-8c
-
-          - name: dist-ohos
-            <<: *job-linux-8c
-
-          - name: dist-powerpc-linux
-            <<: *job-linux-8c
-
-          - name: dist-powerpc64-linux
-            <<: *job-linux-8c
-
-          - name: dist-powerpc64le-linux
-            <<: *job-linux-8c
-
-          - name: dist-riscv64-linux
-            <<: *job-linux-8c
-
-          - name: dist-s390x-linux
-            <<: *job-linux-8c
-
-          - name: dist-various-1
-            <<: *job-linux-8c
-
-          - name: dist-various-2
-            <<: *job-linux-8c
-
-          - name: dist-x86_64-freebsd
-            <<: *job-linux-8c
-
-          - name: dist-x86_64-illumos
-            <<: *job-linux-8c
-
-          - &dist-x86_64-linux
-            name: dist-x86_64-linux
-            env:
-              CODEGEN_BACKENDS: llvm,cranelift
-            <<: *job-linux-16c
-
-          - name: dist-x86_64-linux-alt
-            env:
-              IMAGE: dist-x86_64-linux
-              CODEGEN_BACKENDS: llvm,cranelift
-            <<: *job-linux-16c
-
-          - name: dist-x86_64-musl
-            env:
-              CODEGEN_BACKENDS: llvm,cranelift
-            <<: *job-linux-8c
-
-          - name: dist-x86_64-netbsd
-            <<: *job-linux-8c
-
-          - name: i686-gnu
-            <<: *job-linux-8c
-
-          - name: i686-gnu-nopt
-            <<: *job-linux-8c
-
-          - name: mingw-check
-            <<: *job-linux-4c
-
-          - name: test-various
-            <<: *job-linux-8c
-
-          - name: x86_64-gnu
-            <<: *job-linux-4c
-
-          # This job ensures commits landing on nightly still pass the full
-          # test suite on the stable channel. There are some UI tests that
-          # depend on the channel being built (for example if they include the
-          # channel name on the output), and this builder prevents landing
-          # changes that would result in broken builds after a promotion.
-          - name: x86_64-gnu-stable
-            env:
-              IMAGE: x86_64-gnu
-              RUST_CI_OVERRIDE_RELEASE_CHANNEL: stable
-              # Only run this job on the nightly channel. Running this on beta
-              # could cause failures when `dev: 1` in `stage0.txt`, and running
-              # this on stable is useless.
-              CI_ONLY_WHEN_CHANNEL: nightly
-            <<: *job-linux-4c
-
-          - name: x86_64-gnu-aux
-            <<: *job-linux-4c
-
-          - name: x86_64-gnu-integration
-            env:
-              # Only run this job on the nightly channel. Fuchsia requires
-              # nightly features to compile, and this job would fail if
-              # executed on beta and stable.
-              CI_ONLY_WHEN_CHANNEL: nightly
-            <<: *job-linux-8c
-
-          - name: x86_64-gnu-debug
-            <<: *job-linux-8c
-
-          - name: x86_64-gnu-distcheck
-            <<: *job-linux-8c
-
-          - name: x86_64-gnu-llvm-18
-            env:
-              RUST_BACKTRACE: 1
-            <<: *job-linux-8c
-
-          - name: x86_64-gnu-llvm-17
-            env:
-              RUST_BACKTRACE: 1
-            <<: *job-linux-8c
-
-          - name: x86_64-gnu-nopt
-            <<: *job-linux-4c
-
-          - name: x86_64-gnu-tools
-            env:
-              DEPLOY_TOOLSTATES_JSON: toolstates-linux.json
-            <<: *job-linux-8c
-
-          ####################
-          #  macOS Builders  #
-          ####################
-
-          - name: dist-x86_64-apple
-            env:
-              SCRIPT: ./x.py dist bootstrap --include-default-paths --host=x86_64-apple-darwin --target=x86_64-apple-darwin
-              RUST_CONFIGURE_ARGS: --enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc --set rust.lto=thin --set rust.codegen-units=1
-              RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
-              MACOSX_DEPLOYMENT_TARGET: 10.12
-              SELECT_XCODE: /Applications/Xcode_14.3.1.app
-              NO_LLVM_ASSERTIONS: 1
-              NO_DEBUG_ASSERTIONS: 1
-              NO_OVERFLOW_CHECKS: 1
-              DIST_REQUIRE_ALL_TOOLS: 1
-              CODEGEN_BACKENDS: llvm,cranelift
-            <<: *job-macos-xl
-
-          - name: dist-apple-various
-            env:
-              SCRIPT: ./x.py dist bootstrap --include-default-paths --host='' --target=aarch64-apple-ios,x86_64-apple-ios,aarch64-apple-ios-sim
-              RUST_CONFIGURE_ARGS: --enable-sanitizers --enable-profiler --set rust.jemalloc
-              RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
-              MACOSX_DEPLOYMENT_TARGET: 10.12
-              SELECT_XCODE: /Applications/Xcode_14.3.1.app
-              NO_LLVM_ASSERTIONS: 1
-              NO_DEBUG_ASSERTIONS: 1
-              NO_OVERFLOW_CHECKS: 1
-            <<: *job-macos-xl
-
-          - name: x86_64-apple-1
-            env: &env-x86_64-apple-tests
-              SCRIPT: ./x.py --stage 2 test --skip tests/ui --skip tests/rustdoc --skip tests/run-make-fulldeps
-              RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc
-              RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
-              MACOSX_DEPLOYMENT_TARGET: 10.12
-              MACOSX_STD_DEPLOYMENT_TARGET: 10.12
-              SELECT_XCODE: /Applications/Xcode_14.3.1.app
-              NO_LLVM_ASSERTIONS: 1
-              NO_DEBUG_ASSERTIONS: 1
-              NO_OVERFLOW_CHECKS: 1
-            <<: *job-macos-xl
-
-          - name: x86_64-apple-2
-            env:
-              SCRIPT: ./x.py --stage 2 test tests/ui tests/rustdoc tests/run-make-fulldeps
-              <<: *env-x86_64-apple-tests
-            <<: *job-macos-xl
-
-          # This target only needs to support 11.0 and up as nothing else supports the hardware
-          - name: dist-aarch64-apple
-            env:
-              SCRIPT: ./x.py dist bootstrap --include-default-paths --host=aarch64-apple-darwin --target=aarch64-apple-darwin
-              RUST_CONFIGURE_ARGS: >-
-                --enable-full-tools
-                --enable-sanitizers
-                --enable-profiler
-                --set rust.jemalloc
-                --set llvm.ninja=false
-                --set rust.lto=thin
-              RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
-              SELECT_XCODE: /Applications/Xcode_14.3.1.app
-              USE_XCODE_CLANG: 1
-              MACOSX_DEPLOYMENT_TARGET: 11.0
-              MACOSX_STD_DEPLOYMENT_TARGET: 11.0
-              NO_LLVM_ASSERTIONS: 1
-              NO_DEBUG_ASSERTIONS: 1
-              NO_OVERFLOW_CHECKS: 1
-              DIST_REQUIRE_ALL_TOOLS: 1
-            <<: *job-macos-m1
-
-          # This target only needs to support 11.0 and up as nothing else supports the hardware
-          - name: aarch64-apple
-            env:
-              SCRIPT: ./x.py --stage 2 test --host=aarch64-apple-darwin --target=aarch64-apple-darwin
-              RUST_CONFIGURE_ARGS: >-
-                --enable-sanitizers
-                --enable-profiler
-                --set rust.jemalloc
-              RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
-              SELECT_XCODE: /Applications/Xcode_14.3.1.app
-              USE_XCODE_CLANG: 1
-              MACOSX_DEPLOYMENT_TARGET: 11.0
-              MACOSX_STD_DEPLOYMENT_TARGET: 11.0
-              NO_LLVM_ASSERTIONS: 1
-              NO_DEBUG_ASSERTIONS: 1
-              NO_OVERFLOW_CHECKS: 1
-            <<: *job-macos-m1
-
-          ######################
-          #  Windows Builders  #
-          ######################
-
-          - name: x86_64-msvc
-            env:
-              RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler
-              SCRIPT: make ci-msvc
-            <<: *job-windows-8c
-
-          - name: i686-msvc
-            env:
-              RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc
-              SCRIPT: make ci-msvc
-            <<: *job-windows-8c
-
-          - name: x86_64-msvc-ext
-            env:
-              SCRIPT: python x.py --stage 2 test src/tools/cargotest src/tools/cargo && src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstate/toolstates.json windows
-              HOST_TARGET: x86_64-pc-windows-msvc
-              RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-lld --save-toolstates=/tmp/toolstate/toolstates.json
-              DEPLOY_TOOLSTATES_JSON: toolstates-windows.json
-            <<: *job-windows-8c
-
-          # 32/64-bit MinGW builds.
-          #
-          # We are using MinGW with POSIX threads since LLVM requires
-          # C++'s std::thread which is disabled in libstdc++ with win32 threads.
-          # FIXME: Libc++ doesn't have this limitation so we can avoid
-          # winpthreads if we switch to it.
-          #
-          # Instead of relying on the MinGW version installed on CI we download
-          # and install one ourselves so we won't be surprised by changes to CI's
-          # build image.
-          #
-          # Finally, note that the downloads below are all in the `rust-lang-ci` S3
-          # bucket, but they clearly didn't originate there! The downloads originally
-          # came from the mingw-w64 SourceForge download site. Unfortunately
-          # SourceForge is notoriously flaky, so we mirror it on our own infrastructure.
-
-          - name: i686-mingw
-            env:
-              RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu
-              SCRIPT: make ci-mingw
-              # We are intentionally allowing an old toolchain on this builder (and that's
-              # incompatible with LLVM downloads today).
-              NO_DOWNLOAD_CI_LLVM: 1
-              CUSTOM_MINGW: 1
-            <<: *job-windows-8c
-
-          - name: x86_64-mingw
-            env:
-              SCRIPT: make ci-mingw
-              RUST_CONFIGURE_ARGS: >-
-                --build=x86_64-pc-windows-gnu
-                --enable-profiler
-              # We are intentionally allowing an old toolchain on this builder (and that's
-              # incompatible with LLVM downloads today).
-              NO_DOWNLOAD_CI_LLVM: 1
-              CUSTOM_MINGW: 1
-            <<: *job-windows-8c
-
-          - name: dist-x86_64-msvc
-            env:
-              RUST_CONFIGURE_ARGS: >-
-                --build=x86_64-pc-windows-msvc
-                --host=x86_64-pc-windows-msvc
-                --target=x86_64-pc-windows-msvc
-                --enable-full-tools
-                --enable-profiler
-                --set rust.codegen-units=1
-              SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist windows-ci -- python x.py dist bootstrap --include-default-paths
-              DIST_REQUIRE_ALL_TOOLS: 1
-            <<: *job-windows-8c
-
-          - name: dist-i686-msvc
-            env:
-              RUST_CONFIGURE_ARGS: >-
-                --build=i686-pc-windows-msvc
-                --host=i686-pc-windows-msvc
-                --target=i686-pc-windows-msvc,i586-pc-windows-msvc
-                --enable-full-tools
-                --enable-profiler
-              SCRIPT: python x.py dist bootstrap --include-default-paths
-              DIST_REQUIRE_ALL_TOOLS: 1
-            <<: *job-windows-8c
-
-          - name: dist-aarch64-msvc
-            env:
-              RUST_CONFIGURE_ARGS: >-
-                --build=x86_64-pc-windows-msvc
-                --host=aarch64-pc-windows-msvc
-                --enable-full-tools
-                --enable-profiler
-              SCRIPT: python x.py dist bootstrap --include-default-paths
-              DIST_REQUIRE_ALL_TOOLS: 1
-            <<: *job-windows-8c
-
-          - name: dist-i686-mingw
-            env:
-              RUST_CONFIGURE_ARGS: >-
-                --build=i686-pc-windows-gnu
-                --enable-full-tools
-                --enable-profiler
-              # We are intentionally allowing an old toolchain on this builder (and that's
-              # incompatible with LLVM downloads today).
-              NO_DOWNLOAD_CI_LLVM: 1
-              SCRIPT: python x.py dist bootstrap --include-default-paths
-              CUSTOM_MINGW: 1
-              DIST_REQUIRE_ALL_TOOLS: 1
-            <<: *job-windows-8c
-
-          - name: dist-x86_64-mingw
-            env:
-              SCRIPT: python x.py dist bootstrap --include-default-paths
-              RUST_CONFIGURE_ARGS: >-
-                --build=x86_64-pc-windows-gnu
-                --enable-full-tools
-                --enable-profiler
-              # We are intentionally allowing an old toolchain on this builder (and that's
-              # incompatible with LLVM downloads today).
-              NO_DOWNLOAD_CI_LLVM: 1
-              CUSTOM_MINGW: 1
-              DIST_REQUIRE_ALL_TOOLS: 1
-            <<: *job-windows-8c
-
-          - name: dist-x86_64-msvc-alt
-            env:
-              RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler
-              SCRIPT: python x.py dist bootstrap --include-default-paths
-            <<: *job-windows-8c
-
   master:
     name: master
     runs-on: ubuntu-latest
@@ -790,10 +407,10 @@ jobs:
     if: "!success() && github.event_name == 'push' && (github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust'"
     <<: *base-failure-job
   auto-success:
-    needs: [auto]
+    needs: [ job ]
     if: "success() && github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'"
     <<: *base-success-job
   auto-failure:
-    needs: [auto]
+    needs: [ job ]
     if: "!success() && github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'"
     <<: *base-failure-job

src/ci/github-actions/jobs.yml

@@ -38,7 +38,7 @@ runners:
     os: [ self-hosted, ARM64, linux ]
 
 envs:
-  - &prod-variables
+  production:
     DEPLOY_BUCKET: rust-lang-ci2
     TOOLSTATE_ISSUES_API_URL: https://api.github.com/repos/rust-lang/rust/issues
     TOOLSTATE_PUBLISH: 1
@@ -51,35 +51,409 @@ envs:
     ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZN24CBO55
     AWS_REGION: us-west-1
 
-  - &pr-env
+  pr:
     PR_CI_JOB: 1
 
-# Pull request jobs
+# Jobs that run on each push to a pull request (PR)
+# These jobs automatically inherit envs.pr, to avoid repeating
+# it in each job definition.
 pr:
   - image: mingw-check
-    env:
-      <<: *pr-env
     <<: *job-linux-4c
   - image: mingw-check-tidy
     continue_on_error: true
-    env:
-      <<: *pr-env
     <<: *job-linux-4c
   - image: x86_64-gnu-llvm-17
     env:
       ENABLE_GCC_CODEGEN: "1"
-      <<: *pr-env
     <<: *job-linux-16c
   - image: x86_64-gnu-tools
-    env:
-      <<: *pr-env
     <<: *job-linux-16c
 
-# Try build jobs
+# Jobs that run when you perform a try build (@bors try)
+# These jobs automatically inherit envs.production, to avoid repeating
+# it in each job definition.
 try:
   - image: dist-x86_64-linux
     env:
       DIST_TRY_BUILD: 1
       CODEGEN_BACKENDS: llvm,cranelift
-      <<: *prod-variables
     <<: *job-linux-16c
+
+# Main CI jobs that have to be green to merge a commit into master
+# These jobs automatically inherit envs.production, to avoid repeating
+# it in each job definition.
+auto:
+  #############################
+  #   Linux/Docker builders   #
+  #############################
+
+  - name: aarch64-gnu
+    <<: *job-aarch64-linux
+
+  - name: arm-android
+    <<: *job-linux-8c
+
+  - name: armhf-gnu
+    <<: *job-linux-8c
+
+  - name: dist-aarch64-linux
+    env:
+      CODEGEN_BACKENDS: llvm,cranelift
+    <<: *job-linux-8c
+
+  - name: dist-android
+    <<: *job-linux-8c
+
+  - name: dist-arm-linux
+    <<: *job-linux-16c
+
+  - name: dist-armhf-linux
+    <<: *job-linux-8c
+
+  - name: dist-armv7-linux
+    <<: *job-linux-8c
+
+  - name: dist-i586-gnu-i586-i686-musl
+    <<: *job-linux-8c
+
+  - name: dist-i686-linux
+    <<: *job-linux-8c
+
+  - name: dist-loongarch64-linux
+    <<: *job-linux-8c
+
+  - name: dist-ohos
+    <<: *job-linux-8c
+
+  - name: dist-powerpc-linux
+    <<: *job-linux-8c
+
+  - name: dist-powerpc64-linux
+    <<: *job-linux-8c
+
+  - name: dist-powerpc64le-linux
+    <<: *job-linux-8c
+
+  - name: dist-riscv64-linux
+    <<: *job-linux-8c
+
+  - name: dist-s390x-linux
+    <<: *job-linux-8c
+
+  - name: dist-various-1
+    <<: *job-linux-8c
+
+  - name: dist-various-2
+    <<: *job-linux-8c
+
+  - name: dist-x86_64-freebsd
+    <<: *job-linux-8c
+
+  - name: dist-x86_64-illumos
+    <<: *job-linux-8c
+
+  - name: dist-x86_64-linux
+    env:
+      CODEGEN_BACKENDS: llvm,cranelift
+    <<: *job-linux-16c
+
+  - name: dist-x86_64-linux-alt
+    env:
+      IMAGE: dist-x86_64-linux
+      CODEGEN_BACKENDS: llvm,cranelift
+    <<: *job-linux-16c
+
+  - name: dist-x86_64-musl
+    env:
+      CODEGEN_BACKENDS: llvm,cranelift
+    <<: *job-linux-8c
+
+  - name: dist-x86_64-netbsd
+    <<: *job-linux-8c
+
+  - name: i686-gnu
+    <<: *job-linux-8c
+
+  - name: i686-gnu-nopt
+    <<: *job-linux-8c
+
+  - name: mingw-check
+    <<: *job-linux-4c
+
+  - name: test-various
+    <<: *job-linux-8c
+
+  - name: x86_64-gnu
+    <<: *job-linux-4c
+
+  # This job ensures commits landing on nightly still pass the full
+  # test suite on the stable channel. There are some UI tests that
+  # depend on the channel being built (for example if they include the
+  # channel name on the output), and this builder prevents landing
+  # changes that would result in broken builds after a promotion.
+  - name: x86_64-gnu-stable
+    env:
+      IMAGE: x86_64-gnu
+      RUST_CI_OVERRIDE_RELEASE_CHANNEL: stable
+      # Only run this job on the nightly channel. Running this on beta
+      # could cause failures when `dev: 1` in `stage0.txt`, and running
+      # this on stable is useless.
+      CI_ONLY_WHEN_CHANNEL: nightly
+    <<: *job-linux-4c
+
+  - name: x86_64-gnu-aux
+    <<: *job-linux-4c
+
+  - name: x86_64-gnu-integration
+    env:
+      # Only run this job on the nightly channel. Fuchsia requires
+      # nightly features to compile, and this job would fail if
+      # executed on beta and stable.
+      CI_ONLY_WHEN_CHANNEL: nightly
+    <<: *job-linux-8c
+
+  - name: x86_64-gnu-debug
+    <<: *job-linux-8c
+
+  - name: x86_64-gnu-distcheck
+    <<: *job-linux-8c
+
+  - name: x86_64-gnu-llvm-18
+    env:
+      RUST_BACKTRACE: 1
+    <<: *job-linux-8c
+
+  - name: x86_64-gnu-llvm-17
+    env:
+      RUST_BACKTRACE: 1
+    <<: *job-linux-8c
+
+  - name: x86_64-gnu-nopt
+    <<: *job-linux-4c
+
+  - name: x86_64-gnu-tools
+    env:
+      DEPLOY_TOOLSTATES_JSON: toolstates-linux.json
+    <<: *job-linux-8c
+
+  ####################
+  #  macOS Builders  #
+  ####################
+
+  - name: dist-x86_64-apple
+    env:
+      SCRIPT: ./x.py dist bootstrap --include-default-paths --host=x86_64-apple-darwin --target=x86_64-apple-darwin
+      RUST_CONFIGURE_ARGS: --enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc --set rust.lto=thin --set rust.codegen-units=1
+      RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
+      MACOSX_DEPLOYMENT_TARGET: 10.12
+      SELECT_XCODE: /Applications/Xcode_14.3.1.app
+      NO_LLVM_ASSERTIONS: 1
+      NO_DEBUG_ASSERTIONS: 1
+      NO_OVERFLOW_CHECKS: 1
+      DIST_REQUIRE_ALL_TOOLS: 1
+      CODEGEN_BACKENDS: llvm,cranelift
+    <<: *job-macos-xl
+
+  - name: dist-apple-various
+    env:
+      SCRIPT: ./x.py dist bootstrap --include-default-paths --host='' --target=aarch64-apple-ios,x86_64-apple-ios,aarch64-apple-ios-sim
+      RUST_CONFIGURE_ARGS: --enable-sanitizers --enable-profiler --set rust.jemalloc
+      RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
+      MACOSX_DEPLOYMENT_TARGET: 10.12
+      SELECT_XCODE: /Applications/Xcode_14.3.1.app
+      NO_LLVM_ASSERTIONS: 1
+      NO_DEBUG_ASSERTIONS: 1
+      NO_OVERFLOW_CHECKS: 1
+    <<: *job-macos-xl
+
+  - name: x86_64-apple-1
+    env: &env-x86_64-apple-tests
+      SCRIPT: ./x.py --stage 2 test --skip tests/ui --skip tests/rustdoc --skip tests/run-make-fulldeps
+      RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc
+      RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
+      MACOSX_DEPLOYMENT_TARGET: 10.12
+      MACOSX_STD_DEPLOYMENT_TARGET: 10.12
+      SELECT_XCODE: /Applications/Xcode_14.3.1.app
+      NO_LLVM_ASSERTIONS: 1
+      NO_DEBUG_ASSERTIONS: 1
+      NO_OVERFLOW_CHECKS: 1
+    <<: *job-macos-xl
+
+  - name: x86_64-apple-2
+    env:
+      SCRIPT: ./x.py --stage 2 test tests/ui tests/rustdoc tests/run-make-fulldeps
+      <<: *env-x86_64-apple-tests
+    <<: *job-macos-xl
+
+  # This target only needs to support 11.0 and up as nothing else supports the hardware
+  - name: dist-aarch64-apple
+    env:
+      SCRIPT: ./x.py dist bootstrap --include-default-paths --host=aarch64-apple-darwin --target=aarch64-apple-darwin
+      RUST_CONFIGURE_ARGS: >-
+        --enable-full-tools
+        --enable-sanitizers
+        --enable-profiler
+        --set rust.jemalloc
+        --set llvm.ninja=false
+        --set rust.lto=thin
+      RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
+      SELECT_XCODE: /Applications/Xcode_14.3.1.app
+      USE_XCODE_CLANG: 1
+      MACOSX_DEPLOYMENT_TARGET: 11.0
+      MACOSX_STD_DEPLOYMENT_TARGET: 11.0
+      NO_LLVM_ASSERTIONS: 1
+      NO_DEBUG_ASSERTIONS: 1
+      NO_OVERFLOW_CHECKS: 1
+      DIST_REQUIRE_ALL_TOOLS: 1
+    <<: *job-macos-m1
+
+  # This target only needs to support 11.0 and up as nothing else supports the hardware
+  - name: aarch64-apple
+    env:
+      SCRIPT: ./x.py --stage 2 test --host=aarch64-apple-darwin --target=aarch64-apple-darwin
+      RUST_CONFIGURE_ARGS: >-
+        --enable-sanitizers
+        --enable-profiler
+        --set rust.jemalloc
+      RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
+      SELECT_XCODE: /Applications/Xcode_14.3.1.app
+      USE_XCODE_CLANG: 1
+      MACOSX_DEPLOYMENT_TARGET: 11.0
+      MACOSX_STD_DEPLOYMENT_TARGET: 11.0
+      NO_LLVM_ASSERTIONS: 1
+      NO_DEBUG_ASSERTIONS: 1
+      NO_OVERFLOW_CHECKS: 1
+    <<: *job-macos-m1
+
+  ######################
+  #  Windows Builders  #
+  ######################
+
+  - name: x86_64-msvc
+    env:
+      RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler
+      SCRIPT: make ci-msvc
+    <<: *job-windows-8c
+
+  - name: i686-msvc
+    env:
+      RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc
+      SCRIPT: make ci-msvc
+    <<: *job-windows-8c
+
+  - name: x86_64-msvc-ext
+    env:
+      SCRIPT: python x.py --stage 2 test src/tools/cargotest src/tools/cargo && src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstate/toolstates.json windows
+      HOST_TARGET: x86_64-pc-windows-msvc
+      RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-lld --save-toolstates=/tmp/toolstate/toolstates.json
+      DEPLOY_TOOLSTATES_JSON: toolstates-windows.json
+    <<: *job-windows-8c
+
+  # 32/64-bit MinGW builds.
+  #
+  # We are using MinGW with POSIX threads since LLVM requires
+  # C++'s std::thread which is disabled in libstdc++ with win32 threads.
+  # FIXME: Libc++ doesn't have this limitation so we can avoid
+  # winpthreads if we switch to it.
+  #
+  # Instead of relying on the MinGW version installed on CI we download
+  # and install one ourselves so we won't be surprised by changes to CI's
+  # build image.
+  #
+  # Finally, note that the downloads below are all in the `rust-lang-ci` S3
+  # bucket, but they clearly didn't originate there! The downloads originally
+  # came from the mingw-w64 SourceForge download site. Unfortunately
+  # SourceForge is notoriously flaky, so we mirror it on our own infrastructure.
+
+  - name: i686-mingw
+    env:
+      RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu
+      SCRIPT: make ci-mingw
+      # We are intentionally allowing an old toolchain on this builder (and that's
+      # incompatible with LLVM downloads today).
+      NO_DOWNLOAD_CI_LLVM: 1
+      CUSTOM_MINGW: 1
+    <<: *job-windows-8c
+
+  - name: x86_64-mingw
+    env:
+      SCRIPT: make ci-mingw
+      RUST_CONFIGURE_ARGS: >-
+        --build=x86_64-pc-windows-gnu
+        --enable-profiler
+      # We are intentionally allowing an old toolchain on this builder (and that's
+      # incompatible with LLVM downloads today).
+      NO_DOWNLOAD_CI_LLVM: 1
+      CUSTOM_MINGW: 1
+    <<: *job-windows-8c
+
+  - name: dist-x86_64-msvc
+    env:
+      RUST_CONFIGURE_ARGS: >-
+        --build=x86_64-pc-windows-msvc
+        --host=x86_64-pc-windows-msvc
+        --target=x86_64-pc-windows-msvc
+        --enable-full-tools
+        --enable-profiler
+        --set rust.codegen-units=1
+      SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist windows-ci -- python x.py dist bootstrap --include-default-paths
+      DIST_REQUIRE_ALL_TOOLS: 1
+    <<: *job-windows-8c
+
+  - name: dist-i686-msvc
+    env:
+      RUST_CONFIGURE_ARGS: >-
+        --build=i686-pc-windows-msvc
+        --host=i686-pc-windows-msvc
+        --target=i686-pc-windows-msvc,i586-pc-windows-msvc
+        --enable-full-tools
+        --enable-profiler
+      SCRIPT: python x.py dist bootstrap --include-default-paths
+      DIST_REQUIRE_ALL_TOOLS: 1
+    <<: *job-windows-8c
+
+  - name: dist-aarch64-msvc
+    env:
+      RUST_CONFIGURE_ARGS: >-
+        --build=x86_64-pc-windows-msvc
+        --host=aarch64-pc-windows-msvc
+        --enable-full-tools
+        --enable-profiler
+      SCRIPT: python x.py dist bootstrap --include-default-paths
+      DIST_REQUIRE_ALL_TOOLS: 1
+    <<: *job-windows-8c
+
+  - name: dist-i686-mingw
+    env:
+      RUST_CONFIGURE_ARGS: >-
+        --build=i686-pc-windows-gnu
+        --enable-full-tools
+        --enable-profiler
+      # We are intentionally allowing an old toolchain on this builder (and that's
+      # incompatible with LLVM downloads today).
+      NO_DOWNLOAD_CI_LLVM: 1
+      SCRIPT: python x.py dist bootstrap --include-default-paths
+      CUSTOM_MINGW: 1
+      DIST_REQUIRE_ALL_TOOLS: 1
+    <<: *job-windows-8c
+
+  - name: dist-x86_64-mingw
+    env:
+      SCRIPT: python x.py dist bootstrap --include-default-paths
+      RUST_CONFIGURE_ARGS: >-
+        --build=x86_64-pc-windows-gnu
+        --enable-full-tools
+        --enable-profiler
+      # We are intentionally allowing an old toolchain on this builder (and that's
+      # incompatible with LLVM downloads today).
+      NO_DOWNLOAD_CI_LLVM: 1
+      CUSTOM_MINGW: 1
+      DIST_REQUIRE_ALL_TOOLS: 1
+    <<: *job-windows-8c
+
+  - name: dist-x86_64-msvc-alt
+    env:
+      RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler
+      SCRIPT: python x.py dist bootstrap --include-default-paths
+    <<: *job-windows-8c