CI: try to make caching more reliable (#25259)
- `~/.yarn/cache` is now restored from an hierarchical cache key, if no precise match is found, we fallback to less precise ones. - The yarn install in `fixtures/dom` is also cached. Notably, is utilizes the cache from root, but stores into its more precise key. - Steps running in root no longer have a `yarn install` and rely on the cache from the setup step. - Retry `yarn install` once on failure.
This commit is contained in:
Jan Kassens
GitHub
parent
afe664d9d7
commit
c327b91d22
|
|
@ -10,18 +10,93 @@ aliases:
|
|||
- &restore_yarn_cache
|
||||
restore_cache:
|
||||
name: Restore yarn cache
|
||||
key: v2-node-{{ arch }}-{{ checksum "yarn.lock" }}-yarn
|
||||
keys:
|
||||
- v1-yarn_cache-{{ arch }}-{{ checksum "yarn.lock" }}
|
||||
- v1-yarn_cache-{{ arch }}-
|
||||
- v1-yarn_cache-
|
||||
|
||||
- &prepare_node_modules_cache_key
|
||||
- &yarn_install
|
||||
run:
|
||||
name: Preparing node_modules cache key
|
||||
command: yarn workspaces info | head -n -1 > workspace_info.txt
|
||||
name: Install dependencies
|
||||
command: yarn install --frozen-lockfile --cache-folder ~/.cache/yarn
|
||||
|
||||
- &yarn_install_retry
|
||||
run:
|
||||
name: Install dependencies (retry)
|
||||
when: on_fail
|
||||
command: yarn install --frozen-lockfile --cache-folder ~/.cache/yarn
|
||||
|
||||
- &save_yarn_cache
|
||||
save_cache:
|
||||
name: Save yarn cache
|
||||
key: v1-yarn_cache-{{ arch }}-{{ checksum "yarn.lock" }}
|
||||
paths:
|
||||
- ~/.cache/yarn
|
||||
|
||||
- &restore_yarn_cache_fixtures_dom
|
||||
restore_cache:
|
||||
name: Restore yarn cache for fixtures/dom
|
||||
keys:
|
||||
- v1-yarn_cache-{{ arch }}-{{ checksum "yarn.lock" }}-fixtures/dom
|
||||
- v1-yarn_cache-{{ arch }}-{{ checksum "yarn.lock" }}
|
||||
- v1-yarn_cache-{{ arch }}-
|
||||
- v1-yarn_cache-
|
||||
|
||||
- &yarn_install_fixtures_dom
|
||||
run:
|
||||
name: Install dependencies in fixtures/dom
|
||||
working_directory: fixtures/dom
|
||||
command: yarn install --frozen-lockfile --cache-folder ~/.cache/yarn
|
||||
|
||||
- &yarn_install_fixtures_dom_retry
|
||||
run:
|
||||
name: Install dependencies in fixtures/dom (retry)
|
||||
when: on_fail
|
||||
working_directory: fixtures/dom
|
||||
command: yarn install --frozen-lockfile --cache-folder ~/.cache/yarn
|
||||
|
||||
- &save_yarn_cache_fixtures_dom
|
||||
save_cache:
|
||||
name: Save yarn cache for fixtures/dom
|
||||
key: v1-yarn_cache-{{ arch }}-{{ checksum "yarn.lock" }}-fixtures/dom
|
||||
paths:
|
||||
- ~/.cache/yarn
|
||||
|
||||
- &save_node_modules
|
||||
save_cache:
|
||||
name: Save node_modules cache
|
||||
# Cache only for the current revision to prevent cache injections from
|
||||
# malicious PRs.
|
||||
key: v1-node_modules-{{ arch }}-{{ .Revision }}
|
||||
paths:
|
||||
- node_modules
|
||||
- packages/eslint-plugin-react-hooks/node_modules
|
||||
- packages/react-art/node_modules
|
||||
- packages/react-client/node_modules
|
||||
- packages/react-devtools-core/node_modules
|
||||
- packages/react-devtools-extensions/node_modules
|
||||
- packages/react-devtools-inline/node_modules
|
||||
- packages/react-devtools-shared/node_modules
|
||||
- packages/react-devtools-shell/node_modules
|
||||
- packages/react-devtools-timeline/node_modules
|
||||
- packages/react-devtools/node_modules
|
||||
- packages/react-dom/node_modules
|
||||
- packages/react-interactions/node_modules
|
||||
- packages/react-native-renderer/node_modules
|
||||
- packages/react-reconciler/node_modules
|
||||
- packages/react-server-dom-relay/node_modules
|
||||
- packages/react-server-dom-webpack/node_modules
|
||||
- packages/react-server-native-relay/node_modules
|
||||
- packages/react-server/node_modules
|
||||
- packages/react-test-renderer/node_modules
|
||||
- packages/react/node_modules
|
||||
- packages/scheduler/node_modules
|
||||
|
||||
- &restore_node_modules
|
||||
restore_cache:
|
||||
name: Restore node_modules cache
|
||||
keys:
|
||||
- v2-node-{{ arch }}-{{ .Branch }}-{{ checksum "yarn.lock" }}-{{ checksum "workspace_info.txt" }}-node-modules
|
||||
- v1-node_modules-{{ arch }}-{{ .Revision }}
|
||||
|
||||
- &TEST_PARALLELISM 20
|
||||
|
||||
|
|
@ -49,27 +124,11 @@ jobs:
|
|||
name: Nodejs Version
|
||||
command: node --version
|
||||
- *restore_yarn_cache
|
||||
- run:
|
||||
name: Install Packages
|
||||
command: yarn --frozen-lockfile --cache-folder ~/.cache/yarn
|
||||
- run: yarn workspaces info | head -n -1 > workspace_info.txt
|
||||
- save_cache:
|
||||
# Store the yarn cache globally for all lock files with this same
|
||||
# checksum. This will speed up the setup job for all PRs where the
|
||||
# lockfile is the same.
|
||||
name: Save yarn cache for future installs
|
||||
key: v2-node-{{ arch }}-{{ checksum "yarn.lock" }}-yarn
|
||||
paths:
|
||||
- ~/.cache/yarn
|
||||
- save_cache:
|
||||
# Store node_modules for all jobs in this workflow so that they don't
|
||||
# need to each run a yarn install for each job. This will speed up
|
||||
# all jobs run on this branch with the same lockfile.
|
||||
name: Save node_modules cache
|
||||
# This cache key is per branch, a yarn install in setup is required.
|
||||
key: v2-node-{{ arch }}-{{ .Branch }}-{{ checksum "yarn.lock" }}-{{ checksum "workspace_info.txt" }}-node-modules
|
||||
paths:
|
||||
- node_modules
|
||||
- *restore_node_modules
|
||||
- *yarn_install
|
||||
- *yarn_install_retry
|
||||
- *save_yarn_cache
|
||||
- *save_node_modules
|
||||
|
||||
yarn_lint:
|
||||
docker: *docker
|
||||
|
|
@ -77,7 +136,6 @@ jobs:
|
|||
|
||||
steps:
|
||||
- checkout
|
||||
- *prepare_node_modules_cache_key
|
||||
- *restore_node_modules
|
||||
- run: node ./scripts/prettier/index
|
||||
- run: node ./scripts/tasks/eslint
|
||||
|
|
@ -92,7 +150,6 @@ jobs:
|
|||
|
||||
steps:
|
||||
- checkout
|
||||
- *prepare_node_modules_cache_key
|
||||
- *restore_node_modules
|
||||
- run: node ./scripts/tasks/flow-ci
|
||||
|
||||
|
|
@ -102,7 +159,6 @@ jobs:
|
|||
|
||||
steps:
|
||||
- checkout
|
||||
- *prepare_node_modules_cache_key
|
||||
- *restore_node_modules
|
||||
- run:
|
||||
command: |
|
||||
|
|
@ -119,7 +175,6 @@ jobs:
|
|||
parallelism: 40
|
||||
steps:
|
||||
- checkout
|
||||
- *prepare_node_modules_cache_key
|
||||
- *restore_node_modules
|
||||
- run: yarn build-combined
|
||||
- persist_to_workspace:
|
||||
|
|
@ -135,7 +190,6 @@ jobs:
|
|||
type: string
|
||||
steps:
|
||||
- checkout
|
||||
- *prepare_node_modules_cache_key
|
||||
- *restore_node_modules
|
||||
- run:
|
||||
name: Download artifacts for revision
|
||||
|
|
@ -153,7 +207,6 @@ jobs:
|
|||
environment: *environment
|
||||
steps:
|
||||
- checkout
|
||||
- *prepare_node_modules_cache_key
|
||||
- *restore_node_modules
|
||||
- run:
|
||||
name: Download artifacts for base revision
|
||||
|
|
@ -182,7 +235,6 @@ jobs:
|
|||
- checkout
|
||||
- attach_workspace:
|
||||
at: .
|
||||
- *prepare_node_modules_cache_key
|
||||
- *restore_node_modules
|
||||
- run: echo "<< pipeline.git.revision >>" >> build/COMMIT_SHA
|
||||
# Compress build directory into a single tarball for easy download
|
||||
|
|
@ -202,7 +254,6 @@ jobs:
|
|||
- attach_workspace:
|
||||
at: .
|
||||
- run: echo "<< pipeline.git.revision >>" >> build/COMMIT_SHA
|
||||
- *prepare_node_modules_cache_key
|
||||
- *restore_node_modules
|
||||
- run:
|
||||
command: node ./scripts/tasks/danger
|
||||
|
|
@ -214,11 +265,7 @@ jobs:
|
|||
- checkout
|
||||
- attach_workspace:
|
||||
at: .
|
||||
- *prepare_node_modules_cache_key
|
||||
- *restore_node_modules
|
||||
- run:
|
||||
name: Install Packages
|
||||
command: yarn --frozen-lockfile --cache-folder ~/.cache/yarn
|
||||
- run:
|
||||
environment:
|
||||
RELEASE_CHANNEL: experimental
|
||||
|
|
@ -233,11 +280,7 @@ jobs:
|
|||
- checkout
|
||||
- attach_workspace:
|
||||
at: .
|
||||
- *prepare_node_modules_cache_key
|
||||
- *restore_node_modules
|
||||
- run:
|
||||
name: Install Packages
|
||||
command: yarn --frozen-lockfile --cache-folder ~/.cache/yarn
|
||||
- run:
|
||||
name: Playwright install deps
|
||||
command: |
|
||||
|
|
@ -259,11 +302,7 @@ jobs:
|
|||
- checkout
|
||||
- attach_workspace:
|
||||
at: .
|
||||
- *prepare_node_modules_cache_key
|
||||
- *restore_node_modules
|
||||
- run:
|
||||
name: Install nested packages from Yarn cache
|
||||
command: yarn --frozen-lockfile --cache-folder ~/.cache/yarn
|
||||
- run: ./scripts/circleci/download_devtools_regression_build.js << parameters.version >> --replaceBuild
|
||||
- run: node ./scripts/jest/jest-cli.js --build --project devtools --release-channel=experimental --reactVersion << parameters.version >> --ci
|
||||
|
||||
|
|
@ -278,11 +317,7 @@ jobs:
|
|||
- checkout
|
||||
- attach_workspace:
|
||||
at: .
|
||||
- *prepare_node_modules_cache_key
|
||||
- *restore_node_modules
|
||||
- run:
|
||||
name: Install nested packages from Yarn cache
|
||||
command: yarn --frozen-lockfile --cache-folder ~/.cache/yarn
|
||||
- run:
|
||||
name: Playwright install deps
|
||||
command: |
|
||||
|
|
@ -306,7 +341,6 @@ jobs:
|
|||
- checkout
|
||||
- attach_workspace:
|
||||
at: .
|
||||
- *prepare_node_modules_cache_key
|
||||
- *restore_node_modules
|
||||
- run: yarn lint-build
|
||||
|
||||
|
|
@ -317,7 +351,6 @@ jobs:
|
|||
- checkout
|
||||
- attach_workspace:
|
||||
at: .
|
||||
- *prepare_node_modules_cache_key
|
||||
- *restore_node_modules
|
||||
- run: yarn check-release-dependencies
|
||||
|
||||
|
|
@ -328,7 +361,6 @@ jobs:
|
|||
steps:
|
||||
- checkout
|
||||
- attach_workspace: *attach_workspace
|
||||
- *prepare_node_modules_cache_key
|
||||
- *restore_node_modules
|
||||
- run:
|
||||
name: Search build artifacts for unminified errors
|
||||
|
|
@ -345,7 +377,6 @@ jobs:
|
|||
type: string
|
||||
steps:
|
||||
- checkout
|
||||
- *prepare_node_modules_cache_key
|
||||
- *restore_node_modules
|
||||
- run: yarn test <<parameters.args>> --ci
|
||||
|
||||
|
|
@ -360,11 +391,7 @@ jobs:
|
|||
- checkout
|
||||
- attach_workspace:
|
||||
at: .
|
||||
- *prepare_node_modules_cache_key
|
||||
- *restore_node_modules
|
||||
- run:
|
||||
name: Install nested packages from Yarn cache
|
||||
command: yarn --frozen-lockfile --cache-folder ~/.cache/yarn
|
||||
- run: yarn test --build <<parameters.args>> --ci
|
||||
|
||||
RELEASE_CHANNEL_stable_yarn_test_dom_fixtures:
|
||||
|
|
@ -374,15 +401,17 @@ jobs:
|
|||
- checkout
|
||||
- attach_workspace:
|
||||
at: .
|
||||
- *prepare_node_modules_cache_key
|
||||
- *restore_node_modules
|
||||
- *restore_yarn_cache_fixtures_dom
|
||||
- *yarn_install_fixtures_dom
|
||||
- *yarn_install_fixtures_dom_retry
|
||||
- *save_yarn_cache_fixtures_dom
|
||||
- run:
|
||||
name: Run DOM fixture tests
|
||||
environment:
|
||||
RELEASE_CHANNEL: stable
|
||||
working_directory: fixtures/dom
|
||||
command: |
|
||||
cd fixtures/dom
|
||||
yarn --frozen-lockfile
|
||||
yarn prestart
|
||||
yarn test --maxWorkers=2
|
||||
|
||||
|
|
@ -391,7 +420,6 @@ jobs:
|
|||
environment: *environment
|
||||
steps:
|
||||
- checkout
|
||||
- *prepare_node_modules_cache_key
|
||||
- *restore_node_modules
|
||||
- run:
|
||||
name: Run fuzz tests
|
||||
|
|
@ -411,7 +439,6 @@ jobs:
|
|||
environment: *environment
|
||||
steps:
|
||||
- checkout
|
||||
- *prepare_node_modules_cache_key
|
||||
- *restore_node_modules
|
||||
- run:
|
||||
name: Run publish script
|
||||
|
|
@ -430,7 +457,6 @@ jobs:
|
|||
environment: *environment
|
||||
steps:
|
||||
- checkout
|
||||
- *prepare_node_modules_cache_key
|
||||
- *restore_node_modules
|
||||
- run:
|
||||
name: Fetch revisions that contain an intentional fork
|
||||
|
|
|
|||
Loading…
Reference in New Issue