From c327b91d22d95de2d43c899c3d8eb87a3ffca048 Mon Sep 17 00:00:00 2001 From: Jan Kassens Date: Wed, 14 Sep 2022 13:22:20 -0400 Subject: [PATCH] CI: try to make caching more reliable (#25259) - `~/.yarn/cache` is now restored from an hierarchical cache key, if no precise match is found, we fallback to less precise ones. - The yarn install in `fixtures/dom` is also cached. Notably, is utilizes the cache from root, but stores into its more precise key. - Steps running in root no longer have a `yarn install` and rely on the cache from the setup step. - Retry `yarn install` once on failure. --- .circleci/config.yml | 154 +++++++++++++++++++++++++------------------ 1 file changed, 90 insertions(+), 64 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5950e8bb93..43a299d9fe 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -10,18 +10,93 @@ aliases: - &restore_yarn_cache restore_cache: name: Restore yarn cache - key: v2-node-{{ arch }}-{{ checksum "yarn.lock" }}-yarn + keys: + - v1-yarn_cache-{{ arch }}-{{ checksum "yarn.lock" }} + - v1-yarn_cache-{{ arch }}- + - v1-yarn_cache- - - &prepare_node_modules_cache_key + - &yarn_install run: - name: Preparing node_modules cache key - command: yarn workspaces info | head -n -1 > workspace_info.txt + name: Install dependencies + command: yarn install --frozen-lockfile --cache-folder ~/.cache/yarn + + - &yarn_install_retry + run: + name: Install dependencies (retry) + when: on_fail + command: yarn install --frozen-lockfile --cache-folder ~/.cache/yarn + + - &save_yarn_cache + save_cache: + name: Save yarn cache + key: v1-yarn_cache-{{ arch }}-{{ checksum "yarn.lock" }} + paths: + - ~/.cache/yarn + + - &restore_yarn_cache_fixtures_dom + restore_cache: + name: Restore yarn cache for fixtures/dom + keys: + - v1-yarn_cache-{{ arch }}-{{ checksum "yarn.lock" }}-fixtures/dom + - v1-yarn_cache-{{ arch }}-{{ checksum "yarn.lock" }} + - v1-yarn_cache-{{ arch }}- + - v1-yarn_cache- + + - &yarn_install_fixtures_dom + run: + name: Install dependencies in fixtures/dom + working_directory: fixtures/dom + command: yarn install --frozen-lockfile --cache-folder ~/.cache/yarn + + - &yarn_install_fixtures_dom_retry + run: + name: Install dependencies in fixtures/dom (retry) + when: on_fail + working_directory: fixtures/dom + command: yarn install --frozen-lockfile --cache-folder ~/.cache/yarn + + - &save_yarn_cache_fixtures_dom + save_cache: + name: Save yarn cache for fixtures/dom + key: v1-yarn_cache-{{ arch }}-{{ checksum "yarn.lock" }}-fixtures/dom + paths: + - ~/.cache/yarn + + - &save_node_modules + save_cache: + name: Save node_modules cache + # Cache only for the current revision to prevent cache injections from + # malicious PRs. + key: v1-node_modules-{{ arch }}-{{ .Revision }} + paths: + - node_modules + - packages/eslint-plugin-react-hooks/node_modules + - packages/react-art/node_modules + - packages/react-client/node_modules + - packages/react-devtools-core/node_modules + - packages/react-devtools-extensions/node_modules + - packages/react-devtools-inline/node_modules + - packages/react-devtools-shared/node_modules + - packages/react-devtools-shell/node_modules + - packages/react-devtools-timeline/node_modules + - packages/react-devtools/node_modules + - packages/react-dom/node_modules + - packages/react-interactions/node_modules + - packages/react-native-renderer/node_modules + - packages/react-reconciler/node_modules + - packages/react-server-dom-relay/node_modules + - packages/react-server-dom-webpack/node_modules + - packages/react-server-native-relay/node_modules + - packages/react-server/node_modules + - packages/react-test-renderer/node_modules + - packages/react/node_modules + - packages/scheduler/node_modules - &restore_node_modules restore_cache: name: Restore node_modules cache keys: - - v2-node-{{ arch }}-{{ .Branch }}-{{ checksum "yarn.lock" }}-{{ checksum "workspace_info.txt" }}-node-modules + - v1-node_modules-{{ arch }}-{{ .Revision }} - &TEST_PARALLELISM 20 @@ -49,27 +124,11 @@ jobs: name: Nodejs Version command: node --version - *restore_yarn_cache - - run: - name: Install Packages - command: yarn --frozen-lockfile --cache-folder ~/.cache/yarn - - run: yarn workspaces info | head -n -1 > workspace_info.txt - - save_cache: - # Store the yarn cache globally for all lock files with this same - # checksum. This will speed up the setup job for all PRs where the - # lockfile is the same. - name: Save yarn cache for future installs - key: v2-node-{{ arch }}-{{ checksum "yarn.lock" }}-yarn - paths: - - ~/.cache/yarn - - save_cache: - # Store node_modules for all jobs in this workflow so that they don't - # need to each run a yarn install for each job. This will speed up - # all jobs run on this branch with the same lockfile. - name: Save node_modules cache - # This cache key is per branch, a yarn install in setup is required. - key: v2-node-{{ arch }}-{{ .Branch }}-{{ checksum "yarn.lock" }}-{{ checksum "workspace_info.txt" }}-node-modules - paths: - - node_modules + - *restore_node_modules + - *yarn_install + - *yarn_install_retry + - *save_yarn_cache + - *save_node_modules yarn_lint: docker: *docker @@ -77,7 +136,6 @@ jobs: steps: - checkout - - *prepare_node_modules_cache_key - *restore_node_modules - run: node ./scripts/prettier/index - run: node ./scripts/tasks/eslint @@ -92,7 +150,6 @@ jobs: steps: - checkout - - *prepare_node_modules_cache_key - *restore_node_modules - run: node ./scripts/tasks/flow-ci @@ -102,7 +159,6 @@ jobs: steps: - checkout - - *prepare_node_modules_cache_key - *restore_node_modules - run: command: | @@ -119,7 +175,6 @@ jobs: parallelism: 40 steps: - checkout - - *prepare_node_modules_cache_key - *restore_node_modules - run: yarn build-combined - persist_to_workspace: @@ -135,7 +190,6 @@ jobs: type: string steps: - checkout - - *prepare_node_modules_cache_key - *restore_node_modules - run: name: Download artifacts for revision @@ -153,7 +207,6 @@ jobs: environment: *environment steps: - checkout - - *prepare_node_modules_cache_key - *restore_node_modules - run: name: Download artifacts for base revision @@ -182,7 +235,6 @@ jobs: - checkout - attach_workspace: at: . - - *prepare_node_modules_cache_key - *restore_node_modules - run: echo "<< pipeline.git.revision >>" >> build/COMMIT_SHA # Compress build directory into a single tarball for easy download @@ -202,7 +254,6 @@ jobs: - attach_workspace: at: . - run: echo "<< pipeline.git.revision >>" >> build/COMMIT_SHA - - *prepare_node_modules_cache_key - *restore_node_modules - run: command: node ./scripts/tasks/danger @@ -214,11 +265,7 @@ jobs: - checkout - attach_workspace: at: . - - *prepare_node_modules_cache_key - *restore_node_modules - - run: - name: Install Packages - command: yarn --frozen-lockfile --cache-folder ~/.cache/yarn - run: environment: RELEASE_CHANNEL: experimental @@ -233,11 +280,7 @@ jobs: - checkout - attach_workspace: at: . - - *prepare_node_modules_cache_key - *restore_node_modules - - run: - name: Install Packages - command: yarn --frozen-lockfile --cache-folder ~/.cache/yarn - run: name: Playwright install deps command: | @@ -259,11 +302,7 @@ jobs: - checkout - attach_workspace: at: . - - *prepare_node_modules_cache_key - *restore_node_modules - - run: - name: Install nested packages from Yarn cache - command: yarn --frozen-lockfile --cache-folder ~/.cache/yarn - run: ./scripts/circleci/download_devtools_regression_build.js << parameters.version >> --replaceBuild - run: node ./scripts/jest/jest-cli.js --build --project devtools --release-channel=experimental --reactVersion << parameters.version >> --ci @@ -278,11 +317,7 @@ jobs: - checkout - attach_workspace: at: . - - *prepare_node_modules_cache_key - *restore_node_modules - - run: - name: Install nested packages from Yarn cache - command: yarn --frozen-lockfile --cache-folder ~/.cache/yarn - run: name: Playwright install deps command: | @@ -306,7 +341,6 @@ jobs: - checkout - attach_workspace: at: . - - *prepare_node_modules_cache_key - *restore_node_modules - run: yarn lint-build @@ -317,7 +351,6 @@ jobs: - checkout - attach_workspace: at: . - - *prepare_node_modules_cache_key - *restore_node_modules - run: yarn check-release-dependencies @@ -328,7 +361,6 @@ jobs: steps: - checkout - attach_workspace: *attach_workspace - - *prepare_node_modules_cache_key - *restore_node_modules - run: name: Search build artifacts for unminified errors @@ -345,7 +377,6 @@ jobs: type: string steps: - checkout - - *prepare_node_modules_cache_key - *restore_node_modules - run: yarn test <> --ci @@ -360,11 +391,7 @@ jobs: - checkout - attach_workspace: at: . - - *prepare_node_modules_cache_key - *restore_node_modules - - run: - name: Install nested packages from Yarn cache - command: yarn --frozen-lockfile --cache-folder ~/.cache/yarn - run: yarn test --build <> --ci RELEASE_CHANNEL_stable_yarn_test_dom_fixtures: @@ -374,15 +401,17 @@ jobs: - checkout - attach_workspace: at: . - - *prepare_node_modules_cache_key - *restore_node_modules + - *restore_yarn_cache_fixtures_dom + - *yarn_install_fixtures_dom + - *yarn_install_fixtures_dom_retry + - *save_yarn_cache_fixtures_dom - run: name: Run DOM fixture tests environment: RELEASE_CHANNEL: stable + working_directory: fixtures/dom command: | - cd fixtures/dom - yarn --frozen-lockfile yarn prestart yarn test --maxWorkers=2 @@ -391,7 +420,6 @@ jobs: environment: *environment steps: - checkout - - *prepare_node_modules_cache_key - *restore_node_modules - run: name: Run fuzz tests @@ -411,7 +439,6 @@ jobs: environment: *environment steps: - checkout - - *prepare_node_modules_cache_key - *restore_node_modules - run: name: Run publish script @@ -430,7 +457,6 @@ jobs: environment: *environment steps: - checkout - - *prepare_node_modules_cache_key - *restore_node_modules - run: name: Fetch revisions that contain an intentional fork