CI: try to make caching more reliable (#25259)

- `~/.yarn/cache` is now restored from an hierarchical cache key, if no precise match is found, we fallback to less precise ones. 
- The yarn install in `fixtures/dom` is also cached. Notably, is utilizes the cache from root, but stores into its more precise key.
- Steps running in root no longer have a `yarn install` and rely on the cache from the setup step.
- Retry `yarn install` once on failure.

Author: kassens

.circleci/config.yml

@@ -10,18 +10,93 @@ aliases:
   - &restore_yarn_cache
     restore_cache:
       name: Restore yarn cache
-      key: v2-node-{{ arch }}-{{ checksum "yarn.lock" }}-yarn
+      keys:
+        - v1-yarn_cache-{{ arch }}-{{ checksum "yarn.lock" }}
+        - v1-yarn_cache-{{ arch }}-
+        - v1-yarn_cache-
+
+  - &yarn_install
+    run:
+      name: Install dependencies
+      command: yarn install --frozen-lockfile --cache-folder ~/.cache/yarn
+
+  - &yarn_install_retry
+    run:
+      name: Install dependencies (retry)
+      when: on_fail
+      command: yarn install --frozen-lockfile --cache-folder ~/.cache/yarn
+
+  - &save_yarn_cache
+    save_cache:
+      name: Save yarn cache
+      key: v1-yarn_cache-{{ arch }}-{{ checksum "yarn.lock" }}
+      paths:
+        - ~/.cache/yarn
+
+  - &restore_yarn_cache_fixtures_dom
+    restore_cache:
+      name: Restore yarn cache for fixtures/dom
+      keys:
+        - v1-yarn_cache-{{ arch }}-{{ checksum "yarn.lock" }}-fixtures/dom
+        - v1-yarn_cache-{{ arch }}-{{ checksum "yarn.lock" }}
+        - v1-yarn_cache-{{ arch }}-
+        - v1-yarn_cache-
+
+  - &yarn_install_fixtures_dom
+    run:
+      name: Install dependencies in fixtures/dom
+      working_directory: fixtures/dom
+      command: yarn install --frozen-lockfile --cache-folder ~/.cache/yarn
 
-  - &prepare_node_modules_cache_key
+  - &yarn_install_fixtures_dom_retry
     run:
-      name: Preparing node_modules cache key
-      command: yarn workspaces info | head -n -1 > workspace_info.txt
+      name: Install dependencies in fixtures/dom (retry)
+      when: on_fail
+      working_directory: fixtures/dom
+      command: yarn install --frozen-lockfile --cache-folder ~/.cache/yarn
+
+  - &save_yarn_cache_fixtures_dom
+    save_cache:
+      name: Save yarn cache for fixtures/dom
+      key: v1-yarn_cache-{{ arch }}-{{ checksum "yarn.lock" }}-fixtures/dom
+      paths:
+        - ~/.cache/yarn
+
+  - &save_node_modules
+    save_cache:
+      name: Save node_modules cache
+      # Cache only for the current revision to prevent cache injections from
+      # malicious PRs.
+      key: v1-node_modules-{{ arch }}-{{ .Revision }}
+      paths:
+        - node_modules
+        - packages/eslint-plugin-react-hooks/node_modules
+        - packages/react-art/node_modules
+        - packages/react-client/node_modules
+        - packages/react-devtools-core/node_modules
+        - packages/react-devtools-extensions/node_modules
+        - packages/react-devtools-inline/node_modules
+        - packages/react-devtools-shared/node_modules
+        - packages/react-devtools-shell/node_modules
+        - packages/react-devtools-timeline/node_modules
+        - packages/react-devtools/node_modules
+        - packages/react-dom/node_modules
+        - packages/react-interactions/node_modules
+        - packages/react-native-renderer/node_modules
+        - packages/react-reconciler/node_modules
+        - packages/react-server-dom-relay/node_modules
+        - packages/react-server-dom-webpack/node_modules
+        - packages/react-server-native-relay/node_modules
+        - packages/react-server/node_modules
+        - packages/react-test-renderer/node_modules
+        - packages/react/node_modules
+        - packages/scheduler/node_modules
 
   - &restore_node_modules
     restore_cache:
       name: Restore node_modules cache
       keys:
-        - v2-node-{{ arch }}-{{ .Branch }}-{{ checksum "yarn.lock" }}-{{ checksum "workspace_info.txt" }}-node-modules
+        - v1-node_modules-{{ arch }}-{{ .Revision }}
 
   - &TEST_PARALLELISM 20
 
@@ -49,35 +124,18 @@ jobs:
           name: Nodejs Version
           command: node --version
       - *restore_yarn_cache
-      - run:
-          name: Install Packages
-          command: yarn --frozen-lockfile --cache-folder ~/.cache/yarn
-      - run: yarn workspaces info | head -n -1 > workspace_info.txt
-      - save_cache:
-          # Store the yarn cache globally for all lock files with this same
-          # checksum. This will speed up the setup job for all PRs where the
-          # lockfile is the same.
-          name: Save yarn cache for future installs
-          key: v2-node-{{ arch }}-{{ checksum "yarn.lock" }}-yarn
-          paths:
-            - ~/.cache/yarn
-      - save_cache:
-          # Store node_modules for all jobs in this workflow so that they don't
-          # need to each run a yarn install for each job. This will speed up
-          # all jobs run on this branch with the same lockfile.
-          name: Save node_modules cache
-          # This cache key is per branch, a yarn install in setup is required.
-          key: v2-node-{{ arch }}-{{ .Branch }}-{{ checksum "yarn.lock" }}-{{ checksum "workspace_info.txt" }}-node-modules
-          paths:
-            - node_modules
+      - *restore_node_modules
+      - *yarn_install
+      - *yarn_install_retry
+      - *save_yarn_cache
+      - *save_node_modules
 
   yarn_lint:
     docker: *docker
     environment: *environment
 
     steps:
       - checkout
-      - *prepare_node_modules_cache_key
       - *restore_node_modules
       - run: node ./scripts/prettier/index
       - run: node ./scripts/tasks/eslint
@@ -92,7 +150,6 @@ jobs:
 
     steps:
       - checkout
-      - *prepare_node_modules_cache_key
       - *restore_node_modules
       - run: node ./scripts/tasks/flow-ci
 
@@ -102,7 +159,6 @@ jobs:
 
     steps:
       - checkout
-      - *prepare_node_modules_cache_key
       - *restore_node_modules
       - run:
           command: |
@@ -119,7 +175,6 @@ jobs:
     parallelism: 40
     steps:
       - checkout
-      - *prepare_node_modules_cache_key
       - *restore_node_modules
       - run: yarn build-combined
       - persist_to_workspace:
@@ -135,7 +190,6 @@ jobs:
         type: string
     steps:
       - checkout
-      - *prepare_node_modules_cache_key
       - *restore_node_modules
       - run:
           name: Download artifacts for revision
@@ -153,7 +207,6 @@ jobs:
     environment: *environment
     steps:
       - checkout
-      - *prepare_node_modules_cache_key
       - *restore_node_modules
       - run:
           name: Download artifacts for base revision
@@ -182,7 +235,6 @@ jobs:
       - checkout
       - attach_workspace:
           at: .
-      - *prepare_node_modules_cache_key
       - *restore_node_modules
       - run: echo "<< pipeline.git.revision	>>" >> build/COMMIT_SHA
         # Compress build directory into a single tarball for easy download
@@ -202,7 +254,6 @@ jobs:
       - attach_workspace:
           at: .
       - run: echo "<< pipeline.git.revision	>>" >> build/COMMIT_SHA
-      - *prepare_node_modules_cache_key
       - *restore_node_modules
       - run:
           command: node ./scripts/tasks/danger
@@ -214,11 +265,7 @@ jobs:
       - checkout
       - attach_workspace:
           at: .
-      - *prepare_node_modules_cache_key
       - *restore_node_modules
-      - run:
-          name: Install Packages
-          command: yarn --frozen-lockfile --cache-folder ~/.cache/yarn
       - run:
           environment:
             RELEASE_CHANNEL: experimental
@@ -233,11 +280,7 @@ jobs:
       - checkout
       - attach_workspace:
           at: .
-      - *prepare_node_modules_cache_key
       - *restore_node_modules
-      - run:
-          name: Install Packages
-          command: yarn --frozen-lockfile --cache-folder ~/.cache/yarn
       - run:
           name: Playwright install deps
           command: |
@@ -259,11 +302,7 @@ jobs:
       - checkout
       - attach_workspace:
           at: .
-      - *prepare_node_modules_cache_key
       - *restore_node_modules
-      - run:
-          name: Install nested packages from Yarn cache
-          command: yarn --frozen-lockfile --cache-folder ~/.cache/yarn
       - run: ./scripts/circleci/download_devtools_regression_build.js << parameters.version >> --replaceBuild
       - run: node ./scripts/jest/jest-cli.js --build --project devtools --release-channel=experimental --reactVersion << parameters.version >> --ci
 
@@ -278,11 +317,7 @@ jobs:
       - checkout
       - attach_workspace:
           at: .
-      - *prepare_node_modules_cache_key
       - *restore_node_modules
-      - run:
-          name: Install nested packages from Yarn cache
-          command: yarn --frozen-lockfile --cache-folder ~/.cache/yarn
       - run:
           name: Playwright install deps
           command: |
@@ -306,7 +341,6 @@ jobs:
       - checkout
       - attach_workspace:
           at: .
-      - *prepare_node_modules_cache_key
       - *restore_node_modules
       - run: yarn lint-build
 
@@ -317,7 +351,6 @@ jobs:
       - checkout
       - attach_workspace:
           at: .
-      - *prepare_node_modules_cache_key
       - *restore_node_modules
       - run: yarn check-release-dependencies
 
@@ -328,7 +361,6 @@ jobs:
     steps:
       - checkout
       - attach_workspace: *attach_workspace
-      - *prepare_node_modules_cache_key
       - *restore_node_modules
       - run:
           name: Search build artifacts for unminified errors
@@ -345,7 +377,6 @@ jobs:
         type: string
     steps:
       - checkout
-      - *prepare_node_modules_cache_key
       - *restore_node_modules
       - run: yarn test <<parameters.args>> --ci
 
@@ -360,11 +391,7 @@ jobs:
       - checkout
       - attach_workspace:
           at: .
-      - *prepare_node_modules_cache_key
       - *restore_node_modules
-      - run:
-          name: Install nested packages from Yarn cache
-          command: yarn --frozen-lockfile --cache-folder ~/.cache/yarn
       - run: yarn test --build <<parameters.args>> --ci
 
   RELEASE_CHANNEL_stable_yarn_test_dom_fixtures:
@@ -374,15 +401,17 @@ jobs:
       - checkout
       - attach_workspace:
           at: .
-      - *prepare_node_modules_cache_key
       - *restore_node_modules
+      - *restore_yarn_cache_fixtures_dom
+      - *yarn_install_fixtures_dom
+      - *yarn_install_fixtures_dom_retry
+      - *save_yarn_cache_fixtures_dom
       - run:
           name: Run DOM fixture tests
           environment:
             RELEASE_CHANNEL: stable
+          working_directory: fixtures/dom
           command: |
-            cd fixtures/dom
-            yarn --frozen-lockfile
             yarn prestart
             yarn test --maxWorkers=2
 
@@ -391,7 +420,6 @@ jobs:
     environment: *environment
     steps:
       - checkout
-      - *prepare_node_modules_cache_key
       - *restore_node_modules
       - run:
           name: Run fuzz tests
@@ -411,7 +439,6 @@ jobs:
     environment: *environment
     steps:
       - checkout
-      - *prepare_node_modules_cache_key
       - *restore_node_modules
       - run:
           name: Run publish script
@@ -430,7 +457,6 @@ jobs:
     environment: *environment
     steps:
       - checkout
-      - *prepare_node_modules_cache_key
       - *restore_node_modules
       - run:
           name: Fetch revisions that contain an intentional fork