ci(workflow): update artifact actions (vercel#61061)

### What Closes PACK-2284 This PR upgrades `upload/download-artifact` action to the latest version. Per its claim, it can be faster 90% compare to the current in worst case use case. Below's comparision between this PR vs. current branch for the datadog report ![image](https://github.com/vercel/next.js/assets/1210596/f3db6f4d-e137-4013-9745-b8fa55ba1014) ![image](https://github.com/vercel/next.js/assets/1210596/08734620-8530-4b6e-bd09-a5414c703c06) Cuts download time meaningfully different. Since we use upload/download in other places as well (i.e download next-swc binary) overall CI time would be improved. The challenage is artifact@v4 introduced breaking changes to not to allow implicitly merge upload / download with duplicated name. PR introduced unique key for those, then apply download with pattern & merge.
MLH-Fellowship · Jan 25, 2024 · a9c7559 · a9c7559
1 parent 9a808e6
commit a9c7559
Show file tree

Hide file tree

Showing 8 changed files with 75 additions and 38 deletions.
diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml
@@ -323,21 +323,21 @@ jobs:
 
       - name: Upload turbopack bytesize artifact
         if: ${{ steps.check-did-build.outputs.DID_BUILD == 'true' }}
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: turbopack-bytesize
+          name: turbopack-bytesize-${{ matrix.settings.target }}
           path: turbopack-bin-size/*
 
       - name: Upload swc artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: next-swc-binaries
+          name: next-swc-binaries-${{ matrix.settings.target }}
           path: packages/next-swc/native/next-swc.*.node
 
       - name: Upload turbo summary artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: turbo run summary
+          name: turbo-run-summary-${{ matrix.settings.target }}
           path: .turbo/runs
 
   build-wasm:
@@ -381,13 +381,13 @@ jobs:
         run: '[[ -d "packages/next-swc/crates/wasm/pkg" ]] && mv packages/next-swc/crates/wasm/pkg packages/next-swc/crates/wasm/pkg-${{ matrix.target }} || ls packages/next-swc/crates/wasm'
 
       - name: Upload turbo summary artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: turbo run summary
+          name: turbo-run-summary-wasm
           path: .turbo/runs
 
       - name: Upload swc artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: wasm-binaries
           path: packages/next-swc/crates/wasm/pkg-*
@@ -424,12 +424,13 @@ jobs:
           path: ./*
           key: ${{ github.sha }}-${{ github.run_number }}
 
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
-          name: next-swc-binaries
+          pattern: next-swc-binaries-*
+          merge-multiple: true
           path: packages/next-swc/native
 
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           name: wasm-binaries
           path: packages/next-swc/crates/wasm
@@ -478,9 +479,10 @@ jobs:
         with:
           fetch-depth: 25
 
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
-          name: next-swc-binaries
+          pattern: next-swc-binaries
+          merge-multiple: true
           path: packages/next-swc/native
 
       - run: cp -r packages/next-swc/native .github/actions/next-stats-action/native
@@ -499,9 +501,10 @@ jobs:
       DATADOG_API_KEY: ${{ secrets.DATA_DOG_API_KEY }}
     steps:
       - name: Collect bytesize metrics
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
-          name: turbopack-bytesize
+          pattern: turbopack-bytesize-*
+          merge-multiple: true
           path: turbopack-bin-size
 
       - name: Upload to Datadog

diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
@@ -64,13 +64,15 @@ jobs:
     uses: ./.github/workflows/build_reusable.yml
     with:
       skipInstallBuild: 'yes'
+      stepName: 'build-native'
     secrets: inherit
 
   build-next:
     name: build-next
     uses: ./.github/workflows/build_reusable.yml
     with:
       skipNativeBuild: 'yes'
+      stepName: 'build-next'
     secrets: inherit
 
   lint:
@@ -80,6 +82,7 @@ jobs:
     uses: ./.github/workflows/build_reusable.yml
     with:
       afterBuild: pnpm lint-no-typescript && pnpm check-examples
+      stepName: 'lint'
     secrets: inherit
 
   validate-docs-links:
@@ -103,6 +106,7 @@ jobs:
     uses: ./.github/workflows/build_reusable.yml
     with:
       afterBuild: pnpm types-and-precompiled
+      stepName: 'types-and-precompiled'
     secrets: inherit
 
   test-cargo-unit:
@@ -117,6 +121,7 @@ jobs:
       skipNativeBuild: 'yes'
       afterBuild: turbo run test-cargo-unit
       mold: 'yes'
+      stepName: 'test-cargo-unit'
     secrets: inherit
 
   rust-check:
@@ -130,6 +135,7 @@ jobs:
       skipInstallBuild: 'yes'
       skipNativeBuild: 'yes'
       afterBuild: turbo run rust-check
+      stepName: 'rust-check'
     secrets: inherit
 
   test-turbopack-dev:
@@ -144,6 +150,7 @@ jobs:
     uses: ./.github/workflows/build_reusable.yml
     with:
       afterBuild: RUST_BACKTRACE=0 NEXT_EXTERNAL_TESTS_FILTERS="$(pwd)/test/turbopack-tests-manifest.json" TURBOPACK=1 NEXT_E2E_TEST_TIMEOUT=240000 NEXT_TEST_MODE=dev node run-tests.js --test-pattern '^(test\/(development|e2e))/.*\.test\.(js|jsx|ts|tsx)$' --timings -g ${{ matrix.group }} -c ${TEST_CONCURRENCY}
+      stepName: 'test-turbopack-dev-${{ matrix.group }}'
     secrets: inherit
 
   test-turbopack-integration:
@@ -159,6 +166,7 @@ jobs:
     with:
       nodeVersion: 18.17.0
       afterBuild: RUST_BACKTRACE=0 NEXT_EXTERNAL_TESTS_FILTERS="$(pwd)/test/turbopack-tests-manifest.json" TURBOPACK=1 node run-tests.js --timings -g ${{ matrix.group }} -c ${TEST_CONCURRENCY} --type integration
+      stepName: 'test-turbopack-integration-${{ matrix.group }}'
     secrets: inherit
 
   test-next-swc-wasm:
@@ -169,6 +177,7 @@ jobs:
     uses: ./.github/workflows/build_reusable.yml
     with:
       afterBuild: rustup target add wasm32-unknown-unknown && curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh && node ./scripts/normalize-version-bump.js && turbo run build-wasm -- --target nodejs --features tracing/release_max_level_info && git checkout . && mv packages/next-swc/crates/wasm/pkg packages/next-swc/crates/wasm/pkg-nodejs && node ./scripts/setup-wasm.mjs && NEXT_TEST_MODE=start TEST_WASM=true node run-tests.js test/production/pages-dir/production/test/index.test.ts test/e2e/streaming-ssr/index.test.ts
+      stepName: 'test-next-swc-wasm'
     secrets: inherit
 
   test-unit:
@@ -185,6 +194,7 @@ jobs:
     with:
       nodeVersion: ${{ matrix.node }}
       afterBuild: node run-tests.js -c ${TEST_CONCURRENCY} --type unit
+      stepName: 'test-unit-${{ matrix.node }}'
 
     secrets: inherit
 
@@ -200,6 +210,7 @@ jobs:
     uses: ./.github/workflows/build_reusable.yml
     with:
       afterBuild: NEXT_TEST_MODE=dev node run-tests.js --timings -g ${{ matrix.group }} -c ${TEST_CONCURRENCY} --type development
+      stepName: 'test-dev-${{ matrix.group }}'
     secrets: inherit
 
   test-prod:
@@ -214,6 +225,7 @@ jobs:
     uses: ./.github/workflows/build_reusable.yml
     with:
       afterBuild: NEXT_TEST_MODE=start node run-tests.js --timings -g ${{ matrix.group }} -c ${TEST_CONCURRENCY} --type production
+      stepName: 'test-prod-${{ matrix.group }}'
     secrets: inherit
 
   test-integration:
@@ -241,6 +253,7 @@ jobs:
     with:
       nodeVersion: 18.17.0
       afterBuild: node run-tests.js --timings -g ${{ matrix.group }} -c ${TEST_CONCURRENCY} --type integration
+      stepName: 'test-integration-${{ matrix.group }}'
     secrets: inherit
 
   test-firefox-safari:
@@ -251,6 +264,7 @@ jobs:
     uses: ./.github/workflows/build_reusable.yml
     with:
       afterBuild: pnpm playwright install && BROWSER_NAME=firefox node run-tests.js test/production/pages-dir/production/test/index.test.ts && BROWSER_NAME=safari NEXT_TEST_MODE=start node run-tests.js -c 1 test/production/pages-dir/production/test/index.test.ts test/e2e/basepath.test.ts && BROWSER_NAME=safari DEVICE_NAME='iPhone XR' node run-tests.js -c 1 test/production/prerender-prefetch/index.test.ts
+      stepName: 'test-firefox-safari'
     secrets: inherit
 
   # TODO: remove these jobs once PPR is the default
@@ -266,6 +280,7 @@ jobs:
     with:
       nodeVersion: 18.17.0
       afterBuild: __NEXT_EXPERIMENTAL_PPR=true NEXT_EXTERNAL_TESTS_FILTERS="test/ppr-tests-manifest.json" node run-tests.js --timings -c ${TEST_CONCURRENCY} --type integration
+      stepName: 'test-ppr-integration'
     secrets: inherit
 
   test-ppr-dev:
@@ -280,6 +295,7 @@ jobs:
     uses: ./.github/workflows/build_reusable.yml
     with:
       afterBuild: __NEXT_EXPERIMENTAL_PPR=true NEXT_EXTERNAL_TESTS_FILTERS="test/ppr-tests-manifest.json" NEXT_TEST_MODE=dev node run-tests.js --timings -g ${{ matrix.group }} -c ${TEST_CONCURRENCY} --type development
+      stepName: 'test-ppr-dev-${{ matrix.group }}'
     secrets: inherit
 
   test-ppr-prod:
@@ -294,6 +310,7 @@ jobs:
     uses: ./.github/workflows/build_reusable.yml
     with:
       afterBuild: __NEXT_EXPERIMENTAL_PPR=true NEXT_EXTERNAL_TESTS_FILTERS="test/ppr-tests-manifest.json" NEXT_TEST_MODE=start node run-tests.js --timings -g ${{ matrix.group }} -c ${TEST_CONCURRENCY} --type production
+      stepName: 'test-ppr-prod-${{ matrix.group }}'
     secrets: inherit
 
   report-test-results:
@@ -317,10 +334,11 @@ jobs:
     steps:
       - name: Download test report artifacts
         id: download-test-reports
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
-          name: test-reports
+          pattern: test-reports-*
           path: test
+          merge-multiple: true
 
       - name: Upload test report to datadog
         run: |

diff --git a/.github/workflows/build_reusable.yml b/.github/workflows/build_reusable.yml
@@ -45,6 +45,10 @@ on:
         required: false
         description: 'rustCacheKey to cache shared target assets'
         type: string
+      stepName:
+        required: true
+        description: 'name of the step, to be used for the upload artifact unique key '
+        type: string
 
 env:
   NAPI_CLI_VERSION: 2.14.7
@@ -75,7 +79,17 @@ jobs:
       - 'x64'
       - 'metal'
 
+    outputs:
+      input_step_key: ${{ steps.var.outputs.input_step_key }}
+
     steps:
+      - name: Normalize input step names into path key
+        uses: actions/github-script@v6
+        id: var
+        with:
+          script: |
+            core.setOutput('input_step_key', '${{ inputs.stepName }}'.toLowerCase().replaceAll(/[/.]/g, '-').trim('-'));
+
       - run: fnm use --install-if-missing ${{ inputs.nodeVersion || env.NODE_LTS_VERSION }}
       - run: node -v
       - run: corepack enable
@@ -133,7 +147,7 @@ jobs:
 
       - name: Upload next-swc artifact
         if: ${{ inputs.uploadSwcArtifact == 'yes' }}
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: next-swc-binary
           path: packages/next-swc/native/next-swc.linux-x64-gnu.node
@@ -159,35 +173,35 @@ jobs:
       - run: /bin/bash -c "${{ inputs.afterBuild }}"
 
       - name: Upload artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: turbo run summary
+          name: turbo-run-summary-${{ steps.var.outputs.input_step_key }}
           path: .turbo/runs
           if-no-files-found: ignore
 
       - name: Upload bundle analyzer artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: ${{ inputs.uploadAnalyzerArtifacts == 'yes' }}
         with:
-          name: webpack bundle analysis stats
+          name: webpack bundle analysis stats-${{ steps.var.outputs.input_step_key }}
           path: packages/next/dist/compiled/next-server/report.*.html
 
       - name: Upload test report artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: ${{ inputs.afterBuild && always() }}
         with:
-          name: test-reports
+          name: test-reports-${{ steps.var.outputs.input_step_key }}
           path: |
             test/test-junit-report
             test/turbopack-test-junit-report
           if-no-files-found: ignore
 
       # upload playwright snapshots from failed tests
       - name: Upload test report artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: ${{ inputs.afterBuild && always() }}
         with:
-          name: test-playwright-snapshots
+          name: test-playwright-snapshots-${{ steps.var.outputs.input_step_key }}
           path: |
             test/traces
           if-no-files-found: ignore
diff --git a/.github/workflows/nextjs-integration-test.yml b/.github/workflows/nextjs-integration-test.yml
@@ -87,9 +87,9 @@ jobs:
         continue-on-error: true
 
       - name: Upload test report artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: test-reports
+          name: test-reports-dev-${{ matrix.group }}
           path: |
             test/turbopack-test-junit-report
 
@@ -131,9 +131,9 @@ jobs:
         continue-on-error: true
 
       - name: Upload test report artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: test-reports
+          name: test-reports-integration-${{ matrix.group }}
           path: |
             test/turbopack-test-junit-report
 
@@ -161,7 +161,7 @@ jobs:
           diff_base: ${{ inputs.diff_base }}
 
       - name: Store artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: test-results
           path: |
@@ -182,10 +182,11 @@ jobs:
     steps:
       - name: Download test report artifacts
         id: download-test-reports
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
-          name: test-reports
+          pattern: test-reports-*
           path: test/reports
+          merge-multiple: true
 
       - name: Upload to datadog
         env:

diff --git a/.github/workflows/pull_request_stats.yml b/.github/workflows/pull_request_stats.yml
@@ -25,6 +25,7 @@ jobs:
     uses: ./.github/workflows/build_reusable.yml
     secrets: inherit
     with:
+      stepName: 'generate-pull-request-stats'
       uploadSwcArtifact: 'yes'
       uploadAnalyzerArtifacts: 'yes'
 
@@ -46,7 +47,7 @@ jobs:
         run: echo "DOCS_CHANGE<<EOF" >> $GITHUB_OUTPUT; echo "$(node scripts/run-for-change.js --not --type docs --exec echo 'nope')" >> $GITHUB_OUTPUT; echo 'EOF' >> $GITHUB_OUTPUT
         id: docs-change
 
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         if: ${{ steps.docs-change.outputs.DOCS_CHANGE == 'nope' }}
         with:
           name: next-swc-binary

diff --git a/.github/workflows/setup-nextjs-build.yml b/.github/workflows/setup-nextjs-build.yml
@@ -58,7 +58,7 @@ jobs:
           path: integration-test-data
 
       - name: Download binary
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           path: artifacts