Add option to upload Nova artifacts to S3 directly (#5668)

For the context, Nova jobs upload the artifacts to GitHub because this works on all runners including non-AWS ones like GitHub runners. However, there is a scale issue when trying to upload large artifacts (GB in size) to GitHub that the files are too big, so the upload could either: * Timing out https://github.com/pytorch/executorch/actions/runs/10858058150/job/30136354800 * or just fail in the middle of the upload https://github.com/pytorch/executorch/actions/runs/10856291106/job/30132545832 To address this, I add the option to upload to S3 gha-artifacts bucket here which is direct and more efficient.
pytorch · Sep 14, 2024 · ac26b29 · ac26b29
1 parent ad6b7e0
commit ac26b29
Show file tree

Hide file tree

Showing 6 changed files with 96 additions and 3 deletions.
diff --git a/.github/workflows/linux_job.yml b/.github/workflows/linux_job.yml
@@ -21,6 +21,13 @@ on:
           under dist/ and any files under artifacts-to-be-uploaded/ will be uploaded
         default: ''
         type: string
+      upload-artifact-to-s3:
+        description: |
+          Upload the artifact to S3 instead of GitHub. This is used for large artifacts like
+          exported model
+        required: false
+        default: false
+        type: boolean
       download-artifact:
         description: 'Name to download artifacts to ${RUNNER_ARTIFACT_DIR}'
         default: ''
@@ -292,14 +299,28 @@ jobs:
           fi
           echo "upload-docs=${upload_docs}" >> "${GITHUB_OUTPUT}"
 
+      # NB: Keep this for compatibility with existing jobs and also keep in mind that only
+      # our AWS runners have access to S3
       - name: Upload artifacts to GitHub (if any)
         uses: actions/upload-artifact@v3
-        if: ${{ always() && inputs.upload-artifact != '' }}
+        if: ${{ always() && inputs.upload-artifact != '' && !inputs.upload-artifact-to-s3 }}
         with:
           name: ${{ inputs.upload-artifact }}
           path: ${{ runner.temp }}/artifacts/
           if-no-files-found: ${{ steps.check-artifacts.outputs.if-no-files-found }}
 
+      # NB: This only works with our AWS runners
+      - name: Upload artifacts to S3 (if any)
+        uses: seemethere/upload-artifact-s3@v5
+        if: ${{ always() && inputs.upload-artifact != '' && inputs.upload-artifact-to-s3 }}
+        with:
+          retention-days: 14
+          s3-bucket: gha-artifacts
+          s3-prefix: |
+            ${{ env.REPOSITORY }}/${{ github.run_id }}/artifacts
+          if-no-files-found: ${{ steps.check-artifacts.outputs.if-no-files-found }}
+          path: ${{ runner.temp }}/artifacts/
+
       - name: Upload documentation to S3 (if any)
         uses: seemethere/upload-artifact-s3@v5
         if: ${{ steps.check-artifacts.outputs.upload-docs == 1 && github.event.pull_request.number != '' }}

diff --git a/.github/workflows/macos_job.yml b/.github/workflows/macos_job.yml
@@ -21,6 +21,13 @@ on:
           under dist/ and any files under artifacts-to-be-uploaded/ will be uploaded
         default: ""
         type: string
+      upload-artifact-to-s3:
+        description: |
+          Upload the artifact to S3 instead of GitHub. This is used for large artifacts like
+          exported model
+        required: false
+        default: false
+        type: boolean
       download-artifact:
         description: 'Name to download artifacts to ${RUNNER_ARTIFACT_DIR}'
         default: ""
@@ -183,14 +190,28 @@ jobs:
           # Set to fail upload step if there are no files for upload and expected files for upload
           echo 'if-no-files-found=error' >> "${GITHUB_OUTPUT}"
 
+      # NB: Keep this for compatibility with existing jobs and also keep in mind that only
+      # our AWS runners have access to S3
       - name: Upload artifacts to GitHub (if any)
         uses: actions/upload-artifact@v3
-        if: ${{ always() && inputs.upload-artifact != '' }}
+        if: ${{ always() && inputs.upload-artifact != '' && !inputs.upload-artifact-to-s3 }}
         with:
           name: ${{ inputs.upload-artifact }}
           path: ${{ runner.temp }}/artifacts/
           if-no-files-found: ${{ steps.check-artifacts.outputs.if-no-files-found }}
 
+      # NB: This only works with our AWS runners
+      - name: Upload artifacts to S3 (if any)
+        uses: seemethere/upload-artifact-s3@v5
+        if: ${{ always() && inputs.upload-artifact != '' && inputs.upload-artifact-to-s3 }}
+        with:
+          retention-days: 14
+          s3-bucket: gha-artifacts
+          s3-prefix: |
+            ${{ env.REPOSITORY }}/${{ github.run_id }}/artifacts
+          if-no-files-found: ${{ steps.check-artifacts.outputs.if-no-files-found }}
+          path: ${{ runner.temp }}/artifacts/
+
       - name: Clean up disk space
         if: always()
         continue-on-error: true

diff --git a/.github/workflows/test_linux_job.yml b/.github/workflows/test_linux_job.yml
@@ -93,6 +93,16 @@ jobs:
       upload-artifact: my-cool-artifact
       script: |
         echo "hello" > "${RUNNER_ARTIFACT_DIR}/cool_beans"
+  test-upload-artifact-s3:
+    uses: ./.github/workflows/linux_job.yml
+    with:
+      runner: linux.2xlarge
+      test-infra-repository: ${{ github.repository }}
+      test-infra-ref: ${{ github.ref }}
+      upload-artifact: my-cool-artifact
+      upload-artifact-to-s3: true
+      script: |
+        echo "hello" > "${RUNNER_ARTIFACT_DIR}/cool_beans"
   test-download-artifact:
     needs: test-upload-artifact
     uses: ./.github/workflows/linux_job.yml

diff --git a/.github/workflows/test_macos_job.yml b/.github/workflows/test_macos_job.yml
@@ -78,6 +78,16 @@ jobs:
       upload-artifact: my-cool-artifact
       script: |
         echo "hello" > "${RUNNER_ARTIFACT_DIR}/cool_beans"
+  test-upload-artifact-s3:
+    uses: ./.github/workflows/macos_job.yml
+    with:
+      runner: macos-m1-stable
+      test-infra-repository: ${{ github.repository }}
+      test-infra-ref: ${{ github.ref }}
+      upload-artifact: my-cool-artifact
+      upload-artifact-to-s3: true
+      script: |
+        echo "hello" > "${RUNNER_ARTIFACT_DIR}/cool_beans"
   test-download-artifact:
     needs: test-upload-artifact
     uses: ./.github/workflows/macos_job.yml

diff --git a/.github/workflows/test_windows_job.yml b/.github/workflows/test_windows_job.yml
@@ -46,6 +46,16 @@ jobs:
       upload-artifact: my-cool-artifact
       script: |
         echo "hello" > "${RUNNER_ARTIFACT_DIR}/cool_beans"
+  test-upload-artifact-s3:
+    uses: ./.github/workflows/windows_job.yml
+    with:
+      runner: windows.4xlarge
+      test-infra-repository: ${{ github.repository }}
+      test-infra-ref: ${{ github.ref }}
+      upload-artifact: my-cool-artifact
+      upload-artifact-to-s3: true
+      script: |
+        echo "hello" > "${RUNNER_ARTIFACT_DIR}/cool_beans"
   test-download-artifact:
     needs: test-upload-artifact
     uses: ./.github/workflows/windows_job.yml

diff --git a/.github/workflows/windows_job.yml b/.github/workflows/windows_job.yml
@@ -21,6 +21,13 @@ on:
           under dist/ and any files under artifacts-to-be-uploaded/ will be uploaded
         default: ''
         type: string
+      upload-artifact-to-s3:
+        description: |
+          Upload the artifact to S3 instead of GitHub. This is used for large artifacts like
+          exported model
+        required: false
+        default: false
+        type: boolean
       download-artifact:
         description: 'Name to download artifacts to ${RUNNER_ARTIFACT_DIR}'
         default: ''
@@ -185,14 +192,28 @@ jobs:
           # Set to fail upload step if there are no files for upload and expected files for upload
           echo 'if-no-files-found=error' >> "${GITHUB_OUTPUT}"
 
+      # NB: Keep this for compatibility with existing jobs and also keep in mind that only
+      # our AWS runners have access to S3
       - name: Upload artifacts to GitHub (if any)
         uses: actions/upload-artifact@v3
-        if: ${{ always() && inputs.upload-artifact != '' }}
+        if: ${{ always() && inputs.upload-artifact != '' && !inputs.upload-artifact-to-s3 }}
         with:
           name: ${{ inputs.upload-artifact }}
           path: ${{ runner.temp }}/artifacts/
           if-no-files-found: ${{ steps.check-artifacts.outputs.if-no-files-found }}
 
+      # NB: This only works with our AWS runners
+      - name: Upload artifacts to S3 (if any)
+        uses: seemethere/upload-artifact-s3@v5
+        if: ${{ always() && inputs.upload-artifact != '' && inputs.upload-artifact-to-s3 }}
+        with:
+          retention-days: 14
+          s3-bucket: gha-artifacts
+          s3-prefix: |
+            ${{ env.REPOSITORY }}/${{ github.run_id }}/artifacts
+          if-no-files-found: ${{ steps.check-artifacts.outputs.if-no-files-found }}
+          path: ${{ runner.temp }}/artifacts/
+
       - name: Teardown Windows
         if: ${{ always() }}
         uses: ./test-infra/.github/actions/teardown-windows