divide to batches #14
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "*run replay-verify reusable workflow" | |
| on: | |
| # This allows the workflow to be triggered from another workflow | |
| workflow_call: | |
| inputs: | |
| GIT_SHA: | |
| required: true | |
| type: string | |
| description: The git SHA1 to test. | |
| # replay-verify config | |
| BUCKET: | |
| required: true | |
| type: string | |
| description: The bucket to use for the backup. If not specified, it will use the default bucket. | |
| SUB_DIR: | |
| required: true | |
| type: string | |
| description: The subdirectory to use for the backup. If not specified, it will use the default subdirectory. | |
| HISTORY_START: | |
| required: true | |
| type: string | |
| description: The history start to use for the backup. If not specified, it will use the default history start. | |
| TXNS_TO_SKIP: | |
| required: false | |
| type: string | |
| description: The list of transaction versions to skip. If not specified, it will use the default list. | |
| BACKUP_CONFIG_TEMPLATE_PATH: | |
| description: "The path to the backup config template to use." | |
| type: string | |
| required: true | |
| # GHA job config | |
| RUNS_ON: | |
| description: "The runner to use for the job." | |
| type: string | |
| required: true | |
| default: "runs-on,cpu=8,family=m6id,hdd=100,image=aptos-ubuntu-x64,spot=false" | |
| TIMEOUT_MINUTES: | |
| description: "Github job timeout in minutes" | |
| type: number | |
| required: true | |
| default: 180 | |
| # This allows the workflow to be triggered manually from the Github UI or CLI | |
| # NOTE: because the "number" type is not supported, we default to 720 minute timeout | |
| workflow_dispatch: | |
| inputs: | |
| GIT_SHA: | |
| required: true | |
| type: string | |
| description: The git SHA1 to test. | |
| # replay-verify config | |
| BUCKET: | |
| required: true | |
| type: string | |
| description: The bucket to use for the backup. If not specified, it will use the default bucket. | |
| SUB_DIR: | |
| required: true | |
| type: string | |
| description: The subdirectory to use for the backup. If not specified, it will use the default subdirectory. | |
| HISTORY_START: | |
| required: true | |
| type: string | |
| description: The history start to use for the backup. If not specified, it will use the default history start. | |
| TXNS_TO_SKIP: | |
| required: false | |
| type: string | |
| description: The list of transaction versions to skip. If not specified, it will use the default list. | |
| BACKUP_CONFIG_TEMPLATE_PATH: | |
| description: "The path to the backup config template to use." | |
| type: string | |
| required: true | |
| # GHA job config | |
| RUNS_ON: | |
| description: "The runner to use for the job." | |
| type: string | |
| required: true | |
| default: "runs-on,cpu=8,family=m6id,hdd=100,image=aptos-ubuntu-x64,spot=false" | |
| jobs: | |
| prepare: | |
| runs-on: ${{ inputs.RUNS_ON }} | |
| outputs: | |
| ranges0: ${{ steps.gen-jobs.outputs.ranges0 }} | |
| ranges1: ${{ steps.gen-jobs.outputs.ranges1 }} | |
| ranges2: ${{ steps.gen-jobs.outputs.ranges2 }} | |
| ranges3: ${{ steps.gen-jobs.outputs.ranges3 }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ inputs.GIT_SHA }} | |
| - name: Load cached aptos-debugger binary | |
| id: cache-aptos-debugger-binary | |
| uses: actions/cache@v4 | |
| with: | |
| # copy the binary to the root of the repo and cache it there, because rust-setup calls a cache-rust action | |
| # which cleans up the target directory in its post action | |
| path: aptos-debugger | |
| key: alden-hack-0914 #aptos-debugger-${{ inputs.GIT_SHA || github.sha }} | |
| - name: Prepare for build if not cached | |
| if: steps.cache-aptos-debugger-binary.outputs.cache-hit != 'true' | |
| uses: aptos-labs/aptos-core/.github/actions/rust-setup@main | |
| with: | |
| GIT_CREDENTIALS: ${{ inputs.GIT_CREDENTIALS }} | |
| - name: Build and strip aptos-debugger binary if not cached | |
| if: steps.cache-aptos-debugger-binary.outputs.cache-hit != 'true' | |
| shell: bash | |
| run: | | |
| cargo build --release -p aptos-debugger | |
| strip -s target/release/aptos-debugger | |
| cp target/release/aptos-debugger . | |
| - id: auth | |
| uses: "google-github-actions/auth@v2" | |
| with: | |
| workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} | |
| service_account: ${{ secrets.GCP_SERVICE_ACCOUNT_EMAIL }} | |
| - name: Install GCloud SDK | |
| uses: "google-github-actions/setup-gcloud@v2" | |
| with: | |
| version: ">= 418.0.0" | |
| install_components: "kubectl,gke-gcloud-auth-plugin" | |
| - name: get timestamp to use in cache key | |
| id: get-timestamp | |
| run: echo "ts=$(date +%s)" >> $GITHUB_OUTPUT | |
| - name: Load cached backup storage metadata cache dir (and save back afterwards) | |
| uses: actions/cache@v4 | |
| with: | |
| path: metadata_cache | |
| key: metadata-cache-${{ inputs.BUCKET }}/${{ inputs.SUB_DIR }}-${{ steps.get-timestamp.outputs.ts }} | |
| restore-keys: metadata-cache-${{ inputs.BUCKET }}/${{ inputs.SUB_DIR }}- | |
| - name: Generate job ranges | |
| id: gen-jobs | |
| env: | |
| BUCKET: ${{ inputs.BUCKET }} | |
| SUB_DIR: ${{ inputs.SUB_DIR }} | |
| HISTORY_START: ${{ inputs.HISTORY_START || '0' }} | |
| BACKUP_CONFIG_TEMPLATE_PATH: ${{ inputs.BACKUP_CONFIG_TEMPLATE_PATH }} | |
| run: | | |
| ./aptos-debugger aptos-db gen-replay-verify-jobs \ | |
| --metadata-cache-dir ./metadata_cache \ | |
| --command-adapter-config $BACKUP_CONFIG_TEMPLATE_PATH \ | |
| --start-version $HISTORY_START \ | |
| --output-json-file job_ranges.0.json | |
| --output-json-file job_ranges.1.json | |
| --output-json-file job_ranges.2.json | |
| --output-json-file job_ranges.3.json | |
| echo "ranges0=$(cat job_ranges.0.json)" >> $GITHUB_OUTPUT | |
| echo "ranges1=$(cat job_ranges.1.json)" >> $GITHUB_OUTPUT | |
| echo "ranges2=$(cat job_ranges.2.json)" >> $GITHUB_OUTPUT | |
| echo "ranges3=$(cat job_ranges.3.json)" >> $GITHUB_OUTPUT | |
| - name: Cache backup storage config so the replay jobs don't need to checkout entire repo | |
| uses: actions/cache/save@v4 | |
| with: | |
| path: ${{ inputs.BACKUP_CONFIG_TEMPLATE_PATH }} | |
| key: backup-config-${{ github.run_id }} | |
| replay-verify-batch: | |
| needs: prepare | |
| timeout-minutes: ${{ inputs.TIMEOUT_MINUTES || 180 }} | |
| runs-on: ${{ inputs.RUNS_ON }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| batch: [ | |
| "${{ steps.prepare.outputs.ranges0 }}", | |
| "${{ steps.prepare.outputs.ranges1 }}", | |
| "${{ steps.prepare.outputs.ranges2 }}", | |
| "${{ steps.prepare.outputs.ranges3 }}", | |
| ] | |
| steps: | |
| - name: Call replay-verify-batch workflow | |
| uses: aptos-labs/aptos-core/.github/workflows/workflow-run-replay-verify-batch.yaml@0911-alden-use-gcloud-on-base-image | |
| secrets: inherit | |
| with: | |
| RANGES_JSON: ${{ matrix.batch }} | |
| GIT_SHA: ${{ inputs.GIT_SHA }} | |
| BUCKET: ${{ inputs.BUCKET }} | |
| SUB_DIR: ${{ inputs.SUB_DIR }} | |
| HISTORY_START: ${{ inputs.HISTORY_START }} | |
| BACKUP_CONFIG_TEMPLATE_PATH: ${{ inputs.BACKUP_CONFIG_TEMPLATE_PATH }} | |
| RUNS_ON: ${{ inputs.RUNS_ON }} | |
| TIMEOUT_MINUTES: ${{ inputs.TIMEOUT_MINUTES }} |