-
Notifications
You must be signed in to change notification settings - Fork 7
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Skylar Simoncelli
committed
Sep 19, 2024
1 parent
3b4c996
commit 284b8ba
Showing
2 changed files
with
183 additions
and
141 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
172 changes: 172 additions & 0 deletions
172
.github/workflows/modules/artifacts/earthly-build-and-push.yml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,172 @@ | ||
| name: Build and Push | ||
|
|
||
| on: | ||
| workflow_call: | ||
| inputs: | ||
| repository: | ||
| description: 'Repository to checkout' | ||
| required: true | ||
| type: string | ||
| branch: | ||
| description: 'Branch to checkout' | ||
| required: true | ||
| type: string | ||
| upload: | ||
| description: 'Whether to upload artifacts' | ||
| required: false | ||
| default: 'false' | ||
| type: string | ||
| secrets: | ||
| SUBSTRATE_REPO_SSH_KEY: | ||
| required: true | ||
| EARTHLY_TAR: | ||
| required: true | ||
| AWS_ROLE_ARN_SECRET: | ||
| required: true | ||
| ECR_REGISTRY_SECRET: | ||
| required: true | ||
| permissions: | ||
| id-token: write | ||
| contents: write | ||
|
|
||
| jobs: | ||
| build-and-push: | ||
| runs-on: ubuntu-latest | ||
| if: ${{ github.event.action != 'closed' || github.event.pull_request.merged == true }} | ||
| concurrency: | ||
| group: pr-${{ github.event.pull_request.number }}-author-${{ github.event.pull_request.user.login }} | ||
| cancel-in-progress: true | ||
| steps: | ||
| - name: Checkout | ||
| uses: actions/checkout@v4 | ||
| with: | ||
| repository: ${{ inputs.repository }} | ||
| ref: ${{ inputs.branch }} | ||
|
|
||
| - name: Setup Earthly | ||
| uses: ./.github/earthly-setup | ||
| if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci-off') }} | ||
| with: | ||
| ssh_key: ${{ secrets.SUBSTRATE_REPO_SSH_KEY }} | ||
| config_tar: ${{ secrets.EARTHLY_TAR }} | ||
|
|
||
| - name: Build With Benchmarking Features Enabled | ||
| if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci-off') && (github.ref_name == 'master' || inputs.upload == 'true') }} | ||
| env: | ||
| EARTHLY_CI: true | ||
| run: | | ||
| export EARTHLY_OUTPUT=true | ||
| earthly -P +build --PROFILE=production --FEATURES=runtime-benchmarks | ||
| - name: Generate and Extract Weights | ||
| if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci-off') && (github.ref_name == 'master' || inputs.upload == 'true') }} | ||
| continue-on-error: true | ||
| run: | | ||
| repository_name="${GITHUB_REPOSITORY##*/}" | ||
| echo "Listing contents on the runner host in /home/runner/work/${repository_name}/${repository_name}:" | ||
| ls -la /home/runner/work/${repository_name}/${repository_name} | ||
| echo "Pulling Docker image..." | ||
| docker pull ubuntu:22.04 | ||
| mkdir -p weights | ||
| echo "Running Docker container..." | ||
| docker run -d --name weight_generation \ | ||
| --memory=4096m \ | ||
| --cpus=1 \ | ||
| -v /home/runner/work/${repository_name}/${repository_name}:/workspace \ | ||
| ubuntu:22.04 \ | ||
| /bin/bash -c "sleep infinity" | ||
| echo "Installing necessary packages inside the container..." | ||
| docker exec weight_generation bash -c "\ | ||
| apt-get update && \ | ||
| apt-get install -y jq curl build-essential && \ | ||
| echo 'Checking files in workspace...' && \ | ||
| ls -la /workspace && \ | ||
| mkdir -p /workspace/target/production && \ | ||
| cp /workspace/sidechains-substrate-node /workspace/target/production/sidechains-substrate-node && \ | ||
| echo 'Verifying the binary is in the expected path...' && \ | ||
| ls -la /workspace/target/production && \ | ||
| cd /workspace && \ | ||
| echo 'Setting the current working directory to /workspace...' && \ | ||
| chmod +x scripts/run_all_pallet_overhead_and_machine_benchmarks.sh && \ | ||
| chmod +x scripts/run_storage_benchmarks.sh && \ | ||
| source .envrc || true && \ | ||
| ./scripts/run_all_pallet_overhead_and_machine_benchmarks.sh -b && \ | ||
| ./scripts/run_storage_benchmarks.sh -b || true" | ||
| echo "Finding and copying weight files..." | ||
| weight_files=$(docker exec weight_generation find /workspace/runtime/src/weights -name '*.rs') | ||
| echo "$weight_files" | while read weight_file; do | ||
| weight_file_name=$(basename "$weight_file") | ||
| echo "Copying ${weight_file_name}" | ||
| docker cp "weight_generation:$weight_file" "weights/${weight_file_name}" | ||
| done | ||
| docker stop weight_generation | ||
| docker rm weight_generation | ||
| - name: Upload Weights | ||
| if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci-off') && (github.ref_name == 'master' || inputs.upload == 'true') }} | ||
| continue-on-error: true | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: weights | ||
| path: weights/ | ||
|
|
||
| - name: Overwrite Weights in Runtime Directory | ||
| continue-on-error: true | ||
| if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci-off') && (github.ref_name == 'master' || inputs.upload == 'true') }} | ||
| run: | | ||
| pwd | ||
| ls -la | ||
| sudo chmod -R a+rwx ./runtime/src/weights | ||
| for weight_file in weights/*.rs | ||
| do | ||
| cp "$weight_file" "./runtime/src/weights/$(basename "$weight_file")" | ||
| done | ||
| - name: Acquire AWS credentials | ||
| uses: aws-actions/configure-aws-credentials@v4 | ||
| if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci-off') }} | ||
| with: | ||
| role-to-assume: ${{ secrets.AWS_ROLE_ARN_SECRET }} | ||
| aws-region: ${{ env.AWS_REGION }} | ||
|
|
||
| - name: Login to container registry | ||
| uses: docker/login-action@v3 | ||
| if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci-off') }} | ||
| with: | ||
| registry: ${{ secrets.ECR_REGISTRY_SECRET }} | ||
|
|
||
| - name: Main Build | ||
| env: | ||
| EARTHLY_CI: true | ||
| if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci-off') }} | ||
| run: | | ||
| ref=${{ github.ref_name }} | ||
| type=${{ github.ref_type }} | ||
| args=("--image=${{ secrets.ECR_REGISTRY_SECRET }}/substrate-node") | ||
| event_name="${{ github.event_name }}" | ||
| if [[ "$type" == "tag" && "$ref" =~ ^v ]]; then | ||
| export EARTHLY_PUSH=true | ||
| args+=("--tags=$ref") | ||
| elif [[ \ | ||
| "${{ github.event.pull_request.merged }}" == 'true' \ | ||
| && "$ref" == 'master' \ | ||
| || "${{ inputs.upload }}" == 'true' \ | ||
| ]]; then | ||
| export EARTHLY_PUSH=true | ||
| export EARTHLY_OUTPUT=true | ||
| fi | ||
| if [[ "$EARTHLY_PUSH" == true ]]; then | ||
| args+=(--PROFILE=production) | ||
| fi | ||
| earthly -P +ci ${args[@]} | ||
| - name: Upload chain spec artifacts | ||
| uses: actions/upload-artifact@v4 | ||
| if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci-off') && (github.ref_name == 'master' || inputs.upload == true) }} | ||
| with: | ||
| name: chain-specs | ||
| path: | | ||
| ./devnet_chain_spec.json | ||
| ./staging_chain_spec.json | ||
| ./staging_preview_chain_spec.json | ||
| ./staging_preprod_chain_spec.json |