ref(workflows): consolidate workflows based on their purpose (#7616)

* ref(workflows): consolidate workflows based on their purpose This also renames the workflows to make their naming more consistent and adding a naming convention Fixes: #6166 Fixes: #6167 * fix(workflows): use correct name for patch * fix(workflow): docker unit tests * fix(release): validation error Error: ``` Validation Failed: {"resource":"Release","code":"invalid","field":"target_commitish"} ``` Fixes: release-drafter/release-drafter#1125 * fix(workflows): reference correct name * fix: remove extra workflow * fix(workflows): use larger runners * fix(workflow): remove code already in docker unit-test * fix(unit-tests): start zebra the right way * fix: typo in patch name * chore: move job to logical order * imp(workflows): use better name for gcp tests * add: missing merge changes * chore: use better name for find-disks * fix(ci): use the `entrypoint.sh` to change the Network * fix(ci): add missing `ZEBRA_CONF_PATH` variable * fix(ci): allow to build the entrypoint file with testnet * fix(entrypoint): allow to create a dir and file with a single variable * refactor: test config file in CI and CD with a reusable workflow * fix(ci): wrong name used * fix(ci): use checkout * fix(ci): improve docker config tests * fix(ci): use better name for protection rules * Fix changed workflow file name in docs patch file * Apply suggestions from code review Co-authored-by: teor <teor@riseup.net> Co-authored-by: Arya <aryasolhi@gmail.com> * fix(cd): depend on file tests * fix(docs): adapt to new workflow name * fix: revert test coverage on CD * chore: reduce diff * fix(ci): allow using variable images for reusable workflows * fix(dockerfile): use variables or default for config path and file * fix(entrypoint): if `$ZEBRA_CONF_PATH` is set, do not override it * Fix patch job names and remove failure job testnet dependencies --------- Co-authored-by: teor <teor@riseup.net> Co-authored-by: Arya <aryasolhi@gmail.com>
ZcashFoundation · Oct 18, 2023 · fc0133e · fc0133e
1 parent 08ce2ad
commit fc0133e
Show file tree

Hide file tree

Showing 34 changed files with 595 additions and 477 deletions.
diff --git a/.github/workflows/cd-deploy-nodes-gcp.patch.yml b/.github/workflows/cd-deploy-nodes-gcp.patch.yml
@@ -0,0 +1,39 @@
+name: Deploy Nodes to GCP
+
+on:
+  pull_request:
+   paths-ignore:
+     # code and tests
+     - '**/*.rs'
+     # hard-coded checkpoints and proptest regressions
+     - '**/*.txt'
+     # dependencies
+     - '**/Cargo.toml'
+     - '**/Cargo.lock'
+     # configuration files
+     - '.cargo/config.toml'
+     - '**/clippy.toml'
+     # workflow definitions
+     - 'docker/**'
+     - '.dockerignore'
+     - '.github/workflows/cd-deploy-nodes-gcp.yml'
+     - '.github/workflows/sub-build-docker-image.yml'
+
+jobs:
+  build:
+    name: Build CD Docker / Build images
+    runs-on: ubuntu-latest
+    steps:
+      - run: 'echo "No build required"'
+
+  test-configuration-file:
+    name: Test CD default Docker config file / Test default-conf in Docker
+    runs-on: ubuntu-latest
+    steps:
+      - run: 'echo "No build required"'
+
+  test-zebra-conf-path:
+    name: Test CD custom Docker config file / Test custom-conf in Docker
+    runs-on: ubuntu-latest
+    steps:
+      - run: 'echo "No build required"'
diff --git a/.github/workflows/continous-delivery.yml → .github/workflows/cd-deploy-nodes-gcp.yml b/.github/workflows/continous-delivery.yml → .github/workflows/cd-deploy-nodes-gcp.yml
@@ -1,4 +1,4 @@
-name: CD
+name: Deploy Nodes to GCP
 
 # Ensures that only one workflow task will run at a time. Previous deployments, if
 # already in process, won't get cancelled. Instead, we let the first to complete
@@ -47,8 +47,8 @@ on:
   #    # workflow definitions
   #    - 'docker/**'
   #    - '.dockerignore'
-  #    - '.github/workflows/continous-delivery.yml'
-  #    - '.github/workflows/build-docker-image.yml'
+  #    - '.github/workflows/cd-deploy-nodes-gcp.yml'
+  #    - '.github/workflows/sub-build-docker-image.yml'
 
   # Only runs the Docker image tests, doesn't deploy any instances
   pull_request:
@@ -66,14 +66,13 @@ on:
       # workflow definitions
       - 'docker/**'
       - '.dockerignore'
-      - '.github/workflows/continous-delivery.yml'
-      - '.github/workflows/find-cached-disks.yml'
+      - '.github/workflows/cd-deploy-nodes-gcp.yml'
+      - '.github/workflows/sub-build-docker-image.yml'
 
   release:
     types:
       - published
 
-
 jobs:
   # If a release was made we want to extract the first part of the semver from the
   # tag_name
@@ -108,7 +107,7 @@ jobs:
   # The image will be commonly named `zebrad:<short-hash | github-ref | semver>`
   build:
     name: Build CD Docker
-    uses: ./.github/workflows/build-docker-image.yml
+    uses: ./.github/workflows/sub-build-docker-image.yml
     with:
       dockerfile_path: ./docker/Dockerfile
       dockerfile_target: runtime
@@ -118,99 +117,40 @@ jobs:
 
   # Test that Zebra works using the default config with the latest Zebra version.
   test-configuration-file:
-    name: Test Zebra CD Docker config file
-    timeout-minutes: 15
-    runs-on: ubuntu-latest
+    name: Test CD default Docker config file
     needs: build
-    steps:
-      - uses: r7kamura/rust-problem-matchers@v1.4.0
-
-      - name: Inject slug/short variables
-        uses: rlespinasse/github-slug-action@v4
-        with:
-          short-length: 7
-
-      # Make sure Zebra can sync at least one full checkpoint on mainnet
-      - name: Run tests using the default config
-        shell: /usr/bin/bash -exo pipefail {0}
-        run: |
-          docker pull ${{ vars.GAR_BASE }}/zebrad@${{ needs.build.outputs.image_digest }}
-          docker run --detach --name default-conf-tests -t ${{ vars.GAR_BASE }}/zebrad@${{ needs.build.outputs.image_digest }}
-
-          # Use a subshell to handle the broken pipe error gracefully
-          (
-            trap "" PIPE;
-            docker logs \
-            --tail all \
-            --follow \
-            default-conf-tests | \
-            tee --output-error=exit /dev/stderr | \
-            grep --max-count=1 --extended-regexp --color=always \
-            -e "net.*=.*Main.*estimated progress to chain tip.*BeforeOverwinter"
-          ) || true
-          LOGS_EXIT_STATUS=$?
-
-          docker stop default-conf-tests
-
-          EXIT_STATUS=$(docker wait default-conf-tests || echo "Error retrieving exit status");
-          echo "docker exit status: $EXIT_STATUS";
-
-          # If grep found the pattern, exit with the Docker container exit status
-          if [ $LOGS_EXIT_STATUS -eq 0 ]; then
-              exit $EXIT_STATUS;
-          fi
-
-          # Handle other potential errors here
-          echo "An error occurred while processing the logs.";
-          exit 1;
+    uses: ./.github/workflows/sub-test-zebra-config.yml
+    with:
+      test_id: 'default-conf'
+      docker_image: ${{ vars.GAR_BASE }}/zebrad@${{ needs.build.outputs.image_digest }}
+      grep_patterns: '-e "net.*=.*Main.*estimated progress to chain tip.*BeforeOverwinter"'
+      test_variables: '-e NETWORK'
+      network: 'Mainnet'
 
   # Test reconfiguring the docker image for testnet.
   test-configuration-file-testnet:
-    name: Test testnet Zebra CD Docker config file
-    timeout-minutes: 15
-    runs-on: ubuntu-latest
+    name: Test CD testnet Docker config file
     needs: build
-    steps:
-      - uses: r7kamura/rust-problem-matchers@v1.4.0
-
-      - name: Inject slug/short variables
-        uses: rlespinasse/github-slug-action@v4
-        with:
-          short-length: 7
-
-      # Make sure Zebra can sync the genesis block on testnet
-      - name: Run tests using a testnet config
-        shell: /usr/bin/bash -exo pipefail {0}
-        run: |
-          docker pull ${{ vars.GAR_BASE }}/zebrad@${{ needs.build.outputs.image_digest }}
-          docker run --env "NETWORK=Testnet" --detach --name testnet-conf-tests -t ${{ vars.GAR_BASE }}/zebrad@${{ needs.build.outputs.image_digest }}
-          # Use a subshell to handle the broken pipe error gracefully
-          (
-            trap "" PIPE;
-            docker logs \
-            --tail all \
-            --follow \
-            testnet-conf-tests | \
-            tee --output-error=exit /dev/stderr | \
-            grep --max-count=1 --extended-regexp --color=always \
-            -e "net.*=.*Test.*estimated progress to chain tip.*Genesis" \
-            -e "net.*=.*Test.*estimated progress to chain tip.*BeforeOverwinter";
-          ) || true
-          LOGS_EXIT_STATUS=$?
-
-          docker stop testnet-conf-tests
-
-          EXIT_STATUS=$(docker wait testnet-conf-tests || echo "Error retrieving exit status");
-          echo "docker exit status: $EXIT_STATUS";
-
-          # If grep found the pattern, exit with the Docker container exit status
-          if [ $LOGS_EXIT_STATUS -eq 0 ]; then
-              exit $EXIT_STATUS;
-          fi
-
-          # Handle other potential errors here
-          echo "An error occurred while processing the logs.";
-          exit 1;
+    # Make sure Zebra can sync the genesis block on testnet
+    uses: ./.github/workflows/sub-test-zebra-config.yml
+    with:
+      test_id: 'testnet-conf'
+      docker_image: ${{ vars.GAR_BASE }}/zebrad@${{ needs.build.outputs.image_digest }}
+      grep_patterns: '-e "net.*=.*Test.*estimated progress to chain tip.*Genesis" -e "net.*=.*Test.*estimated progress to chain tip.*BeforeOverwinter"'
+      test_variables: '-e NETWORK'
+      network: 'Testnet'
+
+  # Test that Zebra works using $ZEBRA_CONF_PATH config
+  test-zebra-conf-path:
+    name: Test CD custom Docker config file
+    needs: build
+    uses: ./.github/workflows/sub-test-zebra-config.yml
+    with:
+      test_id: 'custom-conf'
+      docker_image: ${{ vars.GAR_BASE }}/zebrad@${{ needs.build.outputs.image_digest }}
+      grep_patterns: '-e "v1.0.0-rc.2.toml"'
+      test_variables: '-e NETWORK -e ZEBRA_CONF_PATH="zebrad/tests/common/configs/v1.0.0-rc.2.toml"'
+      network: ${{ inputs.network || vars.ZCASH_NETWORK }}
 
   # Deploy Managed Instance Groups (MiGs) for Mainnet and Testnet,
   # with one node in the configured GCP region.
@@ -225,14 +165,14 @@ jobs:
   #   otherwise a new major version is deployed in a new MiG.
   #
   # Runs:
-  # - on every push/merge to the `main` branch
+  # - on every push to the `main` branch
   # - on every release, when it's published
   deploy-nodes:
     strategy:
       matrix:
         network: [Mainnet, Testnet]
     name: Deploy ${{ matrix.network }} nodes
-    needs: [ build, test-configuration-file, versioning ]
+    needs: [ build, versioning, test-configuration-file, test-zebra-conf-path ]
     runs-on: ubuntu-latest
     timeout-minutes: 60
     permissions:
@@ -334,7 +274,7 @@ jobs:
   # Note: this instances are not automatically replaced or deleted
   deploy-instance:
     name: Deploy single ${{ inputs.network }} instance
-    needs: [ build, test-configuration-file ]
+    needs: [ build, test-configuration-file, test-zebra-conf-path ]
     runs-on: ubuntu-latest
     timeout-minutes: 30
     permissions:
@@ -401,7 +341,7 @@ jobs:
   failure-issue:
     name: Open or update issues for release failures
     # When a new job is added to this workflow, add it to this list.
-    needs: [ versioning, build,  test-configuration-file, deploy-nodes, deploy-instance ]
+    needs: [ versioning, build, deploy-nodes, deploy-instance ]
     # Only open tickets for failed or cancelled jobs that are not coming from PRs.
     # (PR statuses are already reported in the PR jobs list, and checked by Mergify.)
     if: (failure() && github.event.pull_request == null) || (cancelled() && github.event.pull_request == null)

diff --git a/.github/workflows/delete-gcp-resources.yml → .../workflows/chore-delete-gcp-resources.yml b/.github/workflows/delete-gcp-resources.yml → .../workflows/chore-delete-gcp-resources.yml
diff --git a/.github/workflows/project-management.yml → ...ub/workflows/chore-project-management.yml b/.github/workflows/project-management.yml → ...ub/workflows/chore-project-management.yml
diff --git a/...flows/build-crates-individually.patch.yml → .github/workflows/ci-build-crates.patch.yml b/...flows/build-crates-individually.patch.yml → .github/workflows/ci-build-crates.patch.yml
@@ -14,7 +14,7 @@ on:
       - '.cargo/config.toml'
       - '**/clippy.toml'
       # workflow definitions
-      - '.github/workflows/build-crates-individually.yml'
+      - '.github/workflows/ci-build-crates.yml'
 
 jobs:
   matrix:

diff --git a/...b/workflows/build-crates-individually.yml → .github/workflows/ci-build-crates.yml b/...b/workflows/build-crates-individually.yml → .github/workflows/ci-build-crates.yml
@@ -22,7 +22,7 @@ on:
       - '.cargo/config.toml'
       - '**/clippy.toml'
       # workflow definitions
-      - '.github/workflows/build-crates-individually.yml'
+      - '.github/workflows/ci-build-crates.yml'
   pull_request:
     paths:
       # production code and test code
@@ -34,7 +34,7 @@ on:
       - '.cargo/config.toml'
       - '**/clippy.toml'
       # workflow definitions
-      - '.github/workflows/build-crates-individually.yml'
+      - '.github/workflows/ci-build-crates.yml'
 
 env:
   CARGO_INCREMENTAL: ${{ vars.CARGO_INCREMENTAL }}

diff --git a/.github/workflows/coverage.patch.yml → .github/workflows/ci-coverage.patch.yml b/.github/workflows/coverage.patch.yml → .github/workflows/ci-coverage.patch.yml
@@ -12,7 +12,7 @@ on:
       - '.cargo/config.toml'
       - '**/clippy.toml'
       - 'codecov.yml'
-      - '.github/workflows/coverage.yml'
+      - '.github/workflows/ci-coverage.yml'
 
 jobs:
   coverage:

diff --git a/.github/workflows/coverage.yml → .github/workflows/ci-coverage.yml b/.github/workflows/coverage.yml → .github/workflows/ci-coverage.yml
@@ -27,7 +27,7 @@ on:
       - '**/clippy.toml'
       # workflow definitions
       - 'codecov.yml'
-      - '.github/workflows/coverage.yml'
+      - '.github/workflows/ci-coverage.yml'
   pull_request:
     paths:
       - '**/*.rs'
@@ -38,7 +38,7 @@ on:
       - '.cargo/config.toml'
       - '**/clippy.toml'
       - 'codecov.yml'
-      - '.github/workflows/coverage.yml'
+      - '.github/workflows/ci-coverage.yml'
 
 env:
   CARGO_INCREMENTAL: ${{ vars.CARGO_INCREMENTAL }}

diff --git a/...ws/continous-integration-docker.patch.yml → ...kflows/ci-integration-tests-gcp.patch.yml b/...ws/continous-integration-docker.patch.yml → ...kflows/ci-integration-tests-gcp.patch.yml
@@ -1,4 +1,4 @@
-name: CI Docker
+name: Integration Tests on GCP
 
 # These jobs *don't* depend on cached Google Cloud state disks,
 # so they can be skipped when the modified files make the actual workflow run.
@@ -20,10 +20,10 @@ on:
       # workflow definitions
       - 'docker/**'
       - '.dockerignore'
-      - '.github/workflows/continous-integration-docker.yml'
-      - '.github/workflows/deploy-gcp-tests.yml'
-      - '.github/workflows/find-cached-disks.yml'
-      - '.github/workflows/build-docker-image.yml'
+      - '.github/workflows/ci-unit-tests-docker.yml'
+      - '.github/workflows/sub-deploy-integration-tests-gcp.yml'
+      - '.github/workflows/sub-find-cached-disks.yml'
+      - '.github/workflows/sub-build-docker-image.yml'
 
 jobs:
   # We don't patch the testnet job, because testnet isn't required to merge (it's too unstable)
@@ -39,48 +39,6 @@ jobs:
     steps:
       - run: 'echo "No build required"'
 
-  test-all:
-    name: Test all
-    runs-on: ubuntu-latest
-    steps:
-      - run: 'echo "No build required"'
-
-  test-all-getblocktemplate-rpcs:
-    name: Test all with getblocktemplate-rpcs feature
-    runs-on: ubuntu-latest
-    steps:
-      - run: 'echo "No build required"'
-
-  test-fake-activation-heights:
-    name: Test with fake activation heights
-    runs-on: ubuntu-latest
-    steps:
-      - run: 'echo "No build required"'
-
-  test-empty-sync:
-    name: Test checkpoint sync from empty state
-    runs-on: ubuntu-latest
-    steps:
-      - run: 'echo "No build required"'
-
-  test-lightwalletd-integration:
-    name: Test integration with lightwalletd
-    runs-on: ubuntu-latest
-    steps:
-      - run: 'echo "No build required"'
-
-  test-configuration-file:
-    name: Test Zebra default Docker config file
-    runs-on: ubuntu-latest
-    steps:
-      - run: 'echo "No build required"'
-
-  test-zebra-conf-path:
-    name: Test Zebra custom Docker config file
-    runs-on: ubuntu-latest
-    steps:
-      - run: 'echo "No build required"'
-
   test-stateful-sync:
     name: Zebra checkpoint update / Run sync-past-checkpoint test
     runs-on: ubuntu-latest