Merge branch 'main' into send-trees-to-from-nonfin-to-fin-state

.github/mergify.yml

@@ -5,7 +5,7 @@ queue_rules:
     allow_inplace_checks: True
     allow_checks_interruption: False
     speculative_checks: 1
-    batch_size: 5
+    batch_size: 8
     # Wait a short time to embark hotfixes together in a merge train
     batch_max_wait_time: "2 minutes"
     conditions:
@@ -18,19 +18,19 @@ queue_rules:
     allow_inplace_checks: True
     allow_checks_interruption: True
     speculative_checks: 1
-    batch_size: 5
+    batch_size: 20
     # Wait for a few minutes to embark high priority tickets together in a merge train
-    batch_max_wait_time: "5 minutes"
+    batch_max_wait_time: "10 minutes"
     conditions:
       - base=main
 
   - name: low
     allow_inplace_checks: True
     allow_checks_interruption: True
     speculative_checks: 1
-    batch_size: 5
+    batch_size: 20
     # Wait a bit longer to embark low priority tickets together in a merge train
-    batch_max_wait_time: "10 minutes"
+    batch_max_wait_time: "20 minutes"
     conditions:
       - base=main
 
@@ -85,4 +85,4 @@ pull_request_rules:
     actions:
       queue:
         name: low
-        method: squash
+        method: squash

.github/workflows/build-crates-individually.patch.yml

@@ -10,6 +10,9 @@ on:
       # dependencies
       - '**/Cargo.toml'
       - '**/Cargo.lock'
+      # configuration files
+      - '.cargo/config.toml'
+      - '**/clippy.toml'
       # workflow definitions
       - '.github/workflows/build-crates-individually.yml'
 

.github/workflows/build-crates-individually.yml

@@ -1,5 +1,12 @@
 name: Build crates individually
 
+# Ensures that only one workflow task will run at a time. Previous builds, if
+# already in process, will get cancelled. Only the latest commit will be allowed
+# to run, cancelling any workflows in between
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
 on:
   workflow_dispatch:
   push:
@@ -11,6 +18,9 @@ on:
       # dependencies
       - '**/Cargo.toml'
       - '**/Cargo.lock'
+      # configuration files
+      - '.cargo/config.toml'
+      - '**/clippy.toml'
       # workflow definitions
       - '.github/workflows/build-crates-individually.yml'
   pull_request:
@@ -20,6 +30,9 @@ on:
       # dependencies
       - '**/Cargo.toml'
       - '**/Cargo.lock'
+      # configuration files
+      - '.cargo/config.toml'
+      - '**/clippy.toml'
       # workflow definitions
       - '.github/workflows/build-crates-individually.yml'
 

.github/workflows/build-docker-image.yml

@@ -37,12 +37,19 @@ on:
         required: false
         type: string
         default: info
+    outputs:
+      image_digest:
+        description: 'The image digest to be used on a caller workflow'
+        value: ${{ jobs.build.outputs.image_digest }}
 
 jobs:
   build:
     name: Build images
     timeout-minutes: 210
     runs-on: ubuntu-latest
+    outputs:
+      image_digest: ${{ steps.docker_build.outputs.digest }}
+      image_name: ${{ fromJSON(steps.docker_build.outputs.metadata)['image.name'] }}
     permissions:
       contents: 'read'
       id-token: 'write'
@@ -67,12 +74,12 @@ jobs:
           # generate Docker tags based on the following events/attributes
           tags: |
             type=schedule
+            type=sha
             type=ref,event=branch
             type=ref,event=pr
             type=semver,pattern={{version}}
             type=semver,pattern={{major}}.{{minor}}
             type=semver,pattern={{major}}
-            type=sha
 
       # Setup Docker Buildx to allow use of docker cache layers from GH
       - name: Set up Docker Buildx

.github/workflows/continous-delivery.yml

@@ -1,5 +1,12 @@
 name: CD
 
+# Ensures that only one workflow task will run at a time. Previous deployments, if
+# already in process, won't get cancelled. Instead, we let the first to complete
+# then queue the latest pending workflow, cancelling any workflows in between
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: false
+
 on:
   workflow_dispatch:
     inputs:
@@ -14,6 +21,9 @@ on:
   push:
     branches:
       - main
+  release:
+    types:
+      - published
 
 env:
   NETWORK: Mainnet
@@ -23,6 +33,37 @@ env:
   MACHINE_TYPE: c2-standard-4
 
 jobs:
+  # If a release was made we want to extract the first part of the semver from the
+  # tag_name
+  #
+  # Generate the following output to pass to subsequent jobs
+  # - If our semver is `v1.3.0` the resulting output from this job would be `v1`
+  #
+  # Note: We just use the first part of the version to replace old instances, and change
+  # it when a major version is released, to keep a segregation between new and old 
+  # versions.
+  versioning:
+    name: Versioning
+    runs-on: ubuntu-latest
+    outputs:
+      major_version: ${{ steps.set.outputs.major_version }}
+    if: ${{ github.event_name == 'release' }}
+    steps:
+      - name: Getting Zebrad Version
+        id: get
+        uses: actions/github-script@v6.2.0
+        with:
+          result-encoding: string
+          script: |
+            return context.payload.release.tag_name.substring(0,2)
+      - name: Setting API Version
+        id: set
+        run: echo "::set-output name=major_version::${{ steps.get.outputs.result }}"
+
+  # Each time this workflow is executed, a build will be triggered to create a new image
+  # with the corresponding tags using information from Git
+  #
+  # The image will be commonly named `zebrad:<short-hash | github-ref | semver>`
   build:
     uses: ./.github/workflows/build-docker-image.yml
     with:
@@ -35,15 +76,26 @@ jobs:
       zebra_skip_ipv6_tests: '1'
       rust_log: info
 
+  # This jobs handles the deployment of a Managed Instance Group (MiG) with 2 nodes in
+  # the us-central1 region. Two different groups of MiGs are deployed one for pushes to
+  # the main branch and another for version releases of Zebra
+  #
+  # Once this workflow is triggered the previous MiG is replaced, on pushes to main its
+  # always replaced, and with releases its only replaced if the same major version is
+  # being deployed, otherwise a new major version is deployed
+  #
+  # Runs:
+  # - on every push/merge to the `main` branch
+  # - on every release, when it's published
   deploy-nodes:
     name: Deploy Mainnet nodes
-    needs: build
+    needs: [ build, versioning ]
     runs-on: ubuntu-latest
     timeout-minutes: 30
     permissions:
       contents: 'read'
       id-token: 'write'
-    if: ${{ github.event_name == 'push' && github.ref_name == 'main' }}
+    if: ${{ !cancelled() && !failure() && ((github.event_name == 'push' && github.ref_name == 'main') || github.event_name == 'release') }}
 
     steps:
       - name: Inject slug/short variables
@@ -63,9 +115,9 @@ jobs:
 
       - name: Create instance template
         run: |
-          gcloud compute instance-templates create-with-container zebrad-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \
+          gcloud compute instance-templates create-with-container zebrad-${{ needs.versioning.outputs.major_version || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \
           --boot-disk-type=pd-ssd \
-          --container-image ${{ env.GAR_BASE }}/${{ env.GITHUB_REF_SLUG_URL }}:${{ env.GITHUB_SHA_SHORT }} \
+          --container-image ${{ env.GAR_BASE }}/zebrad@${{ needs.build.outputs.image_digest }} \
           --create-disk name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }},auto-delete=yes,size=100GB,type=pd-ssd \
           --container-mount-disk mount-path="/zebrad-cache",name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }} \
           --machine-type ${{ env.MACHINE_TYPE }} \
@@ -77,15 +129,15 @@ jobs:
         id: does-group-exist
         continue-on-error: true
         run: |
-          gcloud compute instance-groups list | grep "zebrad-${{ env.GITHUB_REF_SLUG_URL }}" | grep "${{ env.REGION }}"
+          gcloud compute instance-groups list | grep "zebrad-${{ needs.versioning.outputs.major_version || env.GITHUB_REF_SLUG_URL }}" | grep "${{ env.REGION }}"
 
       # Deploy new managed instance group using the new instance template
       - name: Create managed instance group
         if: steps.does-group-exist.outcome == 'failure'
         run: |
           gcloud compute instance-groups managed create \
-          "zebrad-${{ env.GITHUB_REF_SLUG_URL }}" \
-          --template "zebrad-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}" \
+          "zebrad-${{ needs.versioning.outputs.major_version || env.GITHUB_REF_SLUG_URL }}" \
+          --template "zebrad-${{ needs.versioning.outputs.major_version || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}" \
           --health-check zebrad-tracing-filter \
           --initial-delay 30 \
           --region "${{ env.REGION }}" \
@@ -96,10 +148,17 @@ jobs:
         if: steps.does-group-exist.outcome == 'success'
         run: |
           gcloud compute instance-groups managed rolling-action start-update \
-          "zebrad-${{ env.GITHUB_REF_SLUG_URL }}" \
-          --version template="zebrad-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}" \
+          "zebrad-${{ needs.versioning.outputs.major_version || env.GITHUB_REF_SLUG_URL }}" \
+          --version template="zebrad-${{ needs.versioning.outputs.major_version || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}" \
           --region "${{ env.REGION }}"
 
+  # This jobs handles the deployment of a single node (1) in the us-central1-a zone
+  # when an instance is required to test a specific commit
+  #
+  # Runs:
+  # - on request, using workflow_dispatch with regenerate-disks
+  #
+  # Note: this instances are not automatically replaced or deleted
   deploy-instance:
     name: Deploy single instance
     needs: build
@@ -134,7 +193,7 @@ jobs:
           --boot-disk-type=pd-ssd \
           --container-stdin \
           --container-tty \
-          --container-image ${{ env.GAR_BASE }}/${{ env.GITHUB_REF_SLUG_URL }}:${{ env.GITHUB_SHA_SHORT }} \
+          --container-image ${{ env.GAR_BASE }}/zebrad@${{ needs.build.outputs.image_digest }} \
           --create-disk name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }},auto-delete=yes,size=100GB,type=pd-ssd \
           --container-mount-disk mount-path='/zebrad-cache',name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }} \
           --machine-type ${{ env.MACHINE_TYPE }} \

.github/workflows/continous-integration-docker.patch.yml

@@ -14,6 +14,9 @@ on:
       # dependencies
       - '**/Cargo.toml'
       - '**/Cargo.lock'
+      # configuration files
+      - '.cargo/config.toml'
+      - '**/clippy.toml'
       # workflow definitions
       - 'docker/**'
       - '.github/workflows/continous-integration-docker.yml'

.github/workflows/continous-integration-docker.yml

@@ -1,5 +1,12 @@
 name: CI Docker
 
+# Ensures that only one workflow task will run at a time. Previous builds, if
+# already in process, will get cancelled. Only the latest commit will be allowed
+# to run, cancelling any workflows in between
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
 on:
   workflow_dispatch:
     inputs:
@@ -33,6 +40,9 @@ on:
       # dependencies
       - '**/Cargo.toml'
       - '**/Cargo.lock'
+      # configuration files
+      - '.cargo/config.toml'
+      - '**/clippy.toml'
       # workflow definitions
       - 'docker/**'
       - '.github/workflows/continous-integration-docker.yml'
@@ -309,6 +319,14 @@ jobs:
       saves_to_disk: true
       disk_suffix: tip
       height_grep_text: 'current_height.*=.*Height'
+    # We don't want to cancel running full syncs on `main` if a new PR gets merged,
+    # because we might never finish a full sync during busy weeks. Instead, we let the
+    # first sync complete, then queue the latest pending sync, cancelling any syncs in between.
+    # (As the general workflow concurrency group just gets matched in Pull Requests,
+    # it has no impact on this job.)
+    concurrency: 
+      group: github.workflow−{{ github.ref }}
+      cancel-in-progress: false
 
   # Test that Zebra can sync to the chain tip, using a cached Zebra tip state,
   # without launching `lightwalletd`.
@@ -330,11 +348,10 @@ jobs:
       test_description: Test syncing to tip with a Zebra tip state
       test_variables: '-e TEST_UPDATE_SYNC=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache'
       needs_zebra_state: true
-      # TODO: do we want to update the disk on every PR, to increase CI speed?
-      saves_to_disk: false
+      # update the disk on every PR, to increase CI speed
+      saves_to_disk: true
       disk_suffix: tip
       root_state_path: '/var/cache'
-      # TODO: do we also want to test the `zebrad` part of the `lwd-cache`? (But not update it.)
       zebra_state_dir: 'zebrad-cache'
 
   # Test that Zebra can answer a synthetic RPC call, using a cached Zebra tip state
@@ -403,7 +420,6 @@ jobs:
     # to also run on Mergify head branches,
     # add `|| (github.event_name == 'push' && startsWith(github.head_ref, 'mergify/merge-queue/'))`:
     # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#running-your-workflow-based-on-the-head-or-base-branch-of-a-pull-request-1
-    # TODO: this test is unreliable, in the meanwhile we'll only generate a new lwd cached state when a full sync is also triggered
     if: ${{ (!cancelled() && !failure() && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true') || !fromJSON(needs.get-available-disks.outputs.lwd_tip_disk) }}
     with:
       app_name: lightwalletd
@@ -438,8 +454,8 @@ jobs:
       test_variables: '-e TEST_LWD_UPDATE_SYNC=1 -e ZEBRA_TEST_LIGHTWALLETD=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache -e LIGHTWALLETD_DATA_DIR=/var/cache/lwd-cache'
       needs_zebra_state: true
       needs_lwd_state: true
-      # TODO: do we want to update the disk on every PR, to increase CI speed?
-      saves_to_disk: false
+      # update the disk on every PR, to increase CI speed
+      saves_to_disk: true
       disk_prefix: lwd-cache
       disk_suffix: tip
       root_state_path: '/var/cache'

.github/workflows/continous-integration-os.patch.yml

@@ -9,6 +9,8 @@ on:
       - '**/Cargo.toml'
       - '**/Cargo.lock'
       - '**/deny.toml'
+      - '.cargo/config.toml'
+      - '**/clippy.toml'
       - '.github/workflows/continous-integration-os.yml'
 
 jobs:

.github/workflows/continous-integration-os.yml

@@ -1,5 +1,12 @@
 name: CI OSes
 
+# Ensures that only one workflow task will run at a time. Previous builds, if
+# already in process, will get cancelled. Only the latest commit will be allowed
+# to run, cancelling any workflows in between
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
 on:
   workflow_dispatch:
   # we build Rust and Zcash parameter caches on main,
@@ -17,6 +24,9 @@ on:
       # dependencies
       - '**/Cargo.toml'
       - '**/Cargo.lock'
+      # configuration files
+      - '.cargo/config.toml'
+      - '**/clippy.toml'
       # workflow definitions
       - '.github/workflows/ci.yml'
   pull_request:

.github/workflows/coverage.patch.yml

@@ -8,6 +8,9 @@ on:
       - '**/*.snap'
       - '**/Cargo.toml'
       - '**/Cargo.lock'
+      # configuration files
+      - '.cargo/config.toml'
+      - '**/clippy.toml'
       - 'codecov.yml'
       - '.github/workflows/coverage.yml'