Merge branch 'main' into sprout-zebra-db-snapshot

.github/PULL_REQUEST_TEMPLATE/release-checklist.md

@@ -158,10 +158,7 @@ The end of support height is calculated from the current blockchain height:
       and put the output in a comment on the PR.
 
 ## Publish Docker Images
-- [ ] Wait until [the Docker images have been published](https://github.com/ZcashFoundation/zebra/actions/workflows/release-binaries.yml)
-- [ ] Test the Docker image using `docker run --tty --interactive zfnd/zebra:v1.0.0`,
-      and put the output in a comment on the PR.
-      (You can use [gcloud cloud shell](https://console.cloud.google.com/home/dashboard?cloudshell=true))
+- [ ] Wait for the [the Docker images to be published successfully](https://github.com/ZcashFoundation/zebra/actions/workflows/release-binaries.yml).
 - [ ] Un-freeze the [`batched` queue](https://dashboard.mergify.com/github/ZcashFoundation/repo/zebra/queues) using Mergify.
 
 ## Release Failures

.github/workflows/build-docker-image.yml

@@ -38,7 +38,7 @@ on:
       # https://github.com/ZcashFoundation/zebra/blob/main/docker/Dockerfile#L83
       features:
         required: false
-        default: "sentry"
+        default: "default-release-binaries"
         type: string
       test_features:
         required: false

.github/workflows/continous-delivery.patch.yml

@@ -0,0 +1,36 @@
+name: CD
+
+on:
+  # Only patch the Docker image test jobs
+  pull_request:
+    paths-ignore:
+      # code and tests
+      - '**/*.rs'
+      # hard-coded checkpoints and proptest regressions
+      - '**/*.txt'
+      # dependencies
+      - '**/Cargo.toml'
+      - '**/Cargo.lock'
+      # configuration files
+      - '.cargo/config.toml'
+      - '**/clippy.toml'
+      # workflow definitions
+      - 'docker/**'
+      - '.dockerignore'
+      - '.github/workflows/continous-delivery.yml'
+      - '.github/workflows/find-cached-disks.yml'
+
+
+jobs:
+  # Also patched by continous-integration-docker.patch.yml, which has a different paths-ignore
+  build:
+    name: Build CI Docker / Build images
+    runs-on: ubuntu-latest
+    steps:
+      - run: 'echo "No build required"'
+
+  test-configuration-file:
+    name: Test Zebra CD Docker config file
+    runs-on: ubuntu-latest
+    steps:
+      - run: 'echo "No build required"'

.github/workflows/continous-delivery.yml

@@ -6,9 +6,12 @@ name: CD
 #
 # Since the different event types each use a different Managed Instance Group or instance,
 # we can run different event types concurrently.
+#
+# For pull requests, we only run the tests from this workflow, and don't do any deployments.
+# So an in-progress pull request gets cancelled, just like other tests.
 concurrency:
   group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }}
-  cancel-in-progress: false
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 on:
   workflow_dispatch:
@@ -25,14 +28,52 @@ on:
         required: false
         type: boolean
         default: false
+
   # Temporarily disabled to reduce network load, see #6894.
   #push:
   #  branches:
   #    - main
+  #  paths:
+  #    # code and tests
+  #    - '**/*.rs'
+  #    # hard-coded checkpoints and proptest regressions
+  #    - '**/*.txt'
+  #    # dependencies
+  #    - '**/Cargo.toml'
+  #    - '**/Cargo.lock'
+  #    # configuration files
+  #    - '.cargo/config.toml'
+  #    - '**/clippy.toml'
+  #    # workflow definitions
+  #    - 'docker/**'
+  #    - '.dockerignore'
+  #    - '.github/workflows/continous-delivery.yml'
+  #    - '.github/workflows/build-docker-image.yml'
+
+  # Only runs the Docker image tests, doesn't deploy any instances
+  pull_request:
+    paths:
+      # code and tests
+      - '**/*.rs'
+      # hard-coded checkpoints and proptest regressions
+      - '**/*.txt'
+      # dependencies
+      - '**/Cargo.toml'
+      - '**/Cargo.lock'
+      # configuration files
+      - '.cargo/config.toml'
+      - '**/clippy.toml'
+      # workflow definitions
+      - 'docker/**'
+      - '.dockerignore'
+      - '.github/workflows/continous-delivery.yml'
+      - '.github/workflows/find-cached-disks.yml'
+
   release:
     types:
       - published
 
+
 jobs:
   # If a release was made we want to extract the first part of the semver from the
   # tag_name
@@ -82,7 +123,7 @@ jobs:
   # Test that Zebra works using the default config with the latest Zebra version,
   # and test reconfiguring the docker image for testnet.
   test-configuration-file:
-    name: Test Zebra default Docker config file
+    name: Test Zebra CD Docker config file
     timeout-minutes: 15
     runs-on: ubuntu-latest
     needs: build

.github/workflows/continous-integration-docker.patch.yml

@@ -19,8 +19,10 @@ on:
       - '**/clippy.toml'
       # workflow definitions
       - 'docker/**'
+      - '.dockerignore'
       - '.github/workflows/continous-integration-docker.yml'
       - '.github/workflows/deploy-gcp-tests.yml'
+      - '.github/workflows/find-cached-disks.yml'
       - '.github/workflows/build-docker-image.yml'
 
 jobs:

.github/workflows/continous-integration-docker.yml

@@ -80,10 +80,11 @@ on:
       - '**/clippy.toml'
       # workflow definitions
       - 'docker/**'
+      - '.dockerignore'
       - '.github/workflows/continous-integration-docker.yml'
       - '.github/workflows/deploy-gcp-tests.yml'
-      - '.github/workflows/build-docker-image.yml'
       - '.github/workflows/find-cached-disks.yml'
+      - '.github/workflows/build-docker-image.yml'
 
 jobs:
   # to also run a job on Mergify head branches,

.github/workflows/continous-integration-os.yml

@@ -261,8 +261,8 @@ jobs:
         # We don't need to check `--no-default-features` here, because (except in very rare cases):
         # - disabling features isn't going to add duplicate dependencies
         # - disabling features isn't going to add more crate sources
-        features: ['', '--all-features']
-      # We always want to run the --all-features job, because it gives accurate "skip tree root was not found" warnings
+        features: ['', '--features default-release-binaries', '--all-features']
+      # Always run the --all-features job, to get accurate "skip tree root was not found" warnings
       fail-fast: false
 
     # Prevent sudden announcement of a new advisory from failing ci:
@@ -274,12 +274,14 @@ jobs:
           persist-credentials: false
       - uses: r7kamura/rust-problem-matchers@v1.3.0
 
-      # The --all-features job is the only job that gives accurate "skip tree root was not found" warnings.
-      # In other jobs, we expect some of these warnings, due to disabled features.
       - name: Check ${{ matrix.checks }} with features ${{ matrix.features }}
         uses: EmbarkStudios/cargo-deny-action@v1
         with:
-          command: check ${{ matrix.checks }}
+          # --all-features spuriously activates openssl, but we want to ban that dependency in
+          # all of zebrad's production features for security reasons. But the --all-features job is
+          # the only job that gives accurate "skip tree root was not found" warnings.
+          # In other jobs, we expect some of these warnings, due to disabled features.
+          command: check ${{ matrix.checks }} ${{ matrix.features == '--all-features' && '--allow banned' || '--allow unmatched-skip-root' }}
           arguments: --workspace ${{ matrix.features }}
 
   unused-deps:

.github/workflows/lint.yml

@@ -37,7 +37,7 @@ jobs:
 
       - name: Rust files
         id: changed-files-rust
-        uses: tj-actions/changed-files@v36.4.1
+        uses: tj-actions/changed-files@v37.0.3
         with:
           files: |
             **/*.rs
@@ -49,7 +49,7 @@ jobs:
 
       - name: Workflow files
         id: changed-files-workflows
-        uses: tj-actions/changed-files@v36.4.1
+        uses: tj-actions/changed-files@v37.0.3
         with:
           files: |
             .github/workflows/*.yml

.github/workflows/release-binaries.yml

@@ -44,7 +44,7 @@ jobs:
       tag_suffix: .experimental
       network: Testnet
       rpc_port: '18232'
-      features: "sentry getblocktemplate-rpcs"
+      features: "default-release-binaries getblocktemplate-rpcs"
       test_features: ""
       rust_backtrace: '1'
       zebra_skip_ipv6_tests: '1'

Cargo.lock

@@ -12,7 +12,7 @@ dependencies = [
  "arc-swap",
  "backtrace",
  "canonical-path",
- "clap 4.3.6",
+ "clap 4.3.8",
  "color-eyre",
  "fs-err",
  "once_cell",
@@ -773,9 +773,9 @@ dependencies = [
 
 [[package]]
 name = "clap"
-version = "4.3.6"
+version = "4.3.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6320c6d1c98b6981da7bb2dcecbd0be9dc98d42165fa8326b21000f7dbfde6d0"
+checksum = "d9394150f5b4273a1763355bd1c2ec54cc5a2593f790587bcd6b2c947cfa9211"
 dependencies = [
  "clap_builder",
  "clap_derive",
@@ -784,9 +784,9 @@ dependencies = [
 
 [[package]]
 name = "clap_builder"
-version = "4.3.5"
+version = "4.3.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2e53afce1efce6ed1f633cf0e57612fe51db54a1ee4fd8f8503d078fe02d69ae"
+checksum = "9a78fbdd3cc2914ddf37ba444114bc7765bbdcb55ec9cbe6fa054f0137400717"
 dependencies = [
  "anstream",
  "anstyle",
@@ -956,7 +956,7 @@ dependencies = [
  "anes",
  "cast",
  "ciborium",
- "clap 4.3.6",
+ "clap 4.3.8",
  "criterion-plot",
  "is-terminal",
  "itertools 0.10.5",
@@ -5955,7 +5955,7 @@ dependencies = [
  "abscissa_core",
  "atty",
  "chrono",
- "clap 4.3.6",
+ "clap 4.3.8",
  "color-eyre",
  "console-subscriber",
  "dirs",

deny.toml

@@ -10,19 +10,32 @@
 [bans]
 # Lint level for when multiple versions of the same crate are detected
 multiple-versions = "deny"
+
+# Don't allow wildcard ("any version") dependencies
+wildcards = "deny"
+# Allow private and dev wildcard dependencies.
+# Switch this to `false` when #6924 is implemented.
+allow-wildcard-paths = true
+
 # The graph highlighting used when creating dotgraphs for crates
 # with multiple versions
 # * lowest-version - The path to the lowest versioned duplicate is highlighted
 # * simplest-path - The path to the version with the fewest edges is highlighted
 # * all - Both lowest-version and simplest-path are used
 highlight = "all"
 
-# We don't use this for Zebra.
-#
 # List of crates that are allowed. Use with care!
 #allow = [
 #]
 
+# List of crates that can never become Zebra dependencies.
+deny = [
+    # Often has memory safety vulnerabilities.
+    # Enabled by --all-features, use the `cargo hack` script in the deny.toml CI job instead.
+    { name = "openssl" },
+    { name = "openssl-sys" },
+]
+
 # We only use this for some `librustzcash` and `orchard` crates.
 # If we add a crate here, duplicate dependencies of that crate are still shown.
 #
@@ -47,6 +60,9 @@ skip-tree = [
     # wait for criterion to upgrade
     { name = "itertools", version = "=0.10.5" },
 
+    # wait for backtrace and multiple dependents to upgrade
+    { name = "miniz_oxide", version = "=0.6.2" },
+
     # ZF crates
 
     # wait for zcashd and zcash_script to upgrade
@@ -71,6 +87,9 @@ skip-tree = [
     # wait for zcash_address to upgrade
     { name = "bs58", version = "=0.4.0" },
 
+    # wait for minreq and zcash_proofs to upgrade
+    { name = "rustls", version = "=0.20.8" },
+
     # zebra-utils dependencies
 
     # wait for structopt upgrade (or upgrade to clap 4)

docker/Dockerfile

@@ -81,7 +81,7 @@ ENV ZEBRA_SKIP_IPV6_TESTS ${ZEBRA_SKIP_IPV6_TESTS:-1}
 # Build zebrad with these features
 # Keep these in sync with:
 # https://github.com/ZcashFoundation/zebra/blob/main/.github/workflows/build-docker-image.yml#L42
-ARG FEATURES="sentry"
+ARG FEATURES="default-release-binaries"
 ARG TEST_FEATURES="lightwalletd-grpc-tests zebra-checkpoints"
 # Use ENTRYPOINT_FEATURES to override the specific features used to run tests in entrypoint.sh,
 # separately from the test and production image builds.

docker/zcash-params/Dockerfile

@@ -23,7 +23,7 @@ RUN apt-get -qq update && \
 
 ENV CARGO_HOME /opt/zebrad/.cargo/
 # Build dependencies - this is the caching Docker layer!
-RUN cargo chef cook --release --features sentry --package zebrad --recipe-path recipe.json
+RUN cargo chef cook --release --features default-release-binaries --package zebrad --recipe-path recipe.json
 
 ARG RUST_BACKTRACE=0
 ENV RUST_BACKTRACE ${RUST_BACKTRACE}
@@ -36,4 +36,4 @@ ENV COLORBT_SHOW_HIDDEN ${COLORBT_SHOW_HIDDEN}
 
 COPY . .
 # Pre-download Zcash Sprout and Sapling parameters
-RUN cargo run --locked --release --features sentry --package zebrad --bin zebrad download
+RUN cargo run --locked --release --features default-release-binaries --package zebrad --bin zebrad download

zebra-node-services/Cargo.toml

@@ -42,14 +42,15 @@ zebra-chain = { path = "../zebra-chain" , version = "1.0.0-beta.26"}
 # Tool and test feature rpc-client
 color-eyre = { version = "0.6.2", optional = true }
 jsonrpc-core = { version = "18.0.0", optional = true }
-reqwest = { version = "0.11.18", optional = true }
+# Security: avoid default dependency on openssl
+reqwest = { version = "0.11.18", default-features = false, features = ["rustls-tls"], optional = true }
 serde = { version = "1.0.164", optional = true }
 serde_json = { version = "1.0.97", optional = true }
 
 [dev-dependencies]
 
 color-eyre = "0.6.2"
 jsonrpc-core = "18.0.0"
-reqwest = "0.11.18"
+reqwest = { version = "0.11.18", default-features = false, features = ["rustls-tls"] }
 serde = "1.0.164"
 serde_json = "1.0.97"

zebra-state/Cargo.toml

@@ -67,7 +67,8 @@ tower = { version = "0.4.13", features = ["buffer", "util"] }
 tracing = "0.1.37"
 
 # elasticsearch specific dependencies.
-elasticsearch = { version = "8.5.0-alpha.1", package = "elasticsearch", optional = true }
+# Security: avoid default dependency on openssl
+elasticsearch = { version = "8.5.0-alpha.1", default-features = false, features = ["rustls-tls"], optional = true }
 serde_json = { version = "1.0.97", package = "serde_json", optional = true }
 
 zebra-chain = { path = "../zebra-chain", version = "1.0.0-beta.26" }

zebrad/Cargo.toml

@@ -39,6 +39,9 @@ pre-release-replacements = [
 # In release builds, don't compile debug logging code, to improve performance.
 default = ["release_max_level_info"]
 
+# Default features for official ZF binary release builds
+default-release-binaries = ["default", "sentry"]
+
 # Production features that activate extra dependencies, or extra features in dependencies
 
 # Experimental mining RPC support
@@ -135,7 +138,7 @@ zebra-state = { path = "../zebra-state", version = "1.0.0-beta.26" }
 zebra-utils = { path = "../zebra-utils", version = "1.0.0-beta.26", optional = true }
 
 abscissa_core = "0.7.0"
-clap = { version = "4.3.6", features = ["cargo"] }
+clap = { version = "4.3.8", features = ["cargo"] }
 chrono = { version = "0.4.26", default-features = false, features = ["clock", "std"] }
 humantime-serde = "1.1.1"
 indexmap = "1.9.3"