From 9229424ebb588071b1c57262451317ee0a03af74 Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 26 Jun 2023 09:24:10 +1000 Subject: [PATCH 1/5] add(ci): Run release builds and production Docker image tests on pull requests (#7055) * Update the workflow run conditions for CI docker tests * Run release builds and release Docker image tests on pull requests * Remove the manual docker test from the release checklist * Fix workflow syntax * Use the right kind of quotes --- .../release-checklist.md | 5 +-- .../workflows/continous-delivery.patch.yml | 36 ++++++++++++++++ .github/workflows/continous-delivery.yml | 43 ++++++++++++++++++- .../continous-integration-docker.patch.yml | 2 + .../continous-integration-docker.yml | 3 +- 5 files changed, 83 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/continous-delivery.patch.yml diff --git a/.github/PULL_REQUEST_TEMPLATE/release-checklist.md b/.github/PULL_REQUEST_TEMPLATE/release-checklist.md index 774eda60df1..1f010c13a07 100644 --- a/.github/PULL_REQUEST_TEMPLATE/release-checklist.md +++ b/.github/PULL_REQUEST_TEMPLATE/release-checklist.md @@ -158,10 +158,7 @@ The end of support height is calculated from the current blockchain height: and put the output in a comment on the PR. ## Publish Docker Images -- [ ] Wait until [the Docker images have been published](https://github.com/ZcashFoundation/zebra/actions/workflows/release-binaries.yml) -- [ ] Test the Docker image using `docker run --tty --interactive zfnd/zebra:v1.0.0`, - and put the output in a comment on the PR. - (You can use [gcloud cloud shell](https://console.cloud.google.com/home/dashboard?cloudshell=true)) +- [ ] Wait for the [the Docker images to be published successfully](https://github.com/ZcashFoundation/zebra/actions/workflows/release-binaries.yml). - [ ] Un-freeze the [`batched` queue](https://dashboard.mergify.com/github/ZcashFoundation/repo/zebra/queues) using Mergify. ## Release Failures diff --git a/.github/workflows/continous-delivery.patch.yml b/.github/workflows/continous-delivery.patch.yml new file mode 100644 index 00000000000..4dbc98652a7 --- /dev/null +++ b/.github/workflows/continous-delivery.patch.yml @@ -0,0 +1,36 @@ +name: CD + +on: + # Only patch the Docker image test jobs + pull_request: + paths-ignore: + # code and tests + - '**/*.rs' + # hard-coded checkpoints and proptest regressions + - '**/*.txt' + # dependencies + - '**/Cargo.toml' + - '**/Cargo.lock' + # configuration files + - '.cargo/config.toml' + - '**/clippy.toml' + # workflow definitions + - 'docker/**' + - '.dockerignore' + - '.github/workflows/continous-delivery.yml' + - '.github/workflows/find-cached-disks.yml' + + +jobs: + # Also patched by continous-integration-docker.patch.yml, which has a different paths-ignore + build: + name: Build CI Docker / Build images + runs-on: ubuntu-latest + steps: + - run: 'echo "No build required"' + + test-configuration-file: + name: Test Zebra default Docker config file + runs-on: ubuntu-latest + steps: + - run: 'echo "No build required"' diff --git a/.github/workflows/continous-delivery.yml b/.github/workflows/continous-delivery.yml index 15c8c08a615..aee4e80e8b2 100644 --- a/.github/workflows/continous-delivery.yml +++ b/.github/workflows/continous-delivery.yml @@ -6,9 +6,12 @@ name: CD # # Since the different event types each use a different Managed Instance Group or instance, # we can run different event types concurrently. +# +# For pull requests, we only run the tests from this workflow, and don't do any deployments. +# So an in-progress pull request gets cancelled, just like other tests. concurrency: group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }} - cancel-in-progress: false + cancel-in-progress: ${{ github.event_name == 'pull_request' }} on: workflow_dispatch: @@ -25,14 +28,52 @@ on: required: false type: boolean default: false + # Temporarily disabled to reduce network load, see #6894. #push: # branches: # - main + # paths: + # # code and tests + # - '**/*.rs' + # # hard-coded checkpoints and proptest regressions + # - '**/*.txt' + # # dependencies + # - '**/Cargo.toml' + # - '**/Cargo.lock' + # # configuration files + # - '.cargo/config.toml' + # - '**/clippy.toml' + # # workflow definitions + # - 'docker/**' + # - '.dockerignore' + # - '.github/workflows/continous-delivery.yml' + # - '.github/workflows/build-docker-image.yml' + + # Only runs the Docker image tests, doesn't deploy any instances + pull_request: + paths: + # code and tests + - '**/*.rs' + # hard-coded checkpoints and proptest regressions + - '**/*.txt' + # dependencies + - '**/Cargo.toml' + - '**/Cargo.lock' + # configuration files + - '.cargo/config.toml' + - '**/clippy.toml' + # workflow definitions + - 'docker/**' + - '.dockerignore' + - '.github/workflows/continous-delivery.yml' + - '.github/workflows/find-cached-disks.yml' + release: types: - published + jobs: # If a release was made we want to extract the first part of the semver from the # tag_name diff --git a/.github/workflows/continous-integration-docker.patch.yml b/.github/workflows/continous-integration-docker.patch.yml index b3cf3a8b537..489d75b01b3 100644 --- a/.github/workflows/continous-integration-docker.patch.yml +++ b/.github/workflows/continous-integration-docker.patch.yml @@ -19,8 +19,10 @@ on: - '**/clippy.toml' # workflow definitions - 'docker/**' + - '.dockerignore' - '.github/workflows/continous-integration-docker.yml' - '.github/workflows/deploy-gcp-tests.yml' + - '.github/workflows/find-cached-disks.yml' - '.github/workflows/build-docker-image.yml' jobs: diff --git a/.github/workflows/continous-integration-docker.yml b/.github/workflows/continous-integration-docker.yml index 40d2313a894..902218ef266 100644 --- a/.github/workflows/continous-integration-docker.yml +++ b/.github/workflows/continous-integration-docker.yml @@ -80,10 +80,11 @@ on: - '**/clippy.toml' # workflow definitions - 'docker/**' + - '.dockerignore' - '.github/workflows/continous-integration-docker.yml' - '.github/workflows/deploy-gcp-tests.yml' - - '.github/workflows/build-docker-image.yml' - '.github/workflows/find-cached-disks.yml' + - '.github/workflows/build-docker-image.yml' jobs: # to also run a job on Mergify head branches, From f455baaa6e32971c4759705b47a27911ade1143b Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 26 Jun 2023 11:12:29 +1000 Subject: [PATCH 2/5] Rename a CD job with the same name as a CI job (#7063) --- .github/workflows/continous-delivery.patch.yml | 2 +- .github/workflows/continous-delivery.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/continous-delivery.patch.yml b/.github/workflows/continous-delivery.patch.yml index 4dbc98652a7..53ecb144a9a 100644 --- a/.github/workflows/continous-delivery.patch.yml +++ b/.github/workflows/continous-delivery.patch.yml @@ -30,7 +30,7 @@ jobs: - run: 'echo "No build required"' test-configuration-file: - name: Test Zebra default Docker config file + name: Test Zebra CD Docker config file runs-on: ubuntu-latest steps: - run: 'echo "No build required"' diff --git a/.github/workflows/continous-delivery.yml b/.github/workflows/continous-delivery.yml index aee4e80e8b2..8cd334560fc 100644 --- a/.github/workflows/continous-delivery.yml +++ b/.github/workflows/continous-delivery.yml @@ -123,7 +123,7 @@ jobs: # Test that Zebra works using the default config with the latest Zebra version, # and test reconfiguring the docker image for testnet. test-configuration-file: - name: Test Zebra default Docker config file + name: Test Zebra CD Docker config file timeout-minutes: 15 runs-on: ubuntu-latest needs: build From 76a7ff45a990eb1137dc0f5c594534001c887b17 Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 26 Jun 2023 15:44:19 +1000 Subject: [PATCH 3/5] fix(deps): Replace openssl with rustls in tests and experimental features (#7047) * Remove openssl dependency and prevent it coming back * Put the arguments in the right place * Put comment in the right place * Add a default-docker feature to zebrad and use it in workflows and Docker files * Fix a comment typo * Make sure that Docker production builds don't use openssl * Rename feature to default-release-binaries --- .github/workflows/build-docker-image.yml | 2 +- .../workflows/continous-integration-os.yml | 12 ++++++---- .github/workflows/release-binaries.yml | 2 +- deny.toml | 23 +++++++++++++++++-- docker/Dockerfile | 2 +- docker/zcash-params/Dockerfile | 4 ++-- zebra-node-services/Cargo.toml | 5 ++-- zebra-state/Cargo.toml | 3 ++- zebrad/Cargo.toml | 3 +++ 9 files changed, 41 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build-docker-image.yml b/.github/workflows/build-docker-image.yml index 6ae6f28166d..95c187bec97 100644 --- a/.github/workflows/build-docker-image.yml +++ b/.github/workflows/build-docker-image.yml @@ -38,7 +38,7 @@ on: # https://github.com/ZcashFoundation/zebra/blob/main/docker/Dockerfile#L83 features: required: false - default: "sentry" + default: "default-release-binaries" type: string test_features: required: false diff --git a/.github/workflows/continous-integration-os.yml b/.github/workflows/continous-integration-os.yml index 93303977e95..3a189653a82 100644 --- a/.github/workflows/continous-integration-os.yml +++ b/.github/workflows/continous-integration-os.yml @@ -261,8 +261,8 @@ jobs: # We don't need to check `--no-default-features` here, because (except in very rare cases): # - disabling features isn't going to add duplicate dependencies # - disabling features isn't going to add more crate sources - features: ['', '--all-features'] - # We always want to run the --all-features job, because it gives accurate "skip tree root was not found" warnings + features: ['', '--features default-release-binaries', '--all-features'] + # Always run the --all-features job, to get accurate "skip tree root was not found" warnings fail-fast: false # Prevent sudden announcement of a new advisory from failing ci: @@ -274,12 +274,14 @@ jobs: persist-credentials: false - uses: r7kamura/rust-problem-matchers@v1.3.0 - # The --all-features job is the only job that gives accurate "skip tree root was not found" warnings. - # In other jobs, we expect some of these warnings, due to disabled features. - name: Check ${{ matrix.checks }} with features ${{ matrix.features }} uses: EmbarkStudios/cargo-deny-action@v1 with: - command: check ${{ matrix.checks }} + # --all-features spuriously activates openssl, but we want to ban that dependency in + # all of zebrad's production features for security reasons. But the --all-features job is + # the only job that gives accurate "skip tree root was not found" warnings. + # In other jobs, we expect some of these warnings, due to disabled features. + command: check ${{ matrix.checks }} ${{ matrix.features == '--all-features' && '--allow banned' || '--allow unmatched-skip-root' }} arguments: --workspace ${{ matrix.features }} unused-deps: diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml index 2ec7338b5d5..81ee18c7b18 100644 --- a/.github/workflows/release-binaries.yml +++ b/.github/workflows/release-binaries.yml @@ -44,7 +44,7 @@ jobs: tag_suffix: .experimental network: Testnet rpc_port: '18232' - features: "sentry getblocktemplate-rpcs" + features: "default-release-binaries getblocktemplate-rpcs" test_features: "" rust_backtrace: '1' zebra_skip_ipv6_tests: '1' diff --git a/deny.toml b/deny.toml index 2014f775cd5..900d08ef9e0 100644 --- a/deny.toml +++ b/deny.toml @@ -10,6 +10,13 @@ [bans] # Lint level for when multiple versions of the same crate are detected multiple-versions = "deny" + +# Don't allow wildcard ("any version") dependencies +wildcards = "deny" +# Allow private and dev wildcard dependencies. +# Switch this to `false` when #6924 is implemented. +allow-wildcard-paths = true + # The graph highlighting used when creating dotgraphs for crates # with multiple versions # * lowest-version - The path to the lowest versioned duplicate is highlighted @@ -17,12 +24,18 @@ multiple-versions = "deny" # * all - Both lowest-version and simplest-path are used highlight = "all" -# We don't use this for Zebra. -# # List of crates that are allowed. Use with care! #allow = [ #] +# List of crates that can never become Zebra dependencies. +deny = [ + # Often has memory safety vulnerabilities. + # Enabled by --all-features, use the `cargo hack` script in the deny.toml CI job instead. + { name = "openssl" }, + { name = "openssl-sys" }, +] + # We only use this for some `librustzcash` and `orchard` crates. # If we add a crate here, duplicate dependencies of that crate are still shown. # @@ -47,6 +60,9 @@ skip-tree = [ # wait for criterion to upgrade { name = "itertools", version = "=0.10.5" }, + # wait for backtrace and multiple dependents to upgrade + { name = "miniz_oxide", version = "=0.6.2" }, + # ZF crates # wait for zcashd and zcash_script to upgrade @@ -71,6 +87,9 @@ skip-tree = [ # wait for zcash_address to upgrade { name = "bs58", version = "=0.4.0" }, + # wait for minreq and zcash_proofs to upgrade + { name = "rustls", version = "=0.20.8" }, + # zebra-utils dependencies # wait for structopt upgrade (or upgrade to clap 4) diff --git a/docker/Dockerfile b/docker/Dockerfile index 74224019637..3b636339797 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -81,7 +81,7 @@ ENV ZEBRA_SKIP_IPV6_TESTS ${ZEBRA_SKIP_IPV6_TESTS:-1} # Build zebrad with these features # Keep these in sync with: # https://github.com/ZcashFoundation/zebra/blob/main/.github/workflows/build-docker-image.yml#L42 -ARG FEATURES="sentry" +ARG FEATURES="default-release-binaries" ARG TEST_FEATURES="lightwalletd-grpc-tests zebra-checkpoints" # Use ENTRYPOINT_FEATURES to override the specific features used to run tests in entrypoint.sh, # separately from the test and production image builds. diff --git a/docker/zcash-params/Dockerfile b/docker/zcash-params/Dockerfile index 1036a2be40b..a9bea7233a6 100644 --- a/docker/zcash-params/Dockerfile +++ b/docker/zcash-params/Dockerfile @@ -23,7 +23,7 @@ RUN apt-get -qq update && \ ENV CARGO_HOME /opt/zebrad/.cargo/ # Build dependencies - this is the caching Docker layer! -RUN cargo chef cook --release --features sentry --package zebrad --recipe-path recipe.json +RUN cargo chef cook --release --features default-release-binaries --package zebrad --recipe-path recipe.json ARG RUST_BACKTRACE=0 ENV RUST_BACKTRACE ${RUST_BACKTRACE} @@ -36,4 +36,4 @@ ENV COLORBT_SHOW_HIDDEN ${COLORBT_SHOW_HIDDEN} COPY . . # Pre-download Zcash Sprout and Sapling parameters -RUN cargo run --locked --release --features sentry --package zebrad --bin zebrad download +RUN cargo run --locked --release --features default-release-binaries --package zebrad --bin zebrad download diff --git a/zebra-node-services/Cargo.toml b/zebra-node-services/Cargo.toml index 70b469727ea..9d9b3885a50 100644 --- a/zebra-node-services/Cargo.toml +++ b/zebra-node-services/Cargo.toml @@ -42,7 +42,8 @@ zebra-chain = { path = "../zebra-chain" , version = "1.0.0-beta.26"} # Tool and test feature rpc-client color-eyre = { version = "0.6.2", optional = true } jsonrpc-core = { version = "18.0.0", optional = true } -reqwest = { version = "0.11.18", optional = true } +# Security: avoid default dependency on openssl +reqwest = { version = "0.11.18", default-features = false, features = ["rustls-tls"], optional = true } serde = { version = "1.0.164", optional = true } serde_json = { version = "1.0.97", optional = true } @@ -50,6 +51,6 @@ serde_json = { version = "1.0.97", optional = true } color-eyre = "0.6.2" jsonrpc-core = "18.0.0" -reqwest = "0.11.18" +reqwest = { version = "0.11.18", default-features = false, features = ["rustls-tls"] } serde = "1.0.164" serde_json = "1.0.97" diff --git a/zebra-state/Cargo.toml b/zebra-state/Cargo.toml index e4d87463ce5..e4dcda077a9 100644 --- a/zebra-state/Cargo.toml +++ b/zebra-state/Cargo.toml @@ -67,7 +67,8 @@ tower = { version = "0.4.13", features = ["buffer", "util"] } tracing = "0.1.37" # elasticsearch specific dependencies. -elasticsearch = { version = "8.5.0-alpha.1", package = "elasticsearch", optional = true } +# Security: avoid default dependency on openssl +elasticsearch = { version = "8.5.0-alpha.1", default-features = false, features = ["rustls-tls"], optional = true } serde_json = { version = "1.0.97", package = "serde_json", optional = true } zebra-chain = { path = "../zebra-chain", version = "1.0.0-beta.26" } diff --git a/zebrad/Cargo.toml b/zebrad/Cargo.toml index 9682bc185fd..e1ac083e1af 100644 --- a/zebrad/Cargo.toml +++ b/zebrad/Cargo.toml @@ -39,6 +39,9 @@ pre-release-replacements = [ # In release builds, don't compile debug logging code, to improve performance. default = ["release_max_level_info"] +# Default features for official ZF binary release builds +default-release-binaries = ["default", "sentry"] + # Production features that activate extra dependencies, or extra features in dependencies # Experimental mining RPC support From a6f35afe3700c1ba0a19905794143e20b694f2bf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Jun 2023 19:43:29 +0000 Subject: [PATCH 4/5] build(deps): bump clap from 4.3.6 to 4.3.8 (#7066) Bumps [clap](https://github.com/clap-rs/clap) from 4.3.6 to 4.3.8. - [Release notes](https://github.com/clap-rs/clap/releases) - [Changelog](https://github.com/clap-rs/clap/blob/master/CHANGELOG.md) - [Commits](https://github.com/clap-rs/clap/compare/v4.3.6...v4.3.8) --- updated-dependencies: - dependency-name: clap dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 14 +++++++------- zebrad/Cargo.toml | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1c984acaa25..63195c9fd51 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12,7 +12,7 @@ dependencies = [ "arc-swap", "backtrace", "canonical-path", - "clap 4.3.6", + "clap 4.3.8", "color-eyre", "fs-err", "once_cell", @@ -773,9 +773,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.3.6" +version = "4.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6320c6d1c98b6981da7bb2dcecbd0be9dc98d42165fa8326b21000f7dbfde6d0" +checksum = "d9394150f5b4273a1763355bd1c2ec54cc5a2593f790587bcd6b2c947cfa9211" dependencies = [ "clap_builder", "clap_derive", @@ -784,9 +784,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.3.5" +version = "4.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e53afce1efce6ed1f633cf0e57612fe51db54a1ee4fd8f8503d078fe02d69ae" +checksum = "9a78fbdd3cc2914ddf37ba444114bc7765bbdcb55ec9cbe6fa054f0137400717" dependencies = [ "anstream", "anstyle", @@ -956,7 +956,7 @@ dependencies = [ "anes", "cast", "ciborium", - "clap 4.3.6", + "clap 4.3.8", "criterion-plot", "is-terminal", "itertools 0.10.5", @@ -5954,7 +5954,7 @@ dependencies = [ "abscissa_core", "atty", "chrono", - "clap 4.3.6", + "clap 4.3.8", "color-eyre", "console-subscriber", "dirs", diff --git a/zebrad/Cargo.toml b/zebrad/Cargo.toml index e1ac083e1af..56c034e2658 100644 --- a/zebrad/Cargo.toml +++ b/zebrad/Cargo.toml @@ -138,7 +138,7 @@ zebra-state = { path = "../zebra-state", version = "1.0.0-beta.26" } zebra-utils = { path = "../zebra-utils", version = "1.0.0-beta.26", optional = true } abscissa_core = "0.7.0" -clap = { version = "4.3.6", features = ["cargo"] } +clap = { version = "4.3.8", features = ["cargo"] } chrono = { version = "0.4.26", default-features = false, features = ["clock", "std"] } humantime-serde = "1.1.1" indexmap = "1.9.3" From 015a970e16a6c4913c6d1092633dac836570a5b7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Jun 2023 21:30:10 +0000 Subject: [PATCH 5/5] build(deps): bump tj-actions/changed-files from 36.4.1 to 37.0.3 (#7065) Bumps [tj-actions/changed-files](https://github.com/tj-actions/changed-files) from 36.4.1 to 37.0.3. - [Release notes](https://github.com/tj-actions/changed-files/releases) - [Changelog](https://github.com/tj-actions/changed-files/blob/main/HISTORY.md) - [Commits](https://github.com/tj-actions/changed-files/compare/v36.4.1...v37.0.3) --- updated-dependencies: - dependency-name: tj-actions/changed-files dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/lint.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 9c43b4a128a..d3427fd19a3 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -37,7 +37,7 @@ jobs: - name: Rust files id: changed-files-rust - uses: tj-actions/changed-files@v36.4.1 + uses: tj-actions/changed-files@v37.0.3 with: files: | **/*.rs @@ -49,7 +49,7 @@ jobs: - name: Workflow files id: changed-files-workflows - uses: tj-actions/changed-files@v36.4.1 + uses: tj-actions/changed-files@v37.0.3 with: files: | .github/workflows/*.yml