Merge branch 'develop' into feat/clarinet-setup

.github/actions/bitcoin-int-tests/Dockerfile.bitcoin-tests

@@ -1,36 +1,12 @@
-FROM rust:bullseye
+FROM stacks-node:integrations AS test
 
-WORKDIR /src/
-
-COPY . .
-
-WORKDIR /src/testnet/stacks-node
-RUN cargo test --no-run
-
-RUN cd / && wget https://bitcoin.org/bin/bitcoin-core-0.20.0/bitcoin-0.20.0-x86_64-linux-gnu.tar.gz
-RUN cd / && tar -xvzf bitcoin-0.20.0-x86_64-linux-gnu.tar.gz
+ARG test_name
+ENV BITCOIND_TEST 1
 
-RUN ln -s /bitcoin-0.20.0/bin/bitcoind /bin/
+RUN cargo build && \
+    cargo test -- --test-threads 1 --ignored "$test_name"
 
-ENV BITCOIND_TEST 1
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::microblock_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::size_check_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::cost_voting_integration
-RUN cargo test -- --test-threads 1 --ignored tests::integrations::integration_test_get_info
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::bitcoind_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::liquid_ustx_integration
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::stx_transfer_btc_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::bitcoind_forking_test
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::should_fix_2771
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::pox_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::bitcoin_regtest::bitcoind_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::should_succeed_handling_malformed_and_valid_txs
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::size_overflow_unconfirmed_microblocks_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::size_overflow_unconfirmed_stream_microblocks_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::size_overflow_unconfirmed_invalid_stream_microblocks_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::runtime_overflow_unconfirmed_microblocks_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::antientropy_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::filter_low_fee_tx_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::filter_long_runtime_tx_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::mining_transactions_is_fair
+RUN grcov . --binary-path ../../target/debug/ -s ../.. -t lcov --branch --ignore-not-existing --ignore "/*" -o /lcov.info
 
+FROM scratch AS export-stage
+COPY --from=test /lcov.info /

.github/actions/bitcoin-int-tests/Dockerfile.code-cov

@@ -1,4 +1,4 @@
-FROM rust:bullseye AS build
+FROM rust:bullseye AS test
 
 WORKDIR /build
 
@@ -15,5 +15,7 @@ RUN cargo build && \
     cargo test
 
 # Generate coverage report and upload it to codecov
-RUN grcov . --binary-path ./target/debug/ -s . -t lcov --branch --ignore-not-existing --ignore "/*" -o lcov.info && \
-    bash -c "bash <(curl -s https://codecov.io/bash)"
+RUN grcov . --binary-path ./target/debug/ -s . -t lcov --branch --ignore-not-existing --ignore "/*" -o lcov.info
+
+FROM scratch AS export-stage
+COPY --from=test /build/lcov.info /

.github/actions/bitcoin-int-tests/Dockerfile.atlas-test → .github/actions/bitcoin-int-tests/Dockerfile.generic.bitcoin-tests

@@ -1,17 +1,24 @@
 FROM rust:bullseye
 
-WORKDIR /src
+WORKDIR /src/
 
 COPY . .
 
-RUN cargo test --no-run --workspace
+WORKDIR /src/testnet/stacks-node
+
+RUN rustup override set nightly && \
+    rustup component add llvm-tools-preview && \
+    cargo install grcov
+
+ENV RUSTFLAGS="-Zinstrument-coverage" \
+    LLVM_PROFILE_FILE="stacks-blockchain-%p-%m.profraw"
+
+RUN cargo test --no-run && \
+    cargo build
 
 RUN cd / && wget https://bitcoin.org/bin/bitcoin-core-0.20.0/bitcoin-0.20.0-x86_64-linux-gnu.tar.gz
 RUN cd / && tar -xvzf bitcoin-0.20.0-x86_64-linux-gnu.tar.gz
 
 RUN ln -s /bitcoin-0.20.0/bin/bitcoind /bin/
 
 ENV BITCOIND_TEST 1
-WORKDIR /src/testnet/stacks-node
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::atlas_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::atlas_stress_integration_test

.github/actions/bitcoin-int-tests/Dockerfile.large-genesis

@@ -1,15 +1,28 @@
-FROM rust:bullseye
+FROM rust:bullseye AS test
 
 WORKDIR /src
 
 COPY . .
 
-RUN cargo test --no-run --workspace
-
 RUN cd / && wget https://bitcoin.org/bin/bitcoin-core-0.20.0/bitcoin-0.20.0-x86_64-linux-gnu.tar.gz
 RUN cd / && tar -xvzf bitcoin-0.20.0-x86_64-linux-gnu.tar.gz
 
 RUN ln -s /bitcoin-0.20.0/bin/bitcoind /bin/
 
+RUN rustup override set nightly && \
+    rustup component add llvm-tools-preview && \
+    cargo install grcov
+
+ENV RUSTFLAGS="-Zinstrument-coverage" \
+    LLVM_PROFILE_FILE="stacks-blockchain-%p-%m.profraw"
+
+RUN cargo test --no-run --workspace && \
+    cargo build --workspace
+
 ENV BITCOIND_TEST 1
 RUN cd testnet/stacks-node && cargo test --release --features prod-genesis-chainstate -- --test-threads 1 --ignored neon_integrations::bitcoind_integration_test
+
+RUN grcov . --binary-path ./target/debug/ -s . -t lcov --branch --ignore-not-existing --ignore "/*" -o lcov.info
+
+FROM scratch AS export-stage
+COPY --from=test /src/lcov.info /

.github/workflows/bitcoin-tests.yml

@@ -6,22 +6,105 @@ on:
   pull_request:
 
 jobs:
-  # Run sampled genesis tests
+  build-integration-image:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Build bitcoin integration testing image
+        env:
+          DOCKER_BUILDKIT: 1
+        # Remove .dockerignore file so codecov has access to git info
+        run: |
+          rm .dockerignore
+          docker build -f ./.github/actions/bitcoin-int-tests/Dockerfile.generic.bitcoin-tests -t stacks-node:integrations .
+      - name: Export docker image as tarball
+        run: docker save -o integration-image.tar stacks-node:integrations
+      - name: Upload built docker image
+        uses: actions/upload-artifact@v2
+        with:
+          name: integration-image.tar
+          path: integration-image.tar
+  # Run integration tests using sampled genesis block
   sampled-genesis:
     runs-on: ubuntu-latest
+    needs:
+      - build-integration-image
+    strategy:
+      fail-fast: false
+      matrix:
+        test-name:
+          - tests::neon_integrations::microblock_integration_test
+          - tests::neon_integrations::size_check_integration_test
+          - tests::neon_integrations::cost_voting_integration
+          - tests::integrations::integration_test_get_info
+          - tests::neon_integrations::bitcoind_integration_test
+          - tests::neon_integrations::liquid_ustx_integration
+          - tests::neon_integrations::stx_transfer_btc_integration_test
+          - tests::neon_integrations::bitcoind_forking_test
+          - tests::neon_integrations::should_fix_2771
+          - tests::neon_integrations::pox_integration_test
+          - tests::bitcoin_regtest::bitcoind_integration_test
+          - tests::should_succeed_handling_malformed_and_valid_txs
+          - tests::neon_integrations::size_overflow_unconfirmed_microblocks_integration_test
+          - tests::neon_integrations::size_overflow_unconfirmed_stream_microblocks_integration_test
+          - tests::neon_integrations::size_overflow_unconfirmed_invalid_stream_microblocks_integration_test
+          - tests::neon_integrations::runtime_overflow_unconfirmed_microblocks_integration_test
+          - tests::neon_integrations::antientropy_integration_test
+          - tests::neon_integrations::filter_low_fee_tx_integration_test
+          - tests::neon_integrations::filter_long_runtime_tx_integration_test
+          - tests::neon_integrations::mining_transactions_is_fair
+          - tests::neon_integrations::use_latest_tip_integration_test
+          - tests::epoch_205::test_dynamic_db_method_costs
+          - tests::epoch_205::transition_empty_blocks
+          - tests::epoch_205::test_cost_limit_switch_version205
+          - tests::epoch_205::test_exact_block_costs
+          - tests::epoch_205::bigger_microblock_streams_in_2_05
     steps:
       - uses: actions/checkout@v2
+      - name: Download docker image
+        uses: actions/download-artifact@v2
+        with:
+          name: integration-image.tar
+      - name: Load docker image
+        run: docker load -i integration-image.tar && rm integration-image.tar
       - name: All integration tests with sampled genesis
+        timeout-minutes: 30
         env:
           DOCKER_BUILDKIT: 1
-        run: docker build -f ./.github/actions/bitcoin-int-tests/Dockerfile.bitcoin-tests .
+          TEST_NAME: ${{ matrix.test-name }}
+        run: docker build -o coverage-output --build-arg test_name=${{ matrix.test-name }} -f ./.github/actions/bitcoin-int-tests/Dockerfile.bitcoin-tests .
+      - uses: codecov/codecov-action@v2
+        with:
+          files: ./coverage-output/lcov.info
+          name: ${{ matrix.test-name }}
+          fail_ci_if_error: true
   atlas-test:
-    # disable this job/test for now, as the atlas endpoints are currently disabled.
     if: ${{ true }}
     runs-on: ubuntu-latest
+    needs:
+      - build-integration-image
+    strategy:
+      fail-fast: false
+      matrix:
+        test-name:
+          - tests::neon_integrations::atlas_integration_test
+          - tests::neon_integrations::atlas_stress_integration_test
     steps:
       - uses: actions/checkout@v2
-      - name: All integration tests with sampled genesis
+      - name: Download docker image
+        uses: actions/download-artifact@v2
+        with:
+          name: integration-image.tar
+      - name: Load docker image
+        run: docker load -i integration-image.tar && rm integration-image.tar
+      - name: Atlas integration tests
+        timeout-minutes: 40
         env:
           DOCKER_BUILDKIT: 1
-        run: docker build -f ./.github/actions/bitcoin-int-tests/Dockerfile.atlas-test .
+          TEST_NAME: ${{ matrix.test-name }}
+        run: docker build -o coverage-output --build-arg test_name=${{ matrix.test-name }} -f ./.github/actions/bitcoin-int-tests/Dockerfile.bitcoin-tests .
+      - uses: codecov/codecov-action@v2
+        with:
+          files: ./coverage-output/lcov.info
+          name: ${{ matrix.test-name }}
+          fail_ci_if_error: true

.github/workflows/stacks-blockchain.yml

@@ -39,7 +39,15 @@ jobs:
       - name: Single full genesis integration test
         env:
           DOCKER_BUILDKIT: 1
-        run: docker build -f ./.github/actions/bitcoin-int-tests/Dockerfile.large-genesis .
+        # Remove .dockerignore file so codecov has access to git info
+        run: |
+          rm .dockerignore
+          docker build -o coverage-output -f ./.github/actions/bitcoin-int-tests/Dockerfile.large-genesis .
+      - uses: codecov/codecov-action@v2
+        with:
+          files: ./coverage-output/lcov.info
+          name: large_genesis
+          fail_ci_if_error: true
 
   # Run unit tests with code coverage
   unit-tests:
@@ -49,7 +57,15 @@ jobs:
       - name: Run units tests (with coverage)
         env:
           DOCKER_BUILDKIT: 1
-        run: docker build -f ./.github/actions/bitcoin-int-tests/Dockerfile.code-cov .
+        # Remove .dockerignore file so codecov has access to git info
+        run: |
+          rm .dockerignore
+          docker build -o coverage-output -f ./.github/actions/bitcoin-int-tests/Dockerfile.code-cov .
+      - uses: codecov/codecov-action@v2
+        with:
+          files: ./coverage-output/lcov.info
+          name: unit_tests
+          fail_ci_if_error: true
 
   open-api-validation:
     runs-on: ubuntu-latest
@@ -93,7 +109,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        platform: [ windows-x64, macos-x64, linux-x64, linux-musl-x64, linux-armv7, linux-arm64 ]
+        platform: [ windows-x64, macos-x64, macos-arm64, linux-x64, linux-musl-x64, linux-armv7, linux-arm64 ]
 
     steps:
       - uses: actions/checkout@v2
@@ -218,7 +234,7 @@ jobs:
           tag_name: ${{ github.event.inputs.tag || github.ref }}
           release_name: Release ${{ github.event.inputs.tag || github.ref }}
           draft: false
-          prerelease: false
+          prerelease: true
 
   # Upload distributables to a new release if we're building a tag or a tag was passed in
   upload-dist:

CHANGELOG.md

@@ -5,13 +5,57 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to the versioning scheme outlined in the [README.md](README.md).
 
-## [Not yet released]
+## [2.05.0.0.0]
+
+This software update is a consensus changing release and the
+implementation of the proposed cost changes in SIP-012. This release's
+chainstate directory is compatible with chainstate directories from
+2.0.11.4.0. However, this release is only compatible with chainstate
+directories before the 2.05 consensus changes activate (Bitcoin height
+713,000). If you run a 2.00 stacks-node beyond this point, and wish to
+run a 2.05 node afterwards, you must start from a new chainstate
+directory.
+
+## Added
+
+- At height 713,000 a new `costs-2` contract will be launched by the
+  Stacks boot address.
+
+## Changed
+
+- Stacks blocks whose parents are mined >= 713,000 will use default costs
+  from the new `costs-2` contract.
+- Stacks blocks whose parents are mined >= 713,000 will use the real
+  serialized length of Clarity values as the cost inputs to several methods
+  that previously used the maximum possible size for the associated types.
+- Stacks blocks whose parents are mined >= 713,000 will use the new block
+  limit defined in SIP-012.
+
+## Fixed
+
+- Miners are now more aggressive in calculating their block limits
+  when confirming microblocks (#2916)
+
+## [2.0.11.4.0]
+
+This software update is a point-release to change the transaction
+selection logic in the default miner to prioritize by an estimated fee
+rate instead of raw fee. This release's chainstate directory is
+compatible with chainstate directories from 2.0.11.3.0.
 
 ## Added
 
 - FeeEstimator and CostEstimator interfaces. These can be controlled
   via node configuration options. See the `README.md` for more
   information on the configuration.
+- New fee rate estimation endpoint `/v2/fees/transaction` (#2872). See
+  `docs/rpc/openapi.yaml` for more information.
+
+## Changed
+
+- Prioritize transaction inclusion in blocks by estimated fee rates (#2859).
+- MARF sqlite connections will now use `mmap`'ed connections with up to 256MB
+  space (#2869).
 
 ## [2.0.11.3.0]