Merge branch 'develop' into feat/appchain-network-improvements

stacks-network · Dec 6, 2021 · 2415bbb · 2415bbb
2 parents 9d1fd43 + f225538
commit 2415bbb
Show file tree

Hide file tree

Showing 129 changed files with 15,481 additions and 1,530 deletions.
diff --git a/.github/actions/bitcoin-int-tests/Dockerfile.bitcoin-tests b/.github/actions/bitcoin-int-tests/Dockerfile.bitcoin-tests
@@ -1,36 +1,12 @@
-FROM rust:bullseye
-
-WORKDIR /src/
-
-COPY . .
-
-WORKDIR /src/testnet/stacks-node
-RUN cargo test --no-run
-
-RUN cd / && wget https://bitcoin.org/bin/bitcoin-core-0.20.0/bitcoin-0.20.0-x86_64-linux-gnu.tar.gz
-RUN cd / && tar -xvzf bitcoin-0.20.0-x86_64-linux-gnu.tar.gz
-
-RUN ln -s /bitcoin-0.20.0/bin/bitcoind /bin/
+FROM stacks-node:integrations
 
+ARG test_name
 ENV BITCOIND_TEST 1
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::microblock_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::size_check_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::cost_voting_integration
-RUN cargo test -- --test-threads 1 --ignored tests::integrations::integration_test_get_info
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::bitcoind_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::liquid_ustx_integration
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::stx_transfer_btc_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::bitcoind_forking_test
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::should_fix_2771
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::pox_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::bitcoin_regtest::bitcoind_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::should_succeed_handling_malformed_and_valid_txs
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::size_overflow_unconfirmed_microblocks_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::size_overflow_unconfirmed_stream_microblocks_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::size_overflow_unconfirmed_invalid_stream_microblocks_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::runtime_overflow_unconfirmed_microblocks_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::antientropy_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::filter_low_fee_tx_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::filter_long_runtime_tx_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::mining_transactions_is_fair
 
+RUN cargo build && \
+    cargo test -- --test-threads 1 --ignored "$test_name"
+
+RUN grcov . --binary-path ../../target/debug/ -s ../.. -t lcov --branch --ignore-not-existing --ignore "/*" -o lcov.info && \
+    curl -Os https://uploader.codecov.io/latest/linux/codecov && \
+    chmod +x codecov && \
+    ./codecov --name "$test_name"
diff --git a/.github/actions/bitcoin-int-tests/Dockerfile.code-cov b/.github/actions/bitcoin-int-tests/Dockerfile.code-cov
@@ -16,4 +16,6 @@ RUN cargo build && \
 
 # Generate coverage report and upload it to codecov
 RUN grcov . --binary-path ./target/debug/ -s . -t lcov --branch --ignore-not-existing --ignore "/*" -o lcov.info && \
-    bash -c "bash <(curl -s https://codecov.io/bash)"
+    curl -Os https://uploader.codecov.io/latest/linux/codecov && \
+    chmod +x codecov && \
+    ./codecov --name "unit_tests"
diff --git a/...s/bitcoin-int-tests/Dockerfile.atlas-test → ...nt-tests/Dockerfile.generic.bitcoin-tests b/...s/bitcoin-int-tests/Dockerfile.atlas-test → ...nt-tests/Dockerfile.generic.bitcoin-tests
@@ -1,17 +1,24 @@
 FROM rust:bullseye
 
-WORKDIR /src
+WORKDIR /src/
 
 COPY . .
 
-RUN cargo test --no-run --workspace
+WORKDIR /src/testnet/stacks-node
+
+RUN rustup override set nightly && \
+    rustup component add llvm-tools-preview && \
+    cargo install grcov
+
+ENV RUSTFLAGS="-Zinstrument-coverage" \
+    LLVM_PROFILE_FILE="stacks-blockchain-%p-%m.profraw"
+
+RUN cargo test --no-run && \
+    cargo build
 
 RUN cd / && wget https://bitcoin.org/bin/bitcoin-core-0.20.0/bitcoin-0.20.0-x86_64-linux-gnu.tar.gz
 RUN cd / && tar -xvzf bitcoin-0.20.0-x86_64-linux-gnu.tar.gz
 
 RUN ln -s /bitcoin-0.20.0/bin/bitcoind /bin/
 
 ENV BITCOIND_TEST 1
-WORKDIR /src/testnet/stacks-node
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::atlas_integration_test
-RUN cargo test -- --test-threads 1 --ignored tests::neon_integrations::atlas_stress_integration_test
diff --git a/.github/actions/bitcoin-int-tests/Dockerfile.large-genesis b/.github/actions/bitcoin-int-tests/Dockerfile.large-genesis
@@ -4,12 +4,25 @@ WORKDIR /src
 
 COPY . .
 
-RUN cargo test --no-run --workspace
-
 RUN cd / && wget https://bitcoin.org/bin/bitcoin-core-0.20.0/bitcoin-0.20.0-x86_64-linux-gnu.tar.gz
 RUN cd / && tar -xvzf bitcoin-0.20.0-x86_64-linux-gnu.tar.gz
 
 RUN ln -s /bitcoin-0.20.0/bin/bitcoind /bin/
 
+RUN rustup override set nightly && \
+    rustup component add llvm-tools-preview && \
+    cargo install grcov
+
+ENV RUSTFLAGS="-Zinstrument-coverage" \
+    LLVM_PROFILE_FILE="stacks-blockchain-%p-%m.profraw"
+
+RUN cargo test --no-run --workspace && \
+    cargo build --workspace
+
 ENV BITCOIND_TEST 1
 RUN cd testnet/stacks-node && cargo test --release --features prod-genesis-chainstate -- --test-threads 1 --ignored neon_integrations::bitcoind_integration_test
+
+RUN grcov . --binary-path ./target/debug/ -s . -t lcov --branch --ignore-not-existing --ignore "/*" -o lcov.info && \
+    curl -Os https://uploader.codecov.io/latest/linux/codecov && \
+    chmod +x codecov && \
+    ./codecov --name "large_genesis"
diff --git a/.github/workflows/bitcoin-tests.yml b/.github/workflows/bitcoin-tests.yml
@@ -6,22 +6,94 @@ on:
   pull_request:
 
 jobs:
-  # Run sampled genesis tests
+  build-integration-image:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Build bitcoin integration testing image
+        env:
+          DOCKER_BUILDKIT: 1
+        # Remove .dockerignore file so codecov has access to git info
+        run: |
+          rm .dockerignore
+          docker build -f ./.github/actions/bitcoin-int-tests/Dockerfile.generic.bitcoin-tests -t stacks-node:integrations .
+      - name: Export docker image as tarball
+        run: docker save -o integration-image.tar stacks-node:integrations
+      - name: Upload built docker image
+        uses: actions/upload-artifact@v2
+        with:
+          name: integration-image.tar
+          path: integration-image.tar
+  # Run integration tests using sampled genesis block
   sampled-genesis:
     runs-on: ubuntu-latest
+    needs:
+      - build-integration-image
+    strategy:
+      fail-fast: false
+      matrix:
+        test-name:
+          - tests::neon_integrations::microblock_integration_test
+          - tests::neon_integrations::size_check_integration_test
+          - tests::neon_integrations::cost_voting_integration
+          - tests::integrations::integration_test_get_info
+          - tests::neon_integrations::bitcoind_integration_test
+          - tests::neon_integrations::liquid_ustx_integration
+          - tests::neon_integrations::stx_transfer_btc_integration_test
+          - tests::neon_integrations::bitcoind_forking_test
+          - tests::neon_integrations::should_fix_2771
+          - tests::neon_integrations::pox_integration_test
+          - tests::bitcoin_regtest::bitcoind_integration_test
+          - tests::should_succeed_handling_malformed_and_valid_txs
+          - tests::neon_integrations::size_overflow_unconfirmed_microblocks_integration_test
+          - tests::neon_integrations::size_overflow_unconfirmed_stream_microblocks_integration_test
+          - tests::neon_integrations::size_overflow_unconfirmed_invalid_stream_microblocks_integration_test
+          - tests::neon_integrations::runtime_overflow_unconfirmed_microblocks_integration_test
+          - tests::neon_integrations::antientropy_integration_test
+          - tests::neon_integrations::filter_low_fee_tx_integration_test
+          - tests::neon_integrations::filter_long_runtime_tx_integration_test
+          - tests::neon_integrations::mining_transactions_is_fair
+          - tests::epoch_205::test_dynamic_db_method_costs
+          - tests::epoch_205::transition_empty_blocks
+          - tests::epoch_205::test_cost_limit_switch_version205
+          - tests::epoch_205::test_exact_block_costs
+          - tests::epoch_205::bigger_microblock_streams_in_2_05
     steps:
       - uses: actions/checkout@v2
+      - name: Download docker image
+        uses: actions/download-artifact@v2
+        with:
+          name: integration-image.tar
+      - name: Load docker image
+        run: docker load -i integration-image.tar && rm integration-image.tar
       - name: All integration tests with sampled genesis
+        timeout-minutes: 30
         env:
           DOCKER_BUILDKIT: 1
-        run: docker build -f ./.github/actions/bitcoin-int-tests/Dockerfile.bitcoin-tests .
+          TEST_NAME: ${{ matrix.test-name }}
+        run: docker build --build-arg test_name=${{ matrix.test-name }} -f ./.github/actions/bitcoin-int-tests/Dockerfile.bitcoin-tests .
   atlas-test:
-    # disable this job/test for now, as the atlas endpoints are currently disabled.
     if: ${{ true }}
     runs-on: ubuntu-latest
+    needs:
+      - build-integration-image
+    strategy:
+      fail-fast: false
+      matrix:
+        test-name:
+          - tests::neon_integrations::atlas_integration_test
+          - tests::neon_integrations::atlas_stress_integration_test
     steps:
       - uses: actions/checkout@v2
-      - name: All integration tests with sampled genesis
+      - name: Download docker image
+        uses: actions/download-artifact@v2
+        with:
+          name: integration-image.tar
+      - name: Load docker image
+        run: docker load -i integration-image.tar && rm integration-image.tar
+      - name: Atlas integration tests
+        timeout-minutes: 40
         env:
           DOCKER_BUILDKIT: 1
-        run: docker build -f ./.github/actions/bitcoin-int-tests/Dockerfile.atlas-test .
+          TEST_NAME: ${{ matrix.test-name }}
+        run: docker build --build-arg test_name=${{ matrix.test-name }} -f ./.github/actions/bitcoin-int-tests/Dockerfile.bitcoin-tests .
diff --git a/.github/workflows/stacks-blockchain.yml b/.github/workflows/stacks-blockchain.yml
@@ -39,7 +39,10 @@ jobs:
       - name: Single full genesis integration test
         env:
           DOCKER_BUILDKIT: 1
-        run: docker build -f ./.github/actions/bitcoin-int-tests/Dockerfile.large-genesis .
+        # Remove .dockerignore file so codecov has access to git info
+        run: |
+          rm .dockerignore
+          docker build -f ./.github/actions/bitcoin-int-tests/Dockerfile.large-genesis .
 
   # Run unit tests with code coverage
   unit-tests:
@@ -49,7 +52,10 @@ jobs:
       - name: Run units tests (with coverage)
         env:
           DOCKER_BUILDKIT: 1
-        run: docker build -f ./.github/actions/bitcoin-int-tests/Dockerfile.code-cov .
+        # Remove .dockerignore file so codecov has access to git info
+        run: |
+          rm .dockerignore
+          docker build -f ./.github/actions/bitcoin-int-tests/Dockerfile.code-cov .
 
   open-api-validation:
     runs-on: ubuntu-latest
@@ -218,7 +224,7 @@ jobs:
           tag_name: ${{ github.event.inputs.tag || github.ref }}
           release_name: Release ${{ github.event.inputs.tag || github.ref }}
           draft: false
-          prerelease: false
+          prerelease: true
 
   # Upload distributables to a new release if we're building a tag or a tag was passed in
   upload-dist:

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,7 +5,59 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to the versioning scheme outlined in the [README.md](README.md).
 
-## [Not Yet Released]
+## [2.05.0.0.0]
+
+This software update is a consensus changing release and the
+implementation of the proposed cost changes in SIP-012. This release's
+chainstate directory is compatible with chainstate directories from
+2.0.11.4.0. However, this release is only compatible with chainstate
+directories before the 2.05 consensus changes activate (Bitcoin height
+713,000). If you run a 2.00 stacks-node beyond this point, and wish to
+run a 2.05 node afterwards, you must start from a new chainstate
+directory.
+
+## Added
+
+- At height 713,000 a new `costs-2` contract will be launched by the
+  Stacks boot address.
+
+## Changed
+
+- Stacks blocks whose parents are mined >= 713,000 will use default costs
+  from the new `costs-2` contract.
+- Stacks blocks whose parents are mined >= 713,000 will use the real
+  serialized length of Clarity values as the cost inputs to several methods
+  that previously used the maximum possible size for the associated types.
+- Stacks blocks whose parents are mined >= 713,000 will use the new block
+  limit defined in SIP-012.
+
+## Fixed
+
+- Miners are now more aggressive in calculating their block limits
+  when confirming microblocks (#2916)
+
+## [2.0.11.4.0]
+
+This software update is a point-release to change the transaction
+selection logic in the default miner to prioritize by an estimated fee
+rate instead of raw fee. This release's chainstate directory is
+compatible with chainstate directories from 2.0.11.3.0.
+
+## Added
+
+- FeeEstimator and CostEstimator interfaces. These can be controlled
+  via node configuration options. See the `README.md` for more
+  information on the configuration.
+- New fee rate estimation endpoint `/v2/fees/transaction` (#2872). See
+  `docs/rpc/openapi.yaml` for more information.
+
+## Changed
+
+- Prioritize transaction inclusion in blocks by estimated fee rates (#2859).
+- MARF sqlite connections will now use `mmap`'ed connections with up to 256MB
+  space (#2869).
+
+## [2.0.11.3.0]
 
 This software update is a point-release to change the transaction selection
 logic in the default miner to prioritize by fee instead of nonce sequence.  This