From 43b2ca2dd413127cf7f8802e7cf9fc71e69f919c Mon Sep 17 00:00:00 2001 From: Dan Trevino Date: Wed, 25 Mar 2020 19:42:57 -0700 Subject: [PATCH 01/12] add arm build options and instructions --- Cargo.toml | 16 ++-- README-CROSS-COMPILE.md | 188 ++++++++++++++++++++++++++++++++++++++++ README.md | 10 ++- 3 files changed, 205 insertions(+), 9 deletions(-) create mode 100644 README-CROSS-COMPILE.md diff --git a/Cargo.toml b/Cargo.toml index d7d06f3744..266878fb36 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,10 +29,6 @@ path = "src/blockstack_cli.rs" name = "marf_bench" harness = false -[features] -developer-mode = [] -default = ["developer-mode"] - [dependencies] byteorder = "1.1" rust-ini = "0.13" @@ -46,6 +42,8 @@ regex = "1" mio = "0.6.16" libc = "0.2" lazy_static = "1.4.0" +sha2 = { version = "0.8.0", optional = true } +sha2-asm = { version="0.5.3", optional = true } [dependencies.secp256k1] version = "0.11.5" @@ -63,10 +61,6 @@ features = ["serde"] version = "=2.0.0" features = ["serde"] -[dependencies.sha2] -version = "0.8.0" -features = ["asm"] - [dependencies.time] version = "0.2.1" features = ["std"] @@ -74,3 +68,9 @@ features = ["std"] [dev-dependencies] assert-json-diff = "1.0.0" criterion = "0.3" + +[features] +developer-mode = [] +asm = ["sha2", "sha2-asm"] +aarch64 = ["developer-mode", "sha2"] +default = ["developer-mode", "asm"] \ No newline at end of file diff --git a/README-CROSS-COMPILE.md b/README-CROSS-COMPILE.md new file mode 100644 index 0000000000..68402a2321 --- /dev/null +++ b/README-CROSS-COMPILE.md @@ -0,0 +1,188 @@ +# Cross-Compiling Stacks 2.0 for Raspberry Pi + +## Notice + +The instructions below describe how to build the Blockstack Stacks v2.0 Blockchain for Raspberry Pi 2, 3, or the 1GB RAM model Pi 4, with `cross`. Raspberry Pi 1 and Raspberry Pi Zero are not supported by these instructions. Note also that by default the build is a developer-mode build. + +For instructions on building directly on Raspberry Pi 4, with >= 2GB of RAM, see the [README.md](README.md). + +More about cross: https://github.com/rust-embedded/cross + +## Getting started + +### Configure local environment + +The first step is to ensure that you have Rust and the support software installed. + +```bash +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh +cargo install cross +``` + +Follow the instructions for installing Docker, including post-install steps: + +##### Install Docker + +https://docs.docker.com/install/ + +##### Docker Post-install + +https://docs.docker.com/install/linux/linux-postinstall/ + +### Download and build stacks-blockchain + +From there, you can clone this repository: + +```bash +git clone https://github.com/blockstack/stacks-blockchain.git + +cd stacks-blockchain +``` + +Then build the project: + +#### Build with cross + +```bash +cross build --target arm-unknown-linux-gnueabihf +``` + +You now have binaries that will run on Raspberry pi in your "target" directory. Copy the `stacks-blockchain/target/arm-unknown-linux-gnueabihf/debug` directory to your Raspberry Pi before continuing. + +## On your Raspberry Pi + +```bash +cd +``` + +### Encode and sign transactions + +Let's start by generating a keypair, that will be used for signing the upcoming transactions: + +```bash +./blockstack-cli generate-sk --testnet + +# Output +# { +# secretKey: "b8d99fd45da58038d630d9855d3ca2466e8e0f89d3894c4724f0efc9ff4b51f001", +# publicKey: "02781d2d3a545afdb7f6013a8241b9e400475397516a0d0f76863c6742210539b5", +# stacksAddress: "ST2ZRX0K27GW0SP3GJCEMHD95TQGJMKB7G9Y0X1MH" +# } +``` + +We will interact with the following simple contract `kv-store`. In our examples, we will assume this contract is saved to `./kv-store.clar`: + +```scheme +(define-map store ((key (buff 32))) ((value (buff 32)))) + +(define-public (get-value (key (buff 32))) + (match (map-get? store ((key key))) + entry (ok (get value entry)) + (err 0))) + +(define-public (set-value (key (buff 32)) (value (buff 32))) + (begin + (map-set store ((key key)) ((value value))) + (ok 'true))) +``` + +We want to publish this contract on chain, then issue some transactions that interact with it by setting some keys and getting some values, so we can observe read and writes. + +Our first step is to generate and sign, using your private key, the transaction that will publish the contract `kv-store`. +To do that, we will use the subcommand: + +```bash +./blockstack-cli publish --help +``` + +With the following arguments: + +```bash +./blockstack-cli publish b8d99fd45da58038d630d9855d3ca2466e8e0f89d3894c4724f0efc9ff4b51f001 0 0 kv-store ./kv-store.clar --testnet +``` + +This command will output the **binary format** of the transaction. In our case, we want to pipe this output and dump it to a file that will be used later in this tutorial. + +```bash +./blockstack-cli publish b8d99fd45da58038d630d9855d3ca2466e8e0f89d3894c4724f0efc9ff4b51f001 0 0 kv-store ./kv-store.clar --testnet | xxd -r -p > tx1.bin +``` + +### Run the testnet + +You can observe the state machine in action locally by running: + +```bash +./blockstack-core testnet +``` + +In your console, you should observe an output with a similar: + +```bash +*** mempool path: /tmp/stacks-testnet-5fc814cf78dc0636/L1/mempool +``` + +The testnet is watching this directory, decoding and ingesting the transactions materialized as files. This mechanism is a shortcut for simulating a mempool. A RPC server will soon be integrated. + +### Publish your contract + +Assuming that the testnet is running, we can publish our `kv-store` contract. + +In another terminal (or file explorer), you can move the `tx1.bin` generated earlier, to the mempool: + +```bash +cp ./tx1.bin /tmp/stacks-testnet-5fc814cf78dc0636/L1/mempool +``` + +In the terminal window running the testnet, you can observe the state machine's reactions. + +### Reading from / Writing to the contract + +Now that our contract has been published on chain, let's try to submit some read / write transactions. +We will start by trying to read the value associated with the key `foo`. + +To do that, we will use the subcommand: + +```bash +./blockstack-cli contract-call --help +``` + +With the following arguments: + +```bash +./blockstack-cli contract-call b8d99fd45da58038d630d9855d3ca2466e8e0f89d3894c4724f0efc9ff4b51f001 0 1 ST2ZRX0K27GW0SP3GJCEMHD95TQGJMKB7G9Y0X1MH kv-store get-value -e \"foo\" --testnet | xxd -r -p > tx2.bin +``` + +`contract-call` generates and signs a contract-call transaction. +Note: the third argument `1` is a nonce, that must be increased monotonically with each new transaction. + +We can submit the transaction by moving it to the mempool path: + +```bash +cp ./tx2.bin /tmp/stacks-testnet-5fc814cf78dc0636/L1/mempool +``` + +Similarly, we can generate a transaction that would be setting the key `foo` to the value `bar`: + +```bash +./blockstack-cli contract-call b8d99fd45da58038d630d9855d3ca2466e8e0f89d3894c4724f0efc9ff4b51f001 0 2 ST2ZRX0K27GW0SP3GJCEMHD95TQGJMKB7G9Y0X1MH kv-store set-value -e \"foo\" -e \"bar\" --testnet | xxd -r -p > tx3.bin +``` + +And submit it by moving it to the mempool path: + +```bash +cp ./tx3.bin /tmp/stacks-testnet-5fc814cf78dc0636/L1/mempool +``` + +Finally, we can issue a third transaction, reading the key `foo` again, for ensuring that the previous transaction has successfully updated the state machine: + +```bash +./blockstack-cli contract-call b8d99fd45da58038d630d9855d3ca2466e8e0f89d3894c4724f0efc9ff4b51f001 0 3 ST2ZRX0K27GW0SP3GJCEMHD95TQGJMKB7G9Y0X1MH kv-store get-value -e \"foo\" --testnet | xxd -r -p > tx4.bin +``` + +And submit this last transaction by moving it to the mempool path: + +```bash +cp ./tx4.bin /tmp/stacks-testnet-5fc814cf78dc0636/L1/mempool +``` + +Congratulations, you can now [write your own smart contracts with Clarity](https://docs.blockstack.org/core/smart/overview.html). See the [README](https://github.com/blockstack/stacks-blockchain/blob/master/README.md#community) for further documentation and options community links. diff --git a/README.md b/README.md index e95bf470cc..bd7aa9523c 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,15 @@ Then build the project: cargo build ``` -And run the tests: +Building the project on ARM: +```bash +cargo build --features "aarch64" --no-default-features +``` + +Note that the build will consistently fail on memory-constrained systems like Raspberry Pi 3, and 1GB model Pi 4s. In order to cross-compile for lower memory target systems, please see the [Cross Compiling](README-CROSS-COMPILE.md) document. + +s +Run the tests: ```bash cargo test testnet -- --test-threads=1 From 20755bd0089b71f192205173291ff27422c94e00 Mon Sep 17 00:00:00 2001 From: Dan Trevino Date: Sat, 28 Mar 2020 10:21:41 -0700 Subject: [PATCH 02/12] Move cross-compile instructions to community repo --- README-CROSS-COMPILE.md | 188 ---------------------------------------- README.md | 31 ++++--- 2 files changed, 15 insertions(+), 204 deletions(-) delete mode 100644 README-CROSS-COMPILE.md diff --git a/README-CROSS-COMPILE.md b/README-CROSS-COMPILE.md deleted file mode 100644 index 68402a2321..0000000000 --- a/README-CROSS-COMPILE.md +++ /dev/null @@ -1,188 +0,0 @@ -# Cross-Compiling Stacks 2.0 for Raspberry Pi - -## Notice - -The instructions below describe how to build the Blockstack Stacks v2.0 Blockchain for Raspberry Pi 2, 3, or the 1GB RAM model Pi 4, with `cross`. Raspberry Pi 1 and Raspberry Pi Zero are not supported by these instructions. Note also that by default the build is a developer-mode build. - -For instructions on building directly on Raspberry Pi 4, with >= 2GB of RAM, see the [README.md](README.md). - -More about cross: https://github.com/rust-embedded/cross - -## Getting started - -### Configure local environment - -The first step is to ensure that you have Rust and the support software installed. - -```bash -curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -cargo install cross -``` - -Follow the instructions for installing Docker, including post-install steps: - -##### Install Docker - -https://docs.docker.com/install/ - -##### Docker Post-install - -https://docs.docker.com/install/linux/linux-postinstall/ - -### Download and build stacks-blockchain - -From there, you can clone this repository: - -```bash -git clone https://github.com/blockstack/stacks-blockchain.git - -cd stacks-blockchain -``` - -Then build the project: - -#### Build with cross - -```bash -cross build --target arm-unknown-linux-gnueabihf -``` - -You now have binaries that will run on Raspberry pi in your "target" directory. Copy the `stacks-blockchain/target/arm-unknown-linux-gnueabihf/debug` directory to your Raspberry Pi before continuing. - -## On your Raspberry Pi - -```bash -cd -``` - -### Encode and sign transactions - -Let's start by generating a keypair, that will be used for signing the upcoming transactions: - -```bash -./blockstack-cli generate-sk --testnet - -# Output -# { -# secretKey: "b8d99fd45da58038d630d9855d3ca2466e8e0f89d3894c4724f0efc9ff4b51f001", -# publicKey: "02781d2d3a545afdb7f6013a8241b9e400475397516a0d0f76863c6742210539b5", -# stacksAddress: "ST2ZRX0K27GW0SP3GJCEMHD95TQGJMKB7G9Y0X1MH" -# } -``` - -We will interact with the following simple contract `kv-store`. In our examples, we will assume this contract is saved to `./kv-store.clar`: - -```scheme -(define-map store ((key (buff 32))) ((value (buff 32)))) - -(define-public (get-value (key (buff 32))) - (match (map-get? store ((key key))) - entry (ok (get value entry)) - (err 0))) - -(define-public (set-value (key (buff 32)) (value (buff 32))) - (begin - (map-set store ((key key)) ((value value))) - (ok 'true))) -``` - -We want to publish this contract on chain, then issue some transactions that interact with it by setting some keys and getting some values, so we can observe read and writes. - -Our first step is to generate and sign, using your private key, the transaction that will publish the contract `kv-store`. -To do that, we will use the subcommand: - -```bash -./blockstack-cli publish --help -``` - -With the following arguments: - -```bash -./blockstack-cli publish b8d99fd45da58038d630d9855d3ca2466e8e0f89d3894c4724f0efc9ff4b51f001 0 0 kv-store ./kv-store.clar --testnet -``` - -This command will output the **binary format** of the transaction. In our case, we want to pipe this output and dump it to a file that will be used later in this tutorial. - -```bash -./blockstack-cli publish b8d99fd45da58038d630d9855d3ca2466e8e0f89d3894c4724f0efc9ff4b51f001 0 0 kv-store ./kv-store.clar --testnet | xxd -r -p > tx1.bin -``` - -### Run the testnet - -You can observe the state machine in action locally by running: - -```bash -./blockstack-core testnet -``` - -In your console, you should observe an output with a similar: - -```bash -*** mempool path: /tmp/stacks-testnet-5fc814cf78dc0636/L1/mempool -``` - -The testnet is watching this directory, decoding and ingesting the transactions materialized as files. This mechanism is a shortcut for simulating a mempool. A RPC server will soon be integrated. - -### Publish your contract - -Assuming that the testnet is running, we can publish our `kv-store` contract. - -In another terminal (or file explorer), you can move the `tx1.bin` generated earlier, to the mempool: - -```bash -cp ./tx1.bin /tmp/stacks-testnet-5fc814cf78dc0636/L1/mempool -``` - -In the terminal window running the testnet, you can observe the state machine's reactions. - -### Reading from / Writing to the contract - -Now that our contract has been published on chain, let's try to submit some read / write transactions. -We will start by trying to read the value associated with the key `foo`. - -To do that, we will use the subcommand: - -```bash -./blockstack-cli contract-call --help -``` - -With the following arguments: - -```bash -./blockstack-cli contract-call b8d99fd45da58038d630d9855d3ca2466e8e0f89d3894c4724f0efc9ff4b51f001 0 1 ST2ZRX0K27GW0SP3GJCEMHD95TQGJMKB7G9Y0X1MH kv-store get-value -e \"foo\" --testnet | xxd -r -p > tx2.bin -``` - -`contract-call` generates and signs a contract-call transaction. -Note: the third argument `1` is a nonce, that must be increased monotonically with each new transaction. - -We can submit the transaction by moving it to the mempool path: - -```bash -cp ./tx2.bin /tmp/stacks-testnet-5fc814cf78dc0636/L1/mempool -``` - -Similarly, we can generate a transaction that would be setting the key `foo` to the value `bar`: - -```bash -./blockstack-cli contract-call b8d99fd45da58038d630d9855d3ca2466e8e0f89d3894c4724f0efc9ff4b51f001 0 2 ST2ZRX0K27GW0SP3GJCEMHD95TQGJMKB7G9Y0X1MH kv-store set-value -e \"foo\" -e \"bar\" --testnet | xxd -r -p > tx3.bin -``` - -And submit it by moving it to the mempool path: - -```bash -cp ./tx3.bin /tmp/stacks-testnet-5fc814cf78dc0636/L1/mempool -``` - -Finally, we can issue a third transaction, reading the key `foo` again, for ensuring that the previous transaction has successfully updated the state machine: - -```bash -./blockstack-cli contract-call b8d99fd45da58038d630d9855d3ca2466e8e0f89d3894c4724f0efc9ff4b51f001 0 3 ST2ZRX0K27GW0SP3GJCEMHD95TQGJMKB7G9Y0X1MH kv-store get-value -e \"foo\" --testnet | xxd -r -p > tx4.bin -``` - -And submit this last transaction by moving it to the mempool path: - -```bash -cp ./tx4.bin /tmp/stacks-testnet-5fc814cf78dc0636/L1/mempool -``` - -Congratulations, you can now [write your own smart contracts with Clarity](https://docs.blockstack.org/core/smart/overview.html). See the [README](https://github.com/blockstack/stacks-blockchain/blob/master/README.md#community) for further documentation and options community links. diff --git a/README.md b/README.md index bd7aa9523c..ae125eea7e 100644 --- a/README.md +++ b/README.md @@ -6,14 +6,14 @@ Reference implementation of https://blockstack.org/whitepaper.pdf is in Rust. ## Repository -| Blockstack Topic/Tech | Where to learn more more | -|---------------------------------|------------------------------------------------------------------------------| -| Stacks 2.0 | [master branch](https://github.com/blockstack/blockstack-core/tree/master) | -| Stacks 1.0 | [legacy branch](https://github.com/blockstack/blockstack-core/tree/stacks-1.0) | -| Use the package | [our core docs](https://docs.blockstack.org/core/naming/introduction.html) | -| Develop a Blockstack App | [our developer docs](https://docs.blockstack.org/browser/hello-blockstack.html) | -| Use a Blockstack App | [our browser docs](https://docs.blockstack.org/browser/browser-introduction.html) | -| Blockstack the company | [our website](https://blockstack.org) | +| Blockstack Topic/Tech | Where to learn more more | +| ------------------------ | --------------------------------------------------------------------------------- | +| Stacks 2.0 | [master branch](https://github.com/blockstack/blockstack-core/tree/master) | +| Stacks 1.0 | [legacy branch](https://github.com/blockstack/blockstack-core/tree/stacks-1.0) | +| Use the package | [our core docs](https://docs.blockstack.org/core/naming/introduction.html) | +| Develop a Blockstack App | [our developer docs](https://docs.blockstack.org/browser/hello-blockstack.html) | +| Use a Blockstack App | [our browser docs](https://docs.blockstack.org/browser/browser-introduction.html) | +| Blockstack the company | [our website](https://blockstack.org) | ## Design Thesis @@ -21,17 +21,16 @@ Stacks 2.0 is an open-membership replicated state machine produced by the coordi To unpack this definition: -- A replicated state machine is two or more copies (“replicas”) of a given set of rules (a “machine”) that, in processing a common input (such as the same sequence of transactions), will arrive at the same configuration (“state”). Bitcoin is a replicated state machine — its state is the set of UTXOs, which each peer has a full copy of, and given a block, all peers will independently calculate the same new UTXO set from the existing one. +- A replicated state machine is two or more copies (“replicas”) of a given set of rules (a “machine”) that, in processing a common input (such as the same sequence of transactions), will arrive at the same configuration (“state”). Bitcoin is a replicated state machine — its state is the set of UTXOs, which each peer has a full copy of, and given a block, all peers will independently calculate the same new UTXO set from the existing one. - Open-membership means that any host on the Internet can join the blockchain and independently calculate the same full replica as all other peers. -- Non-enumerable means that the set of peers that are producing the blocks don’t know about one another — they don’t know their identities, or even how many exist and are online. They are indistinguishable. +- Non-enumerable means that the set of peers that are producing the blocks don’t know about one another — they don’t know their identities, or even how many exist and are online. They are indistinguishable. ## Roadmap - [x] [SIP 001: Burn Election](https://github.com/blockstack/blockstack-core/blob/master/sip/sip-001-burn-election.md) - [x] [SIP 002: Clarity, a language for predictable smart contracts](https://github.com/blockstack/blockstack-core/blob/master/sip/sip-002-smart-contract-language.md) - [x] [SIP 004: Cryptographic Committment to Materialized Views](https://github.com/blockstack/blockstack-core/blob/master/sip/sip-004-materialized-view.md) -- [x] [SIP 005: Blocks, Transactions, and Accounts](https://github.com/blockstack/blockstack-core/blob/master/sip/sip-005-blocks-and-transactions.md -) +- [x] [SIP 005: Blocks, Transactions, and Accounts](https://github.com/blockstack/blockstack-core/blob/master/sip/sip-005-blocks-and-transactions.md) - [ ] [SIP 003: Peer Network](https://github.com/blockstack/blockstack-core/blob/master/sip/sip-003-peer-network.md) (Q1 2020) - [ ] SIP 006: Clarity Execution Cost Assessment (Q1 2020) @@ -72,13 +71,13 @@ cargo build ``` Building the project on ARM: + ```bash cargo build --features "aarch64" --no-default-features ``` -Note that the build will consistently fail on memory-constrained systems like Raspberry Pi 3, and 1GB model Pi 4s. In order to cross-compile for lower memory target systems, please see the [Cross Compiling](README-CROSS-COMPILE.md) document. +For help building on memory-constrained devices, please see the community supported documentation here: [Cross Compiling](https://github.com/dantrevino/cross-compiling-stacks-blockchain/blob/master/README.md). -s Run the tests: ```bash @@ -221,12 +220,12 @@ Congratulations, you can now [write your own smart contracts with Clarity](https Beyond this Github project, Blockstack maintains a public [forum](https://forum.blockstack.org) and an -opened [Discord](https://discordapp.com/invite/9r94Xkj) channel. In addition, the project +opened [Discord](https://discordapp.com/invite/9r94Xkj) channel. In addition, the project maintains a [mailing list](https://blockstack.org/signup) which sends out community announcements. The greater Blockstack community regularly hosts in-person -[meetups](https://www.meetup.com/topics/blockstack/). The project's +[meetups](https://www.meetup.com/topics/blockstack/). The project's [YouTube channel](https://www.youtube.com/channel/UC3J2iHnyt2JtOvtGVf_jpHQ) includes videos from some of these meetups, as well as video tutorials to help new users get started and help developers wrap their heads around the system's From 5628964427849580add7c24bd9d6749413769e10 Mon Sep 17 00:00:00 2001 From: Dan Trevino Date: Sat, 28 Mar 2020 11:47:32 -0700 Subject: [PATCH 03/12] Update README.md Move community cross-compiling instructions to 'Community' section --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ae125eea7e..ca2d24c788 100644 --- a/README.md +++ b/README.md @@ -76,8 +76,6 @@ Building the project on ARM: cargo build --features "aarch64" --no-default-features ``` -For help building on memory-constrained devices, please see the community supported documentation here: [Cross Compiling](https://github.com/dantrevino/cross-compiling-stacks-blockchain/blob/master/README.md). - Run the tests: ```bash @@ -231,6 +229,8 @@ videos from some of these meetups, as well as video tutorials to help new users get started and help developers wrap their heads around the system's design. +For help cross-compiling on memory-constrained devices, please see the community supported documentation here: [Cross Compiling](https://github.com/dantrevino/cross-compiling-stacks-blockchain/blob/master/README.md). + ## Further Reading You can learn more by visiting [the Blockstack Website](https://blockstack.org) and checking out the in-depth articles and documentation: From 3c771be6a7f1cb98a26cfbb2473e5b51d21e47f7 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Tue, 31 Mar 2020 21:21:10 -0400 Subject: [PATCH 04/12] When processing an invalid block, do not short-circuit on error. This is not a fatal error in block processing. --- src/chainstate/stacks/db/blocks.rs | 45 +++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/src/chainstate/stacks/db/blocks.rs b/src/chainstate/stacks/db/blocks.rs index 7dff7a7c78..8de23c8dab 100644 --- a/src/chainstate/stacks/db/blocks.rs +++ b/src/chainstate/stacks/db/blocks.rs @@ -2715,21 +2715,39 @@ impl StacksChainState { for i in 0..max_blocks { // process up to max_blocks pending blocks - let (next_tip_opt, next_microblock_poison_opt) = self.process_next_staging_block()?; - match next_tip_opt { - Some(next_tip) => { - ret.push((Some(next_tip), next_microblock_poison_opt)); - }, - None => { - match next_microblock_poison_opt { - Some(poison) => { - ret.push((None, Some(poison))); - }, - None => { - debug!("No more staging blocks -- processed {} in total", i); - break; + match self.process_next_staging_block() { + Ok((next_tip_opt, next_microblock_poison_opt)) => match next_tip_opt { + Some(next_tip) => { + ret.push((Some(next_tip), next_microblock_poison_opt)); + }, + None => { + match next_microblock_poison_opt { + Some(poison) => { + ret.push((None, Some(poison))); + }, + None => { + debug!("No more staging blocks -- processed {} in total", i); + break; + } } } + }, + Err(Error::InvalidStacksBlock(msg)) => { + warn!("Encountered invalid block: {}", &msg); + continue; + }, + Err(Error::InvalidStacksMicroblock(msg, hash)) => { + warn!("Encountered invalid microblock {}: {}", hash, &msg); + continue; + }, + Err(Error::NetError(net_error::DeserializeError(msg))) => { + // happens if we load a zero-sized block (i.e. an invalid block) + warn!("Encountered invalid block: {}", &msg); + continue; + }, + Err(e) => { + error!("Unrecoverable error when processing blocks: {:?}", &e); + return Err(e); } } } @@ -2743,6 +2761,7 @@ impl StacksChainState { } } + block_tx.commit().map_err(|e| Error::DBError(e))?; Ok(ret) } From 449554a5be9a45c713419648174d4602ed9a8cb9 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Wed, 1 Apr 2020 11:55:13 -0400 Subject: [PATCH 05/12] tweak the test miner so we can mine invalid blocks but still deduce which block to build them off of (since they won't be accepted). Back-port test for mining invalid blocks to test the new error-handling path in process_blocks() --- src/burnchains/mod.rs | 5 +- src/chainstate/stacks/miner.rs | 146 ++++++++++++++++++++++++++++----- 2 files changed, 127 insertions(+), 24 deletions(-) diff --git a/src/burnchains/mod.rs b/src/burnchains/mod.rs index d4fcaa762f..7080254106 100644 --- a/src/burnchains/mod.rs +++ b/src/burnchains/mod.rs @@ -694,10 +694,9 @@ pub mod test { let last_snapshot_with_sortition = match parent_block_snapshot { Some(sn) => sn.clone(), - None => BurnDB::get_last_snapshot_with_sortition(tx, self.block_height - 1, &self.parent_snapshot.burn_header_hash) - .expect("FATAL: failed to read last snapshot with sortition") + None => BurnDB::get_first_block_snapshot(tx).unwrap() }; - + // prove on the last-ever sortition's hash to produce the new seed let proof = miner.make_proof(&leader_key.public_key, &last_snapshot.sortition_hash) .expect(&format!("FATAL: no private key for {}", leader_key.public_key.to_hex())); diff --git a/src/chainstate/stacks/miner.rs b/src/chainstate/stacks/miner.rs index 11ed13bb7b..6ae15b7f09 100644 --- a/src/chainstate/stacks/miner.rs +++ b/src/chainstate/stacks/miner.rs @@ -137,6 +137,40 @@ impl StacksBlockBuilder { Ok(()) } + /// Append a transaction if doing so won't exceed the epoch data size. + /// Does not check for errors + #[cfg(test)] + pub fn force_mine_tx<'a>(&mut self, clarity_tx: &mut ClarityTx<'a>, tx: &StacksTransaction) -> Result<(), Error> { + let mut tx_bytes = vec![]; + tx.consensus_serialize(&mut tx_bytes).map_err(Error::NetError)?; + let tx_len = tx_bytes.len() as u64; + + if !self.anchored_done { + // save + match StacksChainState::process_transaction(clarity_tx, tx) { + Ok(_) => {}, + Err(e) => { + warn!("Invalid transaction {} in anchored block, but forcing inclusion (error: {:?})", &tx.txid(), &e); + } + } + + self.txs.push(tx.clone()); + } + else { + match StacksChainState::process_transaction(clarity_tx, tx) { + Ok(_) => {}, + Err(e) => { + warn!("Invalid transaction {} in microblock, but forcing inclusion (error: {:?})", &tx.txid(), &e); + } + } + + self.micro_txs.push(tx.clone()); + } + + self.bytes_so_far += tx_len; + Ok(()) + } + /// Finish building the anchored block. /// TODO: expand to deny mining a block whose anchored static checks fail (and allow the caller /// to disable this, in order to test mining invalid blocks) @@ -629,6 +663,22 @@ pub mod test { } } + pub fn get_last_accepted_anchored_block(&self, miner: &TestMiner) -> Option { + for bc in miner.block_commits.iter().rev() { + if StacksChainState::has_stored_block(&self.chainstate.blocks_db, &self.chainstate.blocks_path, &bc.burn_header_hash, &bc.block_header_hash).unwrap() { + match self.commit_ops.get(&bc.block_header_hash) { + None => { + continue; + } + Some(idx) => { + return Some(self.anchored_blocks[*idx].clone()); + } + } + } + } + return None; + } + pub fn get_microblock_stream(&self, miner: &TestMiner, block_hash: &BlockHeaderHash) -> Option> { match self.commit_ops.get(block_hash) { None => None, @@ -910,9 +960,10 @@ pub mod test { /// Simplest end-to-end test: create 1 fork of N Stacks epochs, mined on 1 burn chain fork, /// all from the same miner. - fn mine_stacks_blocks_1_fork_1_miner_1_burnchain(test_name: &String, rounds: usize, mut block_builder: F) -> TestMinerTrace + fn mine_stacks_blocks_1_fork_1_miner_1_burnchain(test_name: &String, rounds: usize, mut block_builder: F, mut check_oracle: G) -> TestMinerTrace where - F: FnMut(&mut ClarityTx, &mut StacksBlockBuilder, &mut TestMiner, usize, Option<&StacksMicroblockHeader>) -> (StacksBlock, Vec) + F: FnMut(&mut ClarityTx, &mut StacksBlockBuilder, &mut TestMiner, usize, Option<&StacksMicroblockHeader>) -> (StacksBlock, Vec), + G: FnMut(&StacksBlock, &Vec) -> bool { let full_test_name = format!("{}-1_fork_1_miner_1_burnchain", test_name); let mut node = TestStacksNode::new(false, 0x80000000, &full_test_name); @@ -943,7 +994,7 @@ pub mod test { }; let last_key = node.get_last_key(&miner); - let parent_block_opt = node.get_last_anchored_block(&miner); + let parent_block_opt = node.get_last_accepted_anchored_block(&miner); let last_microblock_header = get_last_microblock_header(&node, &miner, parent_block_opt.as_ref()); // next key @@ -975,20 +1026,23 @@ pub mod test { test_debug!("Process Stacks block {} and {} microblocks", &stacks_block.block_hash(), microblocks.len()); let tip_info_list = node.chainstate.process_blocks(1).unwrap(); - // processed _this_ block - assert_eq!(tip_info_list.len(), 1); - let (chain_tip_opt, poison_opt) = tip_info_list[0].clone(); + let expect_success = check_oracle(&stacks_block, µblocks); + if expect_success { + // processed _this_ block + assert_eq!(tip_info_list.len(), 1); + let (chain_tip_opt, poison_opt) = tip_info_list[0].clone(); - assert!(chain_tip_opt.is_some()); - assert!(poison_opt.is_none()); + assert!(chain_tip_opt.is_some()); + assert!(poison_opt.is_none()); - let chain_tip = chain_tip_opt.unwrap(); + let chain_tip = chain_tip_opt.unwrap(); - assert_eq!(chain_tip.anchored_header.block_hash(), stacks_block.block_hash()); - assert_eq!(chain_tip.burn_header_hash, fork_snapshot.burn_header_hash); + assert_eq!(chain_tip.anchored_header.block_hash(), stacks_block.block_hash()); + assert_eq!(chain_tip.burn_header_hash, fork_snapshot.burn_header_hash); - // MARF trie exists for the block header's chain state, so we can make merkle proofs on it - assert!(check_block_state_index_root(&mut node.chainstate, &fork_snapshot.burn_header_hash, &chain_tip.anchored_header)); + // MARF trie exists for the block header's chain state, so we can make merkle proofs on it + assert!(check_block_state_index_root(&mut node.chainstate, &fork_snapshot.burn_header_hash, &chain_tip.anchored_header)); + } let mut next_miner_trace = TestMinerTracePoint::new(); next_miner_trace.add(miner.id, full_test_name.clone(), fork_snapshot, stacks_block, microblocks, block_commit_op); @@ -2466,6 +2520,51 @@ pub mod test { (stacks_block, microblocks) } + /// make a token transfer + pub fn make_token_transfer<'a>(clarity_tx: &mut ClarityTx<'a>, builder: &mut StacksBlockBuilder, miner: &mut TestMiner, burnchain_height: usize, nonce: Option, recipient: &StacksAddress, amount: u64, memo: &TokenTransferMemo) -> StacksTransaction { + let addr = miner.origin_address().unwrap(); + let mut tx_stx_transfer = StacksTransaction::new(TransactionVersion::Testnet, + miner.as_transaction_auth().unwrap(), + TransactionPayload::TokenTransfer((*recipient).clone(), amount, (*memo).clone())); + + tx_stx_transfer.chain_id = 0x80000000; + tx_stx_transfer.auth.set_origin_nonce(nonce.unwrap_or(miner.get_nonce())); + tx_stx_transfer.set_fee_rate(0); + + let mut tx_signer = StacksTransactionSigner::new(&tx_stx_transfer); + miner.sign_as_origin(&mut tx_signer); + let tx_stx_transfer_signed = tx_signer.get_tx().unwrap(); + tx_stx_transfer_signed + } + + /// Mine invalid token transfers + pub fn mine_invalid_token_transfers_block<'a>(clarity_tx: &mut ClarityTx<'a>, builder: &mut StacksBlockBuilder, miner: &mut TestMiner, burnchain_height: usize, parent_microblock_header: Option<&StacksMicroblockHeader>) -> (StacksBlock, Vec) { + let miner_account = StacksChainState::get_account(clarity_tx, &miner.origin_address().unwrap().to_account_principal()); + miner.set_nonce(miner_account.nonce); + + // make a coinbase for this miner + let tx_coinbase_signed = mine_coinbase(clarity_tx, builder, miner, burnchain_height); + builder.try_mine_tx(clarity_tx, &tx_coinbase_signed).unwrap(); + + let recipient = StacksAddress::new(C32_ADDRESS_VERSION_TESTNET_SINGLESIG, Hash160([0xff; 20])); + let tx1 = make_token_transfer(clarity_tx, builder, miner, burnchain_height, Some(1), &recipient, 11111, &TokenTransferMemo([1u8; 34])); + builder.force_mine_tx(clarity_tx, &tx1).unwrap(); + + let tx2 = make_token_transfer(clarity_tx, builder, miner, burnchain_height, Some(2), &recipient, 22222, &TokenTransferMemo([2u8; 34])); + builder.force_mine_tx(clarity_tx, &tx2).unwrap(); + + let tx3 = make_token_transfer(clarity_tx, builder, miner, burnchain_height, Some(1), &recipient, 33333, &TokenTransferMemo([3u8; 34])); + builder.force_mine_tx(clarity_tx, &tx3).unwrap(); + + let tx4 = make_token_transfer(clarity_tx, builder, miner, burnchain_height, Some(2), &recipient, 44444, &TokenTransferMemo([4u8; 34])); + builder.force_mine_tx(clarity_tx, &tx4).unwrap(); + + let stacks_block = builder.mine_anchored_block(clarity_tx); + + test_debug!("Produce anchored stacks block {} with invalid token transfers at burnchain height {} stacks height {}", stacks_block.block_hash(), burnchain_height, stacks_block.header.total_work.work); + (stacks_block, vec![]) + } + /* // TODO: blocked on get-block-info's reliance on get_simmed_block_height @@ -2531,12 +2630,12 @@ pub mod test { #[test] fn mine_anchored_empty_blocks_single() { - mine_stacks_blocks_1_fork_1_miner_1_burnchain(&"empty-anchored-blocks".to_string(), 10, mine_empty_anchored_block); + mine_stacks_blocks_1_fork_1_miner_1_burnchain(&"empty-anchored-blocks".to_string(), 10, mine_empty_anchored_block, |_, _| true); } #[test] fn mine_anchored_empty_blocks_random() { - let mut miner_trace = mine_stacks_blocks_1_fork_1_miner_1_burnchain(&"empty-anchored-blocks-random".to_string(), 10, mine_empty_anchored_block); + let mut miner_trace = mine_stacks_blocks_1_fork_1_miner_1_burnchain(&"empty-anchored-blocks-random".to_string(), 10, mine_empty_anchored_block, |_, _| true); miner_trace_replay_randomized(&mut miner_trace); } @@ -2586,12 +2685,12 @@ pub mod test { #[test] fn mine_anchored_smart_contract_contract_call_blocks_single() { - mine_stacks_blocks_1_fork_1_miner_1_burnchain(&"smart-contract-contract-call-anchored-blocks".to_string(), 10, mine_smart_contract_contract_call_block); + mine_stacks_blocks_1_fork_1_miner_1_burnchain(&"smart-contract-contract-call-anchored-blocks".to_string(), 10, mine_smart_contract_contract_call_block, |_, _| true); } #[test] fn mine_anchored_smart_contract_contract_call_blocks_single_random() { - let mut miner_trace = mine_stacks_blocks_1_fork_1_miner_1_burnchain(&"smart-contract-contract-call-anchored-blocks-random".to_string(), 10, mine_smart_contract_contract_call_block); + let mut miner_trace = mine_stacks_blocks_1_fork_1_miner_1_burnchain(&"smart-contract-contract-call-anchored-blocks-random".to_string(), 10, mine_smart_contract_contract_call_block, |_, _| true); miner_trace_replay_randomized(&mut miner_trace); } @@ -2641,12 +2740,12 @@ pub mod test { #[test] fn mine_anchored_smart_contract_block_contract_call_microblock_single() { - mine_stacks_blocks_1_fork_1_miner_1_burnchain(&"smart-contract-block-contract-call-microblock".to_string(), 10, mine_smart_contract_block_contract_call_microblock); + mine_stacks_blocks_1_fork_1_miner_1_burnchain(&"smart-contract-block-contract-call-microblock".to_string(), 10, mine_smart_contract_block_contract_call_microblock, |_, _| true); } #[test] fn mine_anchored_smart_contract_block_contract_call_microblock_single_random() { - let mut miner_trace = mine_stacks_blocks_1_fork_1_miner_1_burnchain(&"smart-contract-block-contract-call-microblock-random".to_string(), 10, mine_smart_contract_block_contract_call_microblock); + let mut miner_trace = mine_stacks_blocks_1_fork_1_miner_1_burnchain(&"smart-contract-block-contract-call-microblock-random".to_string(), 10, mine_smart_contract_block_contract_call_microblock, |_, _| true); miner_trace_replay_randomized(&mut miner_trace); } @@ -2696,12 +2795,12 @@ pub mod test { #[test] fn mine_anchored_smart_contract_block_contract_call_microblock_exception_single() { - mine_stacks_blocks_1_fork_1_miner_1_burnchain(&"smart-contract-block-contract-call-microblock-exception".to_string(), 10, mine_smart_contract_block_contract_call_microblock_exception); + mine_stacks_blocks_1_fork_1_miner_1_burnchain(&"smart-contract-block-contract-call-microblock-exception".to_string(), 10, mine_smart_contract_block_contract_call_microblock_exception, |_, _| true); } #[test] fn mine_anchored_smart_contract_block_contract_call_microblock_exception_single_random() { - let mut miner_trace = mine_stacks_blocks_1_fork_1_miner_1_burnchain(&"smart-contract-block-contract-call-microblock-exception-random".to_string(), 10, mine_smart_contract_block_contract_call_microblock_exception); + let mut miner_trace = mine_stacks_blocks_1_fork_1_miner_1_burnchain(&"smart-contract-block-contract-call-microblock-exception-random".to_string(), 10, mine_smart_contract_block_contract_call_microblock_exception, |_, _| true); miner_trace_replay_randomized(&mut miner_trace); } @@ -2749,6 +2848,11 @@ pub mod test { miner_trace_replay_randomized(&mut miner_trace); } + #[test] + fn mine_anchored_invalid_token_transfer_blocks_single() { + mine_stacks_blocks_1_fork_1_miner_1_burnchain(&"invalid-token-transfers".to_string(), 10, mine_invalid_token_transfers_block, |_, _| false); + } + // TODO: (BLOCKED) build off of different points in the same microblock stream // TODO; skipped blocks // TODO: missing blocks From f07f028021e369a2420777d1ae70d00b37c7b047 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Wed, 1 Apr 2020 14:33:28 -0400 Subject: [PATCH 06/12] make sure we _commit_ the transaction that orphans blocks; orphan blocks if they fail preprocessing; test that blocks get orphaned --- src/chainstate/stacks/db/blocks.rs | 68 +++++++++++++++++++++++++----- src/chainstate/stacks/miner.rs | 17 ++++++-- 2 files changed, 72 insertions(+), 13 deletions(-) diff --git a/src/chainstate/stacks/db/blocks.rs b/src/chainstate/stacks/db/blocks.rs index 8de23c8dab..a09d99c3b2 100644 --- a/src/chainstate/stacks/db/blocks.rs +++ b/src/chainstate/stacks/db/blocks.rs @@ -1186,6 +1186,22 @@ impl StacksChainState { } }) } + + /// Is a block orphaned? + pub fn is_block_orphaned(blocks_conn: &DBConn, burn_hash: &BurnchainHeaderHash, block_hash: &BlockHeaderHash) -> Result { + StacksChainState::read_i64s(blocks_conn, "SELECT orphaned FROM staging_blocks WHERE anchored_block_hash = ?1 AND burn_header_hash = ?2", &[block_hash, burn_hash]) + .and_then(|orphaned| { + if orphaned.len() == 0 { + Ok(false) + } + else if orphaned.len() == 1 { + Ok(orphaned[0] != 0) + } + else { + Err(Error::DBError(db_error::Overflow)) + } + }) + } /// Do we have a microblock queued up, and if so, is it being processed? /// Return Some(processed) if the microblock is queued up @@ -1204,6 +1220,22 @@ impl StacksChainState { } }) } + + /// Is a microblock orphaned? + pub fn is_microblock_orphaned(blocks_conn: &DBConn, burn_hash: &BurnchainHeaderHash, block_hash: &BlockHeaderHash, microblock_hash: &BlockHeaderHash) -> Result { + StacksChainState::read_i64s(blocks_conn, "SELECT orphaned FROM staging_microblocks WHERE anchored_block_hash = ?1 AND microblock_hash = ?2 AND burn_header_hash = ?3", &[block_hash, microblock_hash, burn_hash]) + .and_then(|orphaned| { + if orphaned.len() == 0 { + Ok(false) + } + else if orphaned.len() == 1 { + Ok(orphaned[0] != 0) + } + else { + Err(Error::DBError(db_error::Overflow)) + } + }) + } /// What's the first microblock hash in a stream? fn get_microblock_stream_head_hash(blocks_conn: &DBConn, burn_hash: &BurnchainHeaderHash, anchored_header_hash: &BlockHeaderHash) -> Result, Error> { @@ -1358,6 +1390,7 @@ impl StacksChainState { else { // Otherwise, all descendents of this processed block are never attacheable. // Mark this block's children as orphans, blow away its data, and blow away its descendent microblocks. + test_debug!("Orphan block {}/{}", burn_hash, anchored_block_hash); StacksChainState::delete_orphaned_epoch_data(tx, burn_hash, anchored_block_hash)?; } @@ -1959,22 +1992,29 @@ impl StacksChainState { test_debug!("Block already stored and/or processed: {}/{}", burn_header_hash, &block.block_hash()); return Ok(false); } - + + // find all user burns that supported this block + let user_burns = BurnDB::get_winning_user_burns_by_block(burn_tx, burn_header_hash) + .map_err(Error::DBError)?; + + let mainnet = self.mainnet; + let chain_id = self.chain_id; + let mut block_tx = self.blocks_tx_begin()?; + // does this block match the burnchain state? skip if not - let (commit_burn, sortition_burn) = match StacksChainState::validate_anchored_block_burnchain(burn_tx, burn_header_hash, block, self.mainnet, self.chain_id)? { + let (commit_burn, sortition_burn) = match StacksChainState::validate_anchored_block_burnchain(burn_tx, burn_header_hash, block, mainnet, chain_id)? { Some((commit_burn, sortition_burn)) => (commit_burn, sortition_burn), None => { let msg = format!("Invalid block {}: does not correspond to burn chain state", block.block_hash()); warn!("{}", &msg); + + // orphan it + StacksChainState::set_block_processed(&mut block_tx, burn_header_hash, &block.block_hash(), false)?; + + block_tx.commit().map_err(Error::DBError)?; return Err(Error::InvalidStacksBlock(msg)); } }; - - // find all user burns that supported this block - let user_burns = BurnDB::get_winning_user_burns_by_block(burn_tx, burn_header_hash) - .map_err(Error::DBError)?; - - let mut block_tx = self.blocks_tx_begin()?; // queue block up for processing StacksChainState::store_staging_block(&mut block_tx, burn_header_hash, burn_header_timestamp, &block, parent_burn_header_hash, commit_burn, sortition_burn)?; @@ -2159,6 +2199,7 @@ impl StacksChainState { let sql = "SELECT * FROM staging_blocks WHERE processed = 0 AND orphaned = 1 ORDER BY RANDOM() LIMIT 1".to_string(); let mut rows = query_rows::(blocks_tx, &sql, NO_PARAMS).map_err(Error::DBError)?; if rows.len() == 0 { + test_debug!("No orphans to remove"); return Ok(false); } @@ -2587,7 +2628,10 @@ impl StacksChainState { debug!("Block already processed: {}/{}", &next_staging_block.burn_header_hash, &next_staging_block.anchored_block_hash); // clear out - StacksChainState::set_block_processed(&mut chainstate_tx.blocks_tx, &next_staging_block.burn_header_hash, &next_staging_block.anchored_block_hash, true)?; + StacksChainState::set_block_processed(&mut chainstate_tx.blocks_tx, &next_staging_block.burn_header_hash, &next_staging_block.anchored_block_hash, true)?; + chainstate_tx.commit() + .map_err(Error::DBError)?; + return Ok((None, None)); } @@ -2660,6 +2704,7 @@ impl StacksChainState { // something's wrong with this epoch -- either a microblock was invalid, or the // anchored block was invalid. Either way, the anchored block will _never be_ // valid, so we can drop it from the chunk store and orphan all of its descendents. + test_debug!("Failed to append {}/{}", &next_staging_block.burn_header_hash, &block.block_hash()); StacksChainState::set_block_processed(&mut chainstate_tx.blocks_tx, &next_staging_block.burn_header_hash, &block.header.block_hash(), false) .expect(&format!("FATAL: failed to clear invalid block {}/{}", next_staging_block.burn_header_hash, &block.header.block_hash())); @@ -2679,6 +2724,10 @@ impl StacksChainState { // leave them in the staging database. } } + + chainstate_tx.commit() + .map_err(Error::DBError)?; + return Err(e); } }; @@ -3977,7 +4026,6 @@ pub mod test { if i + 1 < blocks.len() { // block i+1 should be marked as an orphan, but its data should still be there assert!(StacksChainState::load_staging_block(&chainstate.blocks_db, &chainstate.blocks_path, &burn_headers[i+1], &blocks[i+1].block_hash()).unwrap().is_none()); - // assert!(StacksChainState::load_staging_block_bytes(&chainstate.blocks_db, &burn_headers[i+1], &blocks[i+1].block_hash()).unwrap().unwrap().len() > 0); assert!(StacksChainState::load_block_bytes(&chainstate.blocks_path, &burn_headers[i+1], &blocks[i+1].block_hash()).unwrap().unwrap().len() > 0); for mblock in microblocks[i+1].iter() { diff --git a/src/chainstate/stacks/miner.rs b/src/chainstate/stacks/miner.rs index 6ae15b7f09..93c69ea661 100644 --- a/src/chainstate/stacks/miner.rs +++ b/src/chainstate/stacks/miner.rs @@ -665,7 +665,8 @@ pub mod test { pub fn get_last_accepted_anchored_block(&self, miner: &TestMiner) -> Option { for bc in miner.block_commits.iter().rev() { - if StacksChainState::has_stored_block(&self.chainstate.blocks_db, &self.chainstate.blocks_path, &bc.burn_header_hash, &bc.block_header_hash).unwrap() { + if StacksChainState::has_stored_block(&self.chainstate.blocks_db, &self.chainstate.blocks_path, &bc.burn_header_hash, &bc.block_header_hash).unwrap() && + !StacksChainState::is_block_orphaned(&self.chainstate.blocks_db, &bc.burn_header_hash, &bc.block_header_hash).unwrap() { match self.commit_ops.get(&bc.block_header_hash) { None => { continue; @@ -2850,8 +2851,18 @@ pub mod test { #[test] fn mine_anchored_invalid_token_transfer_blocks_single() { - mine_stacks_blocks_1_fork_1_miner_1_burnchain(&"invalid-token-transfers".to_string(), 10, mine_invalid_token_transfers_block, |_, _| false); - } + let miner_trace = mine_stacks_blocks_1_fork_1_miner_1_burnchain(&"invalid-token-transfers".to_string(), 10, mine_invalid_token_transfers_block, |_, _| false); + + let full_test_name = "invalid-token-transfers-1_fork_1_miner_1_burnchain"; + let chainstate = open_chainstate(false, 0x80000000, full_test_name); + + // each block must be orphaned + for point in miner_trace.points.iter() { + for (height, bc) in point.block_commits.iter() { + assert!(StacksChainState::is_block_orphaned(&chainstate.blocks_db, &bc.burn_header_hash, &bc.block_header_hash).unwrap()); + } + } + } // TODO: (BLOCKED) build off of different points in the same microblock stream // TODO; skipped blocks From 6761854e33830f324d8653b86da03ec7bcee8d0d Mon Sep 17 00:00:00 2001 From: Matthew Little Date: Thu, 2 Apr 2020 12:23:30 +0200 Subject: [PATCH 07/12] feat: include contract ABI JSON in sidecar tx msg --- src/chainstate/stacks/db/transactions.rs | 13 +++++++++---- src/chainstate/stacks/events.rs | 2 ++ src/testnet/helium/event_dispatcher.rs | 9 ++++++++- src/vm/analysis/type_checker/contexts.rs | 2 +- src/vm/analysis/types.rs | 2 +- src/vm/costs/mod.rs | 2 +- 6 files changed, 22 insertions(+), 8 deletions(-) diff --git a/src/chainstate/stacks/db/transactions.rs b/src/chainstate/stacks/db/transactions.rs index d485a7b4da..dcc11c5d9e 100644 --- a/src/chainstate/stacks/db/transactions.rs +++ b/src/chainstate/stacks/db/transactions.rs @@ -389,6 +389,7 @@ impl StacksChainState { events: vec![StacksTransactionEvent::STXEvent(STXEventType::STXTransferEvent(event_data))], result: Value::okay_true(), stx_burned: 0, + contract_analysis: None, }; // no burns @@ -426,7 +427,8 @@ impl StacksChainState { transaction: tx.clone(), events, result, - stx_burned: asset_map.get_stx_burned_total() + stx_burned: asset_map.get_stx_burned_total(), + contract_analysis: None, }; Ok(receipt) @@ -465,7 +467,8 @@ impl StacksChainState { transaction: tx.clone(), events: vec![], result: Value::err_none(), - stx_burned: 0 + stx_burned: 0, + contract_analysis: None, }; // abort now -- no burns @@ -504,7 +507,8 @@ impl StacksChainState { transaction: tx.clone(), events, result: Value::okay_true(), - stx_burned: asset_map.get_stx_burned_total() + stx_burned: asset_map.get_stx_burned_total(), + contract_analysis: Some(contract_analysis), }; Ok(receipt) @@ -529,7 +533,8 @@ impl StacksChainState { transaction: tx.clone(), events: vec![], result: Value::okay_true(), - stx_burned: 0 + stx_burned: 0, + contract_analysis: None, }; Ok(receipt) diff --git a/src/chainstate/stacks/events.rs b/src/chainstate/stacks/events.rs index 577afe06fc..d7c2dd4104 100644 --- a/src/chainstate/stacks/events.rs +++ b/src/chainstate/stacks/events.rs @@ -9,6 +9,7 @@ use vm::types::{ QualifiedContractIdentifier, AssetIdentifier }; +use vm::analysis::ContractAnalysis; #[derive(Debug, Clone, PartialEq)] pub struct StacksTransactionReceipt { @@ -16,6 +17,7 @@ pub struct StacksTransactionReceipt { pub events: Vec, pub result: Value, pub stx_burned: u128, + pub contract_analysis: Option, } #[derive(Debug, Clone, PartialEq)] diff --git a/src/testnet/helium/event_dispatcher.rs b/src/testnet/helium/event_dispatcher.rs index 3478762320..4676b71cd1 100644 --- a/src/testnet/helium/event_dispatcher.rs +++ b/src/testnet/helium/event_dispatcher.rs @@ -7,6 +7,7 @@ use serde_json::json; use serde::Serialize; use vm::types::{Value, QualifiedContractIdentifier, AssetIdentifier}; +use vm::analysis::{contract_interface_builder::build_contract_interface}; use burnchains::Txid; use chainstate::stacks::StacksBlock; use chainstate::stacks::events::{StacksTransactionReceipt, StacksTransactionEvent, STXEventType, FTEventType, NFTEventType}; @@ -60,13 +61,19 @@ impl EventObserver { let formatted_bytes: Vec = bytes.iter().map(|b| format!("{:02x}", b)).collect(); formatted_bytes }; - + let contract_interface_json = { + match &artifact.contract_analysis { + Some(analysis) => json!(build_contract_interface(analysis)), + None => json!(null) + } + }; let val = json!({ "txid": format!("0x{}", tx.txid()), "tx_index": tx_index, "success": success, "raw_result": format!("0x{}", raw_result.join("")), "raw_tx": format!("0x{}", raw_tx.join("")), + "contract_abi": contract_interface_json, }); tx_index += 1; val diff --git a/src/vm/analysis/type_checker/contexts.rs b/src/vm/analysis/type_checker/contexts.rs index 9e4b2fcd74..3d4e9d3afd 100644 --- a/src/vm/analysis/type_checker/contexts.rs +++ b/src/vm/analysis/type_checker/contexts.rs @@ -8,7 +8,7 @@ use vm::contexts::MAX_CONTEXT_DEPTH; use vm::analysis::errors::{CheckResult, CheckError, CheckErrors}; use vm::analysis::types::{ContractAnalysis}; -#[derive(Debug, PartialEq, Eq)] +#[derive(Debug, PartialEq, Eq, Clone)] pub struct TypeMap { map: HashMap } diff --git a/src/vm/analysis/types.rs b/src/vm/analysis/types.rs index f93d12ba8d..c5cfb55a6a 100644 --- a/src/vm/analysis/types.rs +++ b/src/vm/analysis/types.rs @@ -14,7 +14,7 @@ pub trait AnalysisPass { fn run_pass(contract_analysis: &mut ContractAnalysis, analysis_db: &mut AnalysisDatabase) -> CheckResult<()>; } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)] pub struct ContractAnalysis { pub contract_identifier: QualifiedContractIdentifier, pub private_function_types: BTreeMap, diff --git a/src/vm/costs/mod.rs b/src/vm/costs/mod.rs index aca7a4175c..eae02adff1 100644 --- a/src/vm/costs/mod.rs +++ b/src/vm/costs/mod.rs @@ -46,7 +46,7 @@ impl CostTracker for () { } } -#[derive(Debug)] +#[derive(Debug, Clone, PartialEq)] pub struct LimitedCostTracker { total: ExecutionCost, limit: ExecutionCost From d3fbd0cc245f52efdaeecdde414d2859d269a35c Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Fri, 3 Apr 2020 10:52:18 -0400 Subject: [PATCH 08/12] remove unneeded clone(); report only confirmed microblocks in a microblock stream --- src/chainstate/burn/db/burndb.rs | 2 +- src/chainstate/stacks/db/blocks.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/chainstate/burn/db/burndb.rs b/src/chainstate/burn/db/burndb.rs index 03ab21f2d6..348ddcf68c 100644 --- a/src/chainstate/burn/db/burndb.rs +++ b/src/chainstate/burn/db/burndb.rs @@ -1289,7 +1289,7 @@ impl BurnDB { num_headers }; - let tip_block_hash = tip_snapshot.burn_header_hash.clone(); + let tip_block_hash = tip_snapshot.burn_header_hash; if tip_snapshot.sortition { ret.push((tip_block_hash.clone(), Some(tip_snapshot.winning_stacks_block_hash))); } diff --git a/src/chainstate/stacks/db/blocks.rs b/src/chainstate/stacks/db/blocks.rs index 9fe92f37e6..84a73f0b37 100644 --- a/src/chainstate/stacks/db/blocks.rs +++ b/src/chainstate/stacks/db/blocks.rs @@ -1642,7 +1642,7 @@ impl StacksChainState { /// How many microblocks are in a given stream? pub fn get_microblock_stream_length(&self, index_anchor_block_hash: &BlockHeaderHash) -> Result { - let sql = "SELECT COUNT(microblock_hash) FROM staging_microblocks WHERE index_block_hash = ?1".to_string(); + let sql = "SELECT COUNT(microblock_hash) FROM staging_microblocks WHERE index_block_hash = ?1 AND processed = 1".to_string(); let args = [&index_anchor_block_hash as &dyn ToSql]; let cnt = query_count(&self.blocks_db, &sql, &args).map_err(Error::DBError)?; Ok(cnt as u64) From 311ab0c0cae676a96bf2c01e0b66ee99fae1eaef Mon Sep 17 00:00:00 2001 From: Ludo Galabru Date: Fri, 3 Apr 2020 15:05:02 -0400 Subject: [PATCH 09/12] Introspect events in test --- src/testnet/helium/run_loop.rs | 2 +- src/testnet/helium/tests.rs | 30 ++++++++++++++++++++++++++++-- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/src/testnet/helium/run_loop.rs b/src/testnet/helium/run_loop.rs index d052e1542a..b98152f501 100644 --- a/src/testnet/helium/run_loop.rs +++ b/src/testnet/helium/run_loop.rs @@ -237,7 +237,7 @@ impl RunLoop { fn handle_burnchain_state_cb(burn_callback: &Option, round_index: u64, state: &BurnchainState) { - info_yellow!("Burnchain block #{} ({}) was produced with sortition #{}", state.chain_tip.block_height, state.chain_tip.burn_header_hash, state.chain_tip.sortition_hash); + info_blue!("Burnchain block #{} ({}) was produced with sortition #{}", state.chain_tip.block_height, state.chain_tip.burn_header_hash, state.chain_tip.sortition_hash); burn_callback.map(|cb| cb(round_index, state)); } diff --git a/src/testnet/helium/tests.rs b/src/testnet/helium/tests.rs index 5bb50c3bd2..e306e02afc 100644 --- a/src/testnet/helium/tests.rs +++ b/src/testnet/helium/tests.rs @@ -3,7 +3,7 @@ use rand::RngCore; use util::hash::{to_hex, hex_bytes}; use testnet::helium::mem_pool::MemPool; use chainstate::stacks::db::{StacksChainState}; -use chainstate::stacks::events::{StacksTransactionEvent}; +use chainstate::stacks::events::{StacksTransactionEvent, STXEventType}; use super::node::{TESTNET_CHAIN_ID}; use super::config::{InitialBalance}; @@ -177,6 +177,14 @@ fn should_succeed_mining_valid_txs() { // 1 event should have been produced let events: Vec = receipts.iter().flat_map(|a| a.events.clone()).collect(); assert!(events.len() == 1); + assert!(match &events[0] { + StacksTransactionEvent::SmartContractEvent(data) => { + format!("{}", data.key.0) == "STGT7GSMZG7EA0TS6MVSKT5JC1DCDFGZWJJZXN8A.store" && + data.key.1 == "print" && + format!("{}", data.value) == "0x53657474696e67206b657920666f6f" // "Setting key foo" in hexa + }, + _ => false + }); }, 4 => { // Inspecting the chain at round 4. @@ -205,8 +213,16 @@ fn should_succeed_mining_valid_txs() { // 1 event should have been produced let events: Vec = receipts.iter().flat_map(|a| a.events.clone()).collect(); assert!(events.len() == 1); + assert!(match &events[0] { + StacksTransactionEvent::SmartContractEvent(data) => { + format!("{}", data.key.0) == "STGT7GSMZG7EA0TS6MVSKT5JC1DCDFGZWJJZXN8A.store" && + data.key.1 == "print" && + format!("{}", data.value) == "0x47657474696e67206b657920666f6f" // "Getting key foo" in hexa + }, + _ => false + }); }, - 6 => { + 5 => { // Inspecting the chain at round 5. // - Chain length should be 6. assert!(chain_tip_info.block_height == 6); @@ -234,6 +250,16 @@ fn should_succeed_mining_valid_txs() { // 1 event should have been produced let events: Vec = receipts.iter().flat_map(|a| a.events.clone()).collect(); assert!(events.len() == 1); + assert!(match &events[0] { + StacksTransactionEvent::STXEvent(STXEventType::STXTransferEvent(event)) => { + format!("{}", event.recipient) == "ST195Q2HPXY576N4CT2A0R94D7DRYSX54A5X3YZTH" && + format!("{}", event.sender) == "ST2VHM28V9E5QCRD6C73215KAPSBKQGPWTEE5CMQT" && + event.amount == 1000 + }, + _ => false + }); + panic!() + }, _ => {} } From 48546daf71e29a36f04f403b2ffd9548b1921d51 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Fri, 3 Apr 2020 15:26:12 -0400 Subject: [PATCH 10/12] clean imports --- src/net/dns.rs | 38 +------------------------------------- 1 file changed, 1 insertion(+), 37 deletions(-) diff --git a/src/net/dns.rs b/src/net/dns.rs index b78dda8d86..b7fe4c46cb 100644 --- a/src/net/dns.rs +++ b/src/net/dns.rs @@ -17,8 +17,6 @@ along with Blockstack. If not, see . */ -use std::collections::VecDeque; - use std::sync::mpsc::sync_channel; use std::sync::mpsc::SyncSender; use std::sync::mpsc::Receiver; @@ -34,54 +32,20 @@ use net::PeerAddress; use net::Neighbor; use net::NeighborKey; use net::Error as net_error; -use net::db::PeerDB; use net::asn::ASEntry4; use net::*; use net::codec::*; -use net::StacksMessage; -use net::StacksP2P; -use net::GetBlocksInv; -use net::BLOCKS_INV_DATA_MAX_BITLEN; -use net::connection::ConnectionP2P; -use net::connection::ReplyHandleP2P; -use net::connection::ConnectionOptions; - -use net::neighbors::MAX_NEIGHBOR_BLOCK_DELAY; - -use net::db::*; - -use net::p2p::PeerNetwork; - use util::sleep_ms; use util::db::Error as db_error; -use util::db::DBConn; -use util::secp256k1::Secp256k1PublicKey; -use util::secp256k1::Secp256k1PrivateKey; - -use chainstate::burn::BlockHeaderHash; -use chainstate::burn::db::burndb; -use chainstate::burn::db::burndb::BurnDB; -use chainstate::burn::db::burndb::BurnDBTx; -use chainstate::burn::BlockSnapshot; - -use chainstate::stacks::db::StacksChainState; - -use burnchains::Burnchain; -use burnchains::BurnchainView; use std::net::SocketAddr; +use std::collections::VecDeque; use std::collections::HashMap; -use std::collections::BTreeMap; use std::collections::HashSet; -use std::io::Read; -use std::io::Write; - -use std::convert::TryFrom; - use util::log; use util::get_epoch_time_secs; use util::get_epoch_time_ms; From 339845ac8e53487790e26fb06646b6fa3f2a01b1 Mon Sep 17 00:00:00 2001 From: Ludo Galabru Date: Fri, 3 Apr 2020 16:12:56 -0400 Subject: [PATCH 11/12] Update doc --- README.md | 22 +++++++++------------- Stacks.toml | 30 +++++++++++++++++++----------- 2 files changed, 28 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 404db54833..2e99a0f327 100644 --- a/README.md +++ b/README.md @@ -42,9 +42,9 @@ See [SIP 000](https://github.com/blockstack/stacks-blockchain/blob/master/sip/si - [x] **Local Testnet** is a developer local setup, mono-node, assembling SIP 001, SIP 002, SIP 004 and SIP 005. With this version, developers can not only run Stacks 2.0 on their development machines, but also write, execute, and test smart contracts. See the instructions below for more details. -- [ ] **Open Testnet** is the upcoming version of our public testnet, that we're anticipating will ship in Q1 2020. This testnet will ship with SIP 003, and will be an open-membership public network, where participants will be able to validate and participate in mining testnet blocks. +- [ ] **Open Testnet** is the upcoming version of our public testnet, that we're anticipating will ship in Q2 2020. This testnet will ship with SIP 003, and will be an open-membership public network, where participants will be able to validate and participate in mining testnet blocks. -- [ ] **Mainet** is the fully functional version, that we're intending to ship in Q2 2020. +- [ ] **Mainet** is the fully functional version, that we're intending to ship in Q3 2020. ## Getting started @@ -139,16 +139,12 @@ cargo run --bin blockstack-cli publish b8d99fd45da58038d630d9855d3ca2466e8e0f89d You can observe the state machine in action locally by running: ```bash -cargo run --bin blockstack-core testnet +cargo run --bin blockstack-core testnet ./Stacks.toml ``` -In your console, you should observe an output with a similar: +`Stacks.toml` is a configuration file that you can use for setting genesis balances or configuring Event observers. -```bash -*** mempool path: /tmp/stacks-testnet-5fc814cf78dc0636/L1/mempool -``` - -The testnet is watching this directory, decoding and ingesting the transactions materialized as files. This mechanism is a shortcut for simulating a mempool. A RPC server will soon be integrated. +The testnet is watching the directory specified by the key `mempool.path`. The transactions, materialized as files, will be decoded and ingested. This mechanism is a shortcut for simulating a mempool. A RPC server will soon be integrated. ### Publish your contract @@ -157,7 +153,7 @@ Assuming that the testnet is running, we can publish our `kv-store` contract. In another terminal (or file explorer), you can move the `tx1.bin` generated earlier, to the mempool: ```bash -cp ./tx1.bin /tmp/stacks-testnet-5fc814cf78dc0636/L1/mempool +cp ./tx1.bin /tmp/mempool ``` In the terminal window running the testnet, you can observe the state machine's reactions. @@ -185,7 +181,7 @@ Note: the third argument `1` is a nonce, that must be increased monotonically wi We can submit the transaction by moving it to the mempool path: ```bash -cp ./tx2.bin /tmp/stacks-testnet-5fc814cf78dc0636/L1/mempool +cp ./tx2.bin /tmp/mempool ``` Similarly, we can generate a transaction that would be setting the key `foo` to the value `bar`: @@ -197,7 +193,7 @@ cargo run --bin blockstack-cli contract-call b8d99fd45da58038d630d9855d3ca2466e8 And submit it by moving it to the mempool path: ```bash -cp ./tx3.bin /tmp/stacks-testnet-5fc814cf78dc0636/L1/mempool +cp ./tx3.bin /tmp/mempool ``` Finally, we can issue a third transaction, reading the key `foo` again, for ensuring that the previous transaction has successfully updated the state machine: @@ -209,7 +205,7 @@ cargo run --bin blockstack-cli contract-call b8d99fd45da58038d630d9855d3ca2466e8 And submit this last transaction by moving it to the mempool path: ```bash -cp ./tx4.bin /tmp/stacks-testnet-5fc814cf78dc0636/L1/mempool +cp ./tx4.bin /tmp/mempool ``` Congratulations, you can now [write your own smart contracts with Clarity](https://docs.blockstack.org/core/smart/overview.html). diff --git a/Stacks.toml b/Stacks.toml index 639e7082b2..4dd1347263 100644 --- a/Stacks.toml +++ b/Stacks.toml @@ -7,17 +7,7 @@ mode = "regtest" block_time = 5000 [mempool] -path = "/home/ludovic/mempool" - -[[events_observer]] -port = 8080 -address = "127.0.0.1" -events_keys = [ - "STGT7GSMZG7EA0TS6MVSKT5JC1DCDFGZWJJZXN8A.store::print", - "STGT7GSMZG7EA0TS6MVSKT5JC1DCDFGZWJJZXN8A.contract.ft-token", - "STGT7GSMZG7EA0TS6MVSKT5JC1DCDFGZWJJZXN8A.contract.nft-token", - "stx" -] +path = "/tmp/mempool" [[mstx_balance]] address = "STGT7GSMZG7EA0TS6MVSKT5JC1DCDFGZWJJZXN8A" @@ -30,3 +20,21 @@ amount = 100000 [[mstx_balance]] address = "ST2VHM28V9E5QCRD6C73215KAPSBKQGPWTEE5CMQT" amount = 10000 + +# Event dispatcher +# The stacks blockchain can be observed by sidecar processes, notified through TCP socket, of events such as: +# - print +# - stx-transfer / stx-burn +# - ft-mint / ft-transfer +# - nft-mint / nft-transfer +# A demo is available here: https://github.com/blockstack/stacks-blockchain-sidecar +# +# [[events_observer]] +# port = 8080 +# address = "127.0.0.1" +# events_keys = [ +# "STGT7GSMZG7EA0TS6MVSKT5JC1DCDFGZWJJZXN8A.store::print", +# "STGT7GSMZG7EA0TS6MVSKT5JC1DCDFGZWJJZXN8A.contract.ft-token", +# "STGT7GSMZG7EA0TS6MVSKT5JC1DCDFGZWJJZXN8A.contract.nft-token", +# "stx" +# ] From 809a036cfc96d563f0e56e114aae0444c950dfb6 Mon Sep 17 00:00:00 2001 From: Ludo Galabru Date: Fri, 3 Apr 2020 16:19:19 -0400 Subject: [PATCH 12/12] Remove explicit panic --- src/testnet/helium/tests.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/testnet/helium/tests.rs b/src/testnet/helium/tests.rs index e306e02afc..795efe2626 100644 --- a/src/testnet/helium/tests.rs +++ b/src/testnet/helium/tests.rs @@ -258,8 +258,6 @@ fn should_succeed_mining_valid_txs() { }, _ => false }); - panic!() - }, _ => {} }