Feat/stackerdb miners contract

.github/workflows/bitcoin-tests.yml

@@ -73,6 +73,7 @@ jobs:
           - tests::nakamoto_integrations::simple_neon_integration
           - tests::nakamoto_integrations::mine_multiple_per_tenure_integration
           - tests::nakamoto_integrations::block_proposal_api_endpoint
+          - tests::nakamoto_integrations::miner_writes_proposed_block_to_stackerdb
     steps:
       ## Setup test environment
       - name: Setup Test Environment

libstackerdb/src/libstackerdb.rs

@@ -33,8 +33,8 @@ use stacks_common::types::PrivateKey;
 use stacks_common::util::hash::{hex_bytes, to_hex, Hash160, Sha512Trunc256Sum};
 use stacks_common::util::secp256k1::MessageSignature;
 
-/// maximum chunk size (1 MB)
-pub const STACKERDB_MAX_CHUNK_SIZE: u32 = 1024 * 1024;
+/// maximum chunk size (16 MB; same as MAX_PAYLOAD_SIZE)
+pub const STACKERDB_MAX_CHUNK_SIZE: u32 = 16 * 1024 * 1024;
 
 #[cfg(test)]
 mod tests;

stackslib/Cargo.toml

@@ -54,6 +54,7 @@ clarity = { path = "../clarity" }
 stacks-common = { path = "../stacks-common" }
 pox-locking = { path = "../pox-locking" }
 libstackerdb = { path = "../libstackerdb" }
+libsigner = { path = "../libsigner" }
 siphasher = "0.3.7"
 wsts = {workspace = true}
 rand_core = {workspace = true}

stackslib/src/chainstate/nakamoto/coordinator/tests.rs

@@ -35,6 +35,7 @@ use crate::chainstate::nakamoto::tests::node::TestSigners;
 use crate::chainstate::nakamoto::{NakamotoBlock, NakamotoChainState};
 use crate::chainstate::stacks::address::PoxAddress;
 use crate::chainstate::stacks::boot::test::{make_pox_4_aggregate_key, make_pox_4_lockup};
+use crate::chainstate::stacks::boot::MINERS_NAME;
 use crate::chainstate::stacks::db::{MinerPaymentTxFees, StacksAccount, StacksChainState};
 use crate::chainstate::stacks::{
     CoinbasePayload, StacksTransaction, StacksTransactionSigner, TenureChangeCause,
@@ -44,7 +45,9 @@ use crate::chainstate::stacks::{
 use crate::clarity::vm::types::StacksAddressExtensions;
 use crate::core::StacksEpochExtension;
 use crate::net::relay::Relayer;
+use crate::net::stackerdb::StackerDBConfig;
 use crate::net::test::{TestPeer, TestPeerConfig};
+use crate::util_lib::boot::boot_code_id;
 
 /// Bring a TestPeer into the Nakamoto Epoch
 fn advance_to_nakamoto(peer: &mut TestPeer) {
@@ -84,7 +87,6 @@ fn advance_to_nakamoto(peer: &mut TestPeer) {
 
         peer.tenure_with_txs(&txs, &mut peer_nonce);
     }
-
     // peer is at the start of cycle 8
 }
 
@@ -96,7 +98,6 @@ pub fn boot_nakamoto(
     aggregate_public_key: Point,
 ) -> TestPeer {
     let mut peer_config = TestPeerConfig::new(test_name, 0, 0);
-    peer_config.aggregate_public_key = Some(aggregate_public_key.clone());
     let private_key = peer_config.private_key.clone();
     let addr = StacksAddress::from_public_keys(
         C32_ADDRESS_VERSION_TESTNET_SINGLESIG,
@@ -110,14 +111,17 @@ pub fn boot_nakamoto(
     // first 25 blocks are boot-up
     // reward cycle 6 instantiates pox-3
     // we stack in reward cycle 7 so pox-3 is evaluated to find reward set participation
+    peer_config.aggregate_public_key = Some(aggregate_public_key.clone());
+    peer_config
+        .stacker_dbs
+        .push(boot_code_id(MINERS_NAME, false));
     peer_config.epochs = Some(StacksEpoch::unit_test_3_0_only(37));
     peer_config.initial_balances = vec![(addr.to_account_principal(), 1_000_000_000_000_000_000)];
     peer_config.initial_balances.append(&mut initial_balances);
     peer_config.burnchain.pox_constants.v2_unlock_height = 21;
     peer_config.burnchain.pox_constants.pox_3_activation_height = 26;
     peer_config.burnchain.pox_constants.v3_unlock_height = 27;
     peer_config.burnchain.pox_constants.pox_4_activation_height = 31;
-
     let mut peer = TestPeer::new(peer_config);
     advance_to_nakamoto(&mut peer);
     peer

stackslib/src/chainstate/nakamoto/miner.rs

@@ -28,8 +28,10 @@ use clarity::vm::clarity::TransactionConnection;
 use clarity::vm::costs::ExecutionCost;
 use clarity::vm::database::BurnStateDB;
 use clarity::vm::errors::Error as InterpreterError;
-use clarity::vm::types::TypeSignature;
+use clarity::vm::types::{QualifiedContractIdentifier, TypeSignature};
+use libstackerdb::StackerDBChunkData;
 use serde::Deserialize;
+use stacks_common::codec::{read_next, write_next, Error as CodecError, StacksMessageCodec};
 use stacks_common::types::chainstate::{
     BlockHeaderHash, BurnchainHeaderHash, ConsensusHash, StacksAddress, StacksBlockId, TrieHash,
 };
@@ -46,6 +48,7 @@ use crate::chainstate::nakamoto::{
     MaturedMinerRewards, NakamotoBlock, NakamotoBlockHeader, NakamotoChainState, SetupBlockResult,
 };
 use crate::chainstate::stacks::address::StacksAddressExtensions;
+use crate::chainstate::stacks::boot::MINERS_NAME;
 use crate::chainstate::stacks::db::accounts::MinerReward;
 use crate::chainstate::stacks::db::blocks::MemPoolRejection;
 use crate::chainstate::stacks::db::transactions::{
@@ -62,7 +65,6 @@ use crate::chainstate::stacks::miner::{
 };
 use crate::chainstate::stacks::{Error, StacksBlockHeader, *};
 use crate::clarity_vm::clarity::{ClarityConnection, ClarityInstance};
-use crate::codec::Error as CodecError;
 use crate::core::mempool::*;
 use crate::core::*;
 use crate::cost_estimates::metrics::CostMetric;
@@ -71,6 +73,9 @@ use crate::monitoring::{
     set_last_mined_block_transaction_count, set_last_mined_execution_cost_observed,
 };
 use crate::net::relay::Relayer;
+use crate::net::stackerdb::StackerDBs;
+use crate::net::Error as net_error;
+use crate::util_lib::boot::boot_code_id;
 use crate::util_lib::db::Error as DBError;
 
 /// Nakamaoto tenure information
@@ -359,7 +364,7 @@ impl NakamotoBlockBuilder {
         );
 
         debug!(
-            "Miner: mined Nakamoto block";
+            "Miner: mined Nakamoto block (miner hashes include zeroed signatures)";
             "consensus_hash" => %block.header.consensus_hash,
             "block_hash" => %block.header.block_hash(),
             "block_height" => block.header.chain_length,
@@ -502,6 +507,40 @@ impl NakamotoBlockBuilder {
     pub fn get_bytes_so_far(&self) -> u64 {
         self.bytes_so_far
     }
+
+    /// Make a StackerDB chunk message containing a proposed block.
+    /// Sign it with the miner's private key.
+    /// Automatically determine which StackerDB slot and version number to use.
+    /// Returns Some(chunk) if the given key corresponds to one of the expected miner slots
+    /// Returns None if not
+    /// Returns an error on signing or DB error
+    pub fn make_stackerdb_block_proposal(
+        sortdb: &SortitionDB,
+        tip: &BlockSnapshot,
+        stackerdbs: &StackerDBs,
+        block: &NakamotoBlock,
+        miner_privkey: &StacksPrivateKey,
+        miners_contract_id: &QualifiedContractIdentifier,
+    ) -> Result<Option<StackerDBChunkData>, Error> {
+        let miner_pubkey = StacksPublicKey::from_private(&miner_privkey);
+        let Some(slot_id) = NakamotoChainState::get_miner_slot(sortdb, tip, &miner_pubkey)? else {
+            // No slot exists for this miner
+            return Ok(None);
+        };
+        // Get the LAST slot version number written to the DB. If not found, use 0.
+        // Add 1 to get the NEXT version number
+        // Note: we already check above for the slot's existence
+        let slot_version = stackerdbs
+            .get_slot_version(&miners_contract_id, slot_id)?
+            .unwrap_or(0)
+            .saturating_add(1);
+        let block_bytes = block.serialize_to_vec();
+        let mut chunk = StackerDBChunkData::new(slot_id, slot_version, block_bytes);
+        chunk
+            .sign(miner_privkey)
+            .map_err(|_| net_error::SigningError("Failed to sign StackerDB chunk".into()))?;
+        Ok(Some(chunk))
+    }
 }
 
 impl BlockBuilder for NakamotoBlockBuilder {

stackslib/src/chainstate/nakamoto/mod.rs

@@ -29,6 +29,7 @@ use rusqlite::{params, Connection, OptionalExtension, ToSql, NO_PARAMS};
 use sha2::{Digest as Sha2Digest, Sha512_256};
 use stacks_common::codec::{
     read_next, write_next, Error as CodecError, StacksMessageCodec, MAX_MESSAGE_LEN,
+    MAX_PAYLOAD_LEN,
 };
 use stacks_common::consts::{
     FIRST_BURNCHAIN_CONSENSUS_HASH, FIRST_STACKS_BLOCK_HASH, MINER_REWARD_MATURITY,
@@ -78,6 +79,7 @@ use crate::clarity_vm::clarity::{ClarityInstance, PreCommitClarityBlock};
 use crate::clarity_vm::database::SortitionDBRef;
 use crate::core::BOOT_BLOCK_HASH;
 use crate::monitoring;
+use crate::net::stackerdb::StackerDBConfig;
 use crate::net::Error as net_error;
 use crate::util_lib::boot::boot_code_id;
 use crate::util_lib::db::{
@@ -461,6 +463,7 @@ impl NakamotoBlockHeader {
             tx_merkle_root: Sha512Trunc256Sum([0u8; 32]),
             state_index_root: TrieHash([0u8; 32]),
             miner_signature: MessageSignature::empty(),
+            // TODO: `mock()` should be updated to `empty()` and rustdocs updated
             signer_signature: ThresholdSignature::mock(),
         }
     }
@@ -3047,6 +3050,123 @@ impl NakamotoChainState {
         Ok((epoch_receipt, clarity_commit))
     }
 
+    /// Create a StackerDB config for the .miners contract.
+    /// It has two slots -- one for the past two sortition winners.
+    pub fn make_miners_stackerdb_config(
+        sortdb: &SortitionDB,
+        tip: &BlockSnapshot,
+    ) -> Result<StackerDBConfig, ChainstateError> {
+        let ih = sortdb.index_handle(&tip.sortition_id);
+        let last_winner_snapshot = ih.get_last_snapshot_with_sortition(tip.block_height)?;
+        let parent_winner_snapshot = ih.get_last_snapshot_with_sortition(
+            last_winner_snapshot.block_height.saturating_sub(1),
+        )?;
+
+        let mut miner_key_hash160s = vec![];
+
+        // go get their corresponding leader keys, but preserve the miner's relative position in
+        // the stackerdb signer list -- if a miner was in slot 0, then it should stay in slot 0
+        // after a sortition (and vice versa for 1)
+        let sns = if last_winner_snapshot.num_sortitions % 2 == 0 {
+            [last_winner_snapshot, parent_winner_snapshot]
+        } else {
+            [parent_winner_snapshot, last_winner_snapshot]
+        };
+
+        for sn in sns {
+            // find the commit
+            let Some(block_commit) =
+                ih.get_block_commit_by_txid(&sn.sortition_id, &sn.winning_block_txid)?
+            else {
+                warn!(
+                    "No block commit for {} in sortition for {}",
+                    &sn.winning_block_txid, &sn.consensus_hash
+                );
+                return Err(ChainstateError::InvalidStacksBlock(
+                    "No block-commit in sortition for block's consensus hash".into(),
+                ));
+            };
+
+            // key register of the winning miner
+            let leader_key = ih
+                .get_leader_key_at(
+                    u64::from(block_commit.key_block_ptr),
+                    u32::from(block_commit.key_vtxindex),
+                )?
+                .expect("FATAL: have block commit but no leader key");
+
+            // the leader key should always be valid (i.e. the unwrap_or() should be unreachable),
+            // but be defensive and just use the "null" address
+            miner_key_hash160s.push(
+                leader_key
+                    .interpret_nakamoto_signing_key()
+                    .unwrap_or(Hash160([0x00; 20])),
+            );
+        }
+
+        let signers = miner_key_hash160s
+            .into_iter()
+            .map(|hash160|
+                // each miner gets one slot
+                (
+                    StacksAddress {
+                        version: 1, // NOTE: the version is ignored in stackerdb; we only care about the hashbytes
+                        bytes: hash160
+                    },
+                    1
+                ))
+            .collect();
+
+        Ok(StackerDBConfig {
+            chunk_size: MAX_PAYLOAD_LEN.into(),
+            signers,
+            write_freq: 5,
+            max_writes: u32::MAX,  // no limit on number of writes
+            max_neighbors: 200, // TODO: const -- just has to be equal to or greater than the number of signers
+            hint_replicas: vec![], // TODO: is there a way to get the IP addresses of stackers' preferred nodes?
+        })
+    }
+
+    /// Get the slot number for the given miner's public key.
+    /// Returns Some(u32) if the miner is in the StackerDB config.
+    /// Returns None if the miner is not in the StackerDB config.
+    /// Returns an error if the miner is in the StackerDB config but the slot number is invalid.
+    pub fn get_miner_slot(
+        sortdb: &SortitionDB,
+        tip: &BlockSnapshot,
+        miner_pubkey: &StacksPublicKey,
+    ) -> Result<Option<u32>, ChainstateError> {
+        let miner_hash160 = Hash160::from_node_public_key(&miner_pubkey);
+        let stackerdb_config = Self::make_miners_stackerdb_config(sortdb, &tip)?;
+
+        // find out which slot we're in
+        let Some(slot_id_res) =
+            stackerdb_config
+                .signers
+                .iter()
+                .enumerate()
+                .find_map(|(i, (addr, _))| {
+                    if addr.bytes == miner_hash160 {
+                        Some(u32::try_from(i).map_err(|_| {
+                            CodecError::OverflowError(
+                                "stackerdb config slot ID cannot fit into u32".into(),
+                            )
+                        }))
+                    } else {
+                        None
+                    }
+                })
+        else {
+            // miner key does not match any slot
+            warn!("Miner is not in the miners StackerDB config";
+                  "miner" => %miner_hash160,
+                  "stackerdb_slots" => format!("{:?}", &stackerdb_config.signers));
+
+            return Ok(None);
+        };
+        Ok(Some(slot_id_res?))
+    }
+
     /// Boot code instantiation for the aggregate public key.
     /// TODO: This should be removed once it's possible for stackers to vote on the aggregate
     /// public key