WIP store utxo set in the database and not in ram 2

data_structures/Cargo.toml

@@ -9,6 +9,7 @@ workspace = ".."
 [dependencies]
 bls-signatures-rs = "0.1.0"
 bech32 = "0.7.2"
+bincode = "1.2.1"
 byteorder = "1.3.4"
 cbor-codec = { git = "https://github.com/witnet/cbor-codec.git", branch = "feat/ldexpf-shim" }
 chrono = "0.4.10"
@@ -32,6 +33,7 @@ vrf = "0.2.3"
 witnet_crypto = { path = "../crypto" }
 witnet_reputation = { path = "../reputation", features = ["serde"] }
 witnet_protected = { path = "../protected", features = ["serde"] }
+witnet_storage = { path = "../storage", features = ["rocksdb-backend"] }
 witnet_util = { path = "../util" }
 
 [build-dependencies]

data_structures/src/chain.rs

@@ -45,7 +45,7 @@ use crate::{
     transaction::{
         MemoHash, MemoizedHashable, BETA, COMMIT_WEIGHT, OUTPUT_SIZE, REVEAL_WEIGHT, TALLY_WEIGHT,
     },
-    utxo_pool::{OwnUnspentOutputsPool, UnspentOutputsPool},
+    utxo_pool::{OldUnspentOutputsPool, OwnUnspentOutputsPool, UnspentOutputsPool},
     vrf::{BlockEligibilityClaim, DataRequestEligibilityClaim},
 };
 
@@ -3213,7 +3213,7 @@ pub struct ChainState {
     /// Blockchain information data structure
     pub chain_info: Option<ChainInfo>,
     /// Unspent Outputs Pool
-    pub unspent_outputs_pool: UnspentOutputsPool,
+    pub unspent_outputs_pool_old_migration_db: OldUnspentOutputsPool,
     /// Collection of state structures for active data requests
     pub data_request_pool: DataRequestPool,
     /// List of consolidated blocks by epoch
@@ -3231,6 +3231,9 @@ pub struct ChainState {
     pub superblock_state: SuperBlockState,
     /// TAPI Engine
     pub tapi_engine: TapiEngine,
+    /// Unspent Outputs Pool
+    #[serde(skip)]
+    pub unspent_outputs_pool: UnspentOutputsPool,
 }
 
 impl ChainState {
@@ -3850,86 +3853,6 @@ pub fn penalize_factor(
     }
 }
 
-fn update_utxo_inputs(utxo: &mut UnspentOutputsPool, inputs: &[Input]) {
-    for input in inputs {
-        // Obtain the OutputPointer of each input and remove it from the utxo_set
-        let output_pointer = input.output_pointer();
-
-        // This does not check for missing inputs
-        utxo.remove(output_pointer);
-    }
-}
-
-fn update_utxo_outputs(
-    utxo: &mut UnspentOutputsPool,
-    outputs: &[ValueTransferOutput],
-    txn_hash: Hash,
-    block_number: u32,
-) {
-    for (index, output) in outputs.iter().enumerate() {
-        // Add the new outputs to the utxo_set
-        let output_pointer = OutputPointer {
-            transaction_id: txn_hash,
-            output_index: u32::try_from(index).unwrap(),
-        };
-
-        utxo.insert(output_pointer, output.clone(), block_number);
-    }
-}
-
-/// Method to update the unspent outputs pool
-pub fn generate_unspent_outputs_pool(
-    unspent_outputs_pool: &UnspentOutputsPool,
-    transactions: &[Transaction],
-    block_number: u32,
-) -> UnspentOutputsPool {
-    // Create a copy of the state "unspent_outputs_pool"
-    let mut unspent_outputs = unspent_outputs_pool.clone();
-
-    for transaction in transactions {
-        let txn_hash = transaction.hash();
-        match transaction {
-            Transaction::ValueTransfer(vt_transaction) => {
-                update_utxo_inputs(&mut unspent_outputs, &vt_transaction.body.inputs);
-                update_utxo_outputs(
-                    &mut unspent_outputs,
-                    &vt_transaction.body.outputs,
-                    txn_hash,
-                    block_number,
-                );
-            }
-            Transaction::DataRequest(dr_transaction) => {
-                update_utxo_inputs(&mut unspent_outputs, &dr_transaction.body.inputs);
-                update_utxo_outputs(
-                    &mut unspent_outputs,
-                    &dr_transaction.body.outputs,
-                    txn_hash,
-                    block_number,
-                );
-            }
-            Transaction::Tally(tally_transaction) => {
-                update_utxo_outputs(
-                    &mut unspent_outputs,
-                    &tally_transaction.outputs,
-                    txn_hash,
-                    block_number,
-                );
-            }
-            Transaction::Mint(mint_transaction) => {
-                update_utxo_outputs(
-                    &mut unspent_outputs,
-                    &mint_transaction.outputs,
-                    txn_hash,
-                    block_number,
-                );
-            }
-            _ => {}
-        }
-    }
-
-    unspent_outputs
-}
-
 /// Constants used to convert between epoch and timestamp
 #[derive(Copy, Clone, Debug)]
 pub struct EpochConstants {
@@ -5748,140 +5671,6 @@ mod tests {
         assert_eq!(rep_engine.threshold_factor(9), u32::max_value());
     }
 
-    #[test]
-    fn utxo_set_coin_age() {
-        let mut p = UnspentOutputsPool::default();
-        let v = ValueTransferOutput::default;
-
-        let k0: OutputPointer =
-            "0222222222222222222222222222222222222222222222222222222222222222:0"
-                .parse()
-                .unwrap();
-        p.insert(k0.clone(), v(), 0);
-        assert_eq!(p.included_in_block_number(&k0), Some(0));
-
-        let k1: OutputPointer =
-            "1222222222222222222222222222222222222222222222222222222222222222:0"
-                .parse()
-                .unwrap();
-        p.insert(k1.clone(), v(), 1);
-        assert_eq!(p.included_in_block_number(&k1), Some(1));
-
-        // k2 points to the same transaction as k1, so they must have the same coin age
-        let k2: OutputPointer =
-            "1222222222222222222222222222222222222222222222222222222222222222:1"
-                .parse()
-                .unwrap();
-        p.insert(k2.clone(), v(), 1);
-        assert_eq!(p.included_in_block_number(&k2), Some(1));
-
-        // Removing k2 should not affect k1
-        p.remove(&k2);
-        assert_eq!(p.included_in_block_number(&k2), None);
-        assert_eq!(p.included_in_block_number(&k1), Some(1));
-        assert_eq!(p.included_in_block_number(&k0), Some(0));
-
-        p.remove(&k1);
-        assert_eq!(p.included_in_block_number(&k2), None);
-        assert_eq!(p.included_in_block_number(&k1), None);
-        assert_eq!(p.included_in_block_number(&k0), Some(0));
-
-        p.remove(&k0);
-        assert_eq!(p.included_in_block_number(&k0), None);
-
-        assert_eq!(p, UnspentOutputsPool::default());
-    }
-
-    #[test]
-    fn utxo_set_insert_twice() {
-        // Inserting the same input twice into the UTXO set overwrites the transaction
-        let mut p = UnspentOutputsPool::default();
-        let v = ValueTransferOutput::default;
-
-        let k0: OutputPointer =
-            "0222222222222222222222222222222222222222222222222222222222222222:0"
-                .parse()
-                .unwrap();
-        p.insert(k0.clone(), v(), 0);
-        p.insert(k0.clone(), v(), 0);
-        assert_eq!(p.included_in_block_number(&k0), Some(0));
-        // Removing once is enough
-        p.remove(&k0);
-        assert_eq!(p.included_in_block_number(&k0), None);
-    }
-
-    #[test]
-    fn utxo_set_insert_same_transaction_different_epoch() {
-        // Inserting the same transaction twice with different indexes means a different UTXO
-        // so, each UTXO keeps their own block number
-        let mut p = UnspentOutputsPool::default();
-        let v = ValueTransferOutput::default;
-
-        let k0: OutputPointer =
-            "0222222222222222222222222222222222222222222222222222222222222222:0"
-                .parse()
-                .unwrap();
-        p.insert(k0.clone(), v(), 0);
-        assert_eq!(p.included_in_block_number(&k0), Some(0));
-        let k1: OutputPointer =
-            "0222222222222222222222222222222222222222222222222222222222222222:1"
-                .parse()
-                .unwrap();
-
-        p.insert(k1.clone(), v(), 1);
-        assert_eq!(p.included_in_block_number(&k1), Some(1));
-    }
-
-    #[test]
-    fn test_sort_own_utxos() {
-        let vto1 = ValueTransferOutput {
-            value: 100,
-            ..ValueTransferOutput::default()
-        };
-        let vto2 = ValueTransferOutput {
-            value: 500,
-            ..ValueTransferOutput::default()
-        };
-        let vto3 = ValueTransferOutput {
-            value: 200,
-            ..ValueTransferOutput::default()
-        };
-        let vto4 = ValueTransferOutput {
-            value: 300,
-            ..ValueTransferOutput::default()
-        };
-
-        let vt = Transaction::ValueTransfer(VTTransaction::new(
-            VTTransactionBody::new(vec![], vec![vto1, vto2, vto3, vto4]),
-            vec![],
-        ));
-
-        let utxo_pool = generate_unspent_outputs_pool(&UnspentOutputsPool::default(), &[vt], 0);
-        assert_eq!(utxo_pool.iter().len(), 4);
-
-        let mut own_utxos = OwnUnspentOutputsPool::default();
-        for (o, _) in utxo_pool.iter() {
-            own_utxos.insert(o.clone(), 0);
-        }
-        assert_eq!(own_utxos.len(), 4);
-
-        let sorted_bigger = own_utxos.sort(&utxo_pool, true);
-        let mut aux = 1000;
-        for o in sorted_bigger.iter() {
-            let value = utxo_pool.get(o).unwrap().value;
-            assert!(value < aux);
-            aux = value;
-        }
-
-        let sorted_lower = own_utxos.sort(&utxo_pool, false);
-        let mut aux = 0;
-        for o in sorted_lower.iter() {
-            let value = utxo_pool.get(o).unwrap().value;
-            assert!(value > aux);
-            aux = value;
-        }
-    }
-
     #[test]
     fn test_calculate_backup_witnesses() {
         assert_eq!(calculate_backup_witnesses(10, 1), 5);