Merge pull request #2887 from blockstack/fix/2885

Fix/2885

src/burnchains/bitcoin/spv.rs

@@ -42,8 +42,8 @@ use rusqlite::Transaction;
 use rusqlite::{Connection, OpenFlags, NO_PARAMS};
 
 use util::db::{
-    query_row, query_rows, tx_begin_immediate, tx_busy_handler, u64_to_sql, DBConn, DBTx,
-    Error as db_error, FromColumn, FromRow,
+    query_row, query_rows, sqlite_open, tx_begin_immediate, tx_busy_handler, u64_to_sql, DBConn,
+    DBTx, Error as db_error, FromColumn, FromRow,
 };
 use util::get_epoch_time_secs;
 use util::hash::{hex_bytes, to_hex};
@@ -212,11 +212,8 @@ impl SpvClient {
             }
         };
 
-        let mut conn =
-            Connection::open_with_flags(headers_path, open_flags).map_err(db_error::SqliteError)?;
-
-        conn.busy_handler(Some(tx_busy_handler))
-            .map_err(db_error::SqliteError)?;
+        let mut conn = sqlite_open(headers_path, open_flags, false)
+            .map_err(|e| btc_error::DBError(db_error::SqliteError(e)))?;
 
         if create_flag {
             SpvClient::db_instantiate(&mut conn)?;

src/burnchains/db.rs

@@ -27,8 +27,8 @@ use burnchains::{Burnchain, BurnchainBlock, BurnchainBlockHeader, Error as Burnc
 use chainstate::burn::operations::BlockstackOperationType;
 use chainstate::stacks::index::MarfTrieId;
 use util::db::{
-    query_row, query_rows, sql_pragma, tx_begin_immediate, tx_busy_handler, u64_to_sql,
-    Error as DBError, FromColumn, FromRow,
+    query_row, query_rows, sql_pragma, sqlite_open, tx_begin_immediate, tx_busy_handler,
+    u64_to_sql, Error as DBError, FromColumn, FromRow,
 };
 
 use crate::types::chainstate::BurnchainHeaderHash;
@@ -209,16 +209,11 @@ impl BurnchainDB {
             }
         };
 
-        let conn = Connection::open_with_flags(path, open_flags)
-            .expect(&format!("FAILED to open: {}", path));
-
-        conn.busy_handler(Some(tx_busy_handler))?;
-
+        let conn = sqlite_open(path, open_flags, true)?;
         let mut db = BurnchainDB { conn };
 
         if create_flag {
             let db_tx = db.tx_begin()?;
-            sql_pragma(&db_tx.sql_tx, "PRAGMA journal_mode = WAL;")?;
             db_tx.sql_tx.execute_batch(BURNCHAIN_DB_INITIAL_SCHEMA)?;
 
             db_tx.sql_tx.execute(
@@ -247,9 +242,7 @@ impl BurnchainDB {
         } else {
             OpenFlags::SQLITE_OPEN_READ_ONLY
         };
-        let conn = Connection::open_with_flags(path, open_flags)?;
-        conn.busy_handler(Some(tx_busy_handler))?;
-
+        let conn = sqlite_open(path, open_flags, true)?;
         Ok(BurnchainDB { conn })
     }
 

src/chainstate/burn/db/sortdb.rs

@@ -1980,7 +1980,9 @@ impl SortitionDB {
 
     fn open_index(index_path: &str) -> Result<MARF<SortitionId>, db_error> {
         test_debug!("Open index at {}", index_path);
-        MARF::from_path(index_path).map_err(|_e| db_error::Corruption)
+        let marf = MARF::from_path(index_path).map_err(|_e| db_error::Corruption)?;
+        sql_pragma(marf.sqlite_conn(), "foreign_keys", &true)?;
+        Ok(marf)
     }
 
     /// Open the database on disk.  It must already exist and be instantiated.
@@ -2004,6 +2006,7 @@ impl SortitionDB {
             first_block_height: first_snapshot.block_height,
             first_burn_header_hash: first_snapshot.burn_header_hash.clone(),
         };
+
         Ok(db)
     }
 
@@ -2101,7 +2104,8 @@ impl SortitionDB {
     ) -> Result<(), db_error> {
         debug!("Instantiate SortDB");
 
-        sql_pragma(self.conn(), "PRAGMA journal_mode = WAL;")?;
+        sql_pragma(self.conn(), "journal_mode", &"WAL")?;
+        sql_pragma(self.conn(), "foreign_keys", &true)?;
 
         let mut db_tx = SortitionHandleTx::begin(self, &SortitionId::sentinel())?;
 

src/chainstate/stacks/index/storage.rs

@@ -45,6 +45,7 @@ use chainstate::stacks::index::node::{
 };
 use chainstate::stacks::index::Error;
 use chainstate::stacks::index::{trie_sql, BlockMap, MarfTrieId};
+use util::db::sqlite_open;
 use util::db::tx_begin_immediate;
 use util::db::tx_busy_handler;
 use util::db::Error as db_error;
@@ -795,9 +796,7 @@ impl<T: MarfTrieId> TrieFileStorage<T> {
             }
         };
 
-        let mut db = Connection::open_with_flags(db_path, open_flags)?;
-        db.busy_handler(Some(tx_busy_handler))?;
-
+        let mut db = sqlite_open(db_path, open_flags, false)?;
         let db_path = db_path.to_string();
 
         if create_flag {
@@ -866,8 +865,7 @@ impl<T: MarfTrieId> TrieFileStorage<T> {
     }
 
     pub fn reopen_readonly(&self) -> Result<TrieFileStorage<T>, Error> {
-        let db = Connection::open_with_flags(&self.db_path, OpenFlags::SQLITE_OPEN_READ_ONLY)?;
-        db.busy_handler(Some(tx_busy_handler))?;
+        let db = sqlite_open(&self.db_path, OpenFlags::SQLITE_OPEN_READ_ONLY, false)?;
 
         trace!("Make read-only view of TrieFileStorage: {}", &self.db_path);
 
@@ -911,8 +909,7 @@ impl<'a, T: MarfTrieId> TrieStorageTransaction<'a, T> {
     /// reopen this transaction as a read-only marf.
     ///  _does not_ preserve the cur_block/open tip
     pub fn reopen_readonly(&self) -> Result<TrieFileStorage<T>, Error> {
-        let db = Connection::open_with_flags(&self.db_path, OpenFlags::SQLITE_OPEN_READ_ONLY)?;
-        db.busy_handler(Some(tx_busy_handler))?;
+        let db = sqlite_open(&self.db_path, OpenFlags::SQLITE_OPEN_READ_ONLY, false)?;
 
         trace!(
             "Make read-only view of TrieStorageTransaction: {}",
@@ -1286,9 +1283,7 @@ impl<'a, T: MarfTrieId> TrieStorageConnection<'a, T> {
     /// Recover from partially-written state -- i.e. blow it away.
     /// Doesn't get called automatically.
     pub fn recover(db_path: &String) -> Result<(), Error> {
-        let conn = Connection::open(db_path)?;
-        conn.busy_handler(Some(tx_busy_handler))?;
-
+        let conn = sqlite_open(db_path, OpenFlags::SQLITE_OPEN_READ_WRITE, false)?;
         trie_sql::clear_lock_data(&conn)
     }
 

src/chainstate/stacks/index/trie_sql.rs

@@ -83,8 +83,6 @@ CREATE TABLE IF NOT EXISTS block_extension_locks (block_hash TEXT PRIMARY KEY);
 ";
 
 pub fn create_tables_if_needed(conn: &mut Connection) -> Result<(), Error> {
-    sql_pragma(conn, "PRAGMA journal_mode = WAL;")?;
-
     let tx = tx_begin_immediate(conn)?;
 
     tx.execute_batch(SQL_MARF_DATA_TABLE)?;

src/clarity.rs

@@ -31,6 +31,7 @@ use rusqlite::{Connection, OpenFlags, NO_PARAMS};
 
 use address::c32::c32_address;
 use chainstate::stacks::index::{storage::TrieFileStorage, MarfTrieId};
+use util::db::sqlite_open;
 use util::db::FromColumn;
 use util::hash::{bytes_to_hex, Sha512Trunc256Sum};
 
@@ -252,7 +253,7 @@ fn create_or_open_db(path: &String) -> Connection {
     };
 
     let conn = friendly_expect(
-        Connection::open_with_flags(path, open_flags),
+        sqlite_open(path, open_flags, false),
         &format!("FATAL: failed to open '{}'", path),
     );
     conn

src/core/mempool.rs

@@ -45,6 +45,7 @@ use monitoring::increment_stx_mempool_gc;
 use std::time::Instant;
 use util::db::query_row_columns;
 use util::db::query_rows;
+use util::db::sqlite_open;
 use util::db::tx_begin_immediate;
 use util::db::tx_busy_handler;
 use util::db::u64_to_sql;
@@ -395,8 +396,6 @@ impl MemPoolTxInfo {
 
 impl MemPoolDB {
     fn instantiate_mempool_db(conn: &mut DBConn) -> Result<(), db_error> {
-        sql_pragma(conn, "PRAGMA journal_mode = WAL;")?;
-
         let tx = tx_begin_immediate(conn)?;
 
         for cmd in MEMPOOL_INITIAL_SCHEMA {
@@ -464,13 +463,7 @@ impl MemPoolDB {
             OpenFlags::SQLITE_OPEN_READ_WRITE
         };
 
-        let mut conn =
-            DBConn::open_with_flags(&db_path, open_flags).map_err(db_error::SqliteError)?;
-        conn.busy_handler(Some(tx_busy_handler))
-            .map_err(db_error::SqliteError)?;
-
-        conn.execute_batch("PRAGMA foreign_keys = ON;")?;
-
+        let mut conn = sqlite_open(&db_path, open_flags, true)?;
         if create_flag {
             // instantiate!
             MemPoolDB::instantiate_mempool_db(&mut conn)?;

src/cost_estimates/fee_scalar.rs

@@ -10,6 +10,7 @@ use rusqlite::{
 use serde_json::Value as JsonValue;
 
 use chainstate::stacks::TransactionPayload;
+use util::db::sqlite_open;
 use util::db::tx_begin_immediate_sqlite;
 use util::db::u64_to_sql;
 
@@ -20,7 +21,6 @@ use core::BLOCK_LIMIT_MAINNET;
 use chainstate::stacks::db::StacksEpochReceipt;
 use chainstate::stacks::events::TransactionOrigin;
 
-use crate::util::db::set_wal_mode;
 use crate::util::db::sql_pragma;
 use crate::util::db::table_exists;
 
@@ -54,16 +54,16 @@ pub struct ScalarFeeRateEstimator<M: CostMetric> {
 impl<M: CostMetric> ScalarFeeRateEstimator<M> {
     /// Open a fee rate estimator at the given db path. Creates if not existent.
     pub fn open(p: &Path, metric: M) -> Result<Self, SqliteError> {
-        let db = match Connection::open_with_flags(p, rusqlite::OpenFlags::SQLITE_OPEN_READ_WRITE) {
-            Ok(db) => {
-                set_wal_mode(&db)?;
-                Ok(db)
-            }
-            Err(e) => {
+        let db =
+            sqlite_open(p, rusqlite::OpenFlags::SQLITE_OPEN_READ_WRITE, false).or_else(|e| {
                 if let SqliteError::SqliteFailure(ref internal, _) = e {
                     if let rusqlite::ErrorCode::CannotOpen = internal.code {
-                        let mut db = Connection::open(p)?;
-                        set_wal_mode(&db)?;
+                        let mut db = sqlite_open(
+                            p,
+                            rusqlite::OpenFlags::SQLITE_OPEN_CREATE
+                                | rusqlite::OpenFlags::SQLITE_OPEN_READ_WRITE,
+                            false,
+                        )?;
                         let tx = tx_begin_immediate_sqlite(&mut db)?;
                         Self::instantiate_db(&tx)?;
                         tx.commit()?;
@@ -74,8 +74,7 @@ impl<M: CostMetric> ScalarFeeRateEstimator<M> {
                 } else {
                     Err(e)
                 }
-            }
-        }?;
+            })?;
 
         Ok(Self {
             db,

src/cost_estimates/pessimistic.rs

@@ -10,12 +10,12 @@ use rusqlite::{
 use serde_json::Value as JsonValue;
 
 use chainstate::stacks::TransactionPayload;
+use util::db::sqlite_open;
 use util::db::u64_to_sql;
 use vm::costs::ExecutionCost;
 
 use core::BLOCK_LIMIT_MAINNET;
 
-use crate::util::db::set_wal_mode;
 use crate::util::db::sql_pragma;
 use crate::util::db::table_exists;
 use crate::util::db::tx_begin_immediate_sqlite;
@@ -178,16 +178,16 @@ impl Samples {
 
 impl PessimisticEstimator {
     pub fn open(p: &Path, log_error: bool) -> Result<PessimisticEstimator, EstimatorError> {
-        let db = match Connection::open_with_flags(p, rusqlite::OpenFlags::SQLITE_OPEN_READ_WRITE) {
-            Ok(db) => {
-                set_wal_mode(&db)?;
-                Ok(db)
-            }
-            Err(e) => {
+        let db =
+            sqlite_open(p, rusqlite::OpenFlags::SQLITE_OPEN_READ_WRITE, false).or_else(|e| {
                 if let SqliteError::SqliteFailure(ref internal, _) = e {
                     if let rusqlite::ErrorCode::CannotOpen = internal.code {
-                        let mut db = Connection::open(p)?;
-                        set_wal_mode(&db)?;
+                        let mut db = sqlite_open(
+                            p,
+                            rusqlite::OpenFlags::SQLITE_OPEN_CREATE
+                                | rusqlite::OpenFlags::SQLITE_OPEN_READ_WRITE,
+                            false,
+                        )?;
                         let tx = tx_begin_immediate_sqlite(&mut db)?;
                         PessimisticEstimator::instantiate_db(&tx)?;
                         tx.commit()?;
@@ -198,8 +198,8 @@ impl PessimisticEstimator {
                 } else {
                     Err(e)
                 }
-            }
-        }?;
+            })?;
+
         Ok(PessimisticEstimator { db, log_error })
     }
 

src/main.rs

@@ -63,6 +63,7 @@ use blockstack_lib::{
         stacks::db::{StacksChainState, StacksHeaderInfo},
     },
     core::MemPoolDB,
+    util::db::sqlite_open,
     util::{hash::Hash160, vrf::VRFProof},
     vm::costs::ExecutionCost,
 };
@@ -608,7 +609,7 @@ simulating a miner.
         let value_opt = marf.get(&marf_bhh, marf_key).expect("Failed to read MARF");
 
         if let Some(value) = value_opt {
-            let conn = Connection::open_with_flags(&db_path, OpenFlags::SQLITE_OPEN_READ_ONLY)
+            let conn = sqlite_open(&db_path, OpenFlags::SQLITE_OPEN_READ_ONLY, false)
                 .expect("Failed to open DB");
             let args: &[&dyn ToSql] = &[&value.to_hex()];
             let res: Result<String, rusqlite::Error> = conn.query_row_and_then(

src/monitoring/mod.rs

@@ -31,6 +31,7 @@ use burnchains::BurnchainSigner;
 use std::error::Error;
 use std::sync::atomic::{AtomicUsize, Ordering};
 use std::sync::Mutex;
+use util::db::sqlite_open;
 use util::db::Error as DatabaseError;
 use util::uint::{Uint256, Uint512};
 
@@ -144,9 +145,7 @@ fn txid_tracking_db(chainstate_root_path: &str) -> Result<DBConn, DatabaseError>
         OpenFlags::SQLITE_OPEN_READ_WRITE
     };
 
-    let conn = DBConn::open_with_flags(&db_path, open_flags)?;
-
-    conn.busy_handler(Some(tx_busy_handler))?;
+    let conn = sqlite_open(&db_path, open_flags, false)?;
 
     if create_flag {
         conn.execute(
@@ -195,8 +194,7 @@ pub fn log_transaction_processed(
     #[cfg(feature = "monitoring_prom")]
     {
         let mempool_db_path = MemPoolDB::db_path(chainstate_root_path)?;
-        let mempool_conn =
-            DBConn::open_with_flags(&mempool_db_path, OpenFlags::SQLITE_OPEN_READ_ONLY)?;
+        let mempool_conn = sqlite_open(&mempool_db_path, OpenFlags::SQLITE_OPEN_READ_ONLY, false)?;
         let tracking_db = txid_tracking_db(chainstate_root_path)?;
 
         let tx = match MemPoolDB::get_tx(&mempool_conn, txid)? {

src/net/atlas/db.rs

@@ -24,6 +24,7 @@ use std::convert::From;
 use std::convert::TryFrom;
 use std::fs;
 
+use util::db::sqlite_open;
 use util::db::tx_begin_immediate;
 use util::db::DBConn;
 use util::db::Error as db_error;
@@ -197,9 +198,8 @@ impl AtlasDB {
                 OpenFlags::SQLITE_OPEN_READ_ONLY
             }
         };
-        let conn =
-            Connection::open_with_flags(path, open_flags).map_err(|e| db_error::SqliteError(e))?;
 
+        let conn = sqlite_open(path, open_flags, false)?;
         let mut db = AtlasDB {
             atlas_config,
             conn,

src/net/db.rs

@@ -25,6 +25,7 @@ use std::convert::From;
 use std::convert::TryFrom;
 use std::fs;
 
+use util::db::sqlite_open;
 use util::db::tx_begin_immediate;
 use util::db::DBConn;
 use util::db::Error as db_error;
@@ -526,10 +527,8 @@ impl PeerDB {
             }
         };
 
-        let conn =
-            Connection::open_with_flags(path, open_flags).map_err(|e| db_error::SqliteError(e))?;
+        let conn = sqlite_open(path, open_flags, false)?;
 
-        conn.busy_handler(Some(tx_busy_handler))?;
         let mut db = PeerDB {
             conn: conn,
             readwrite: readwrite,