diff --git a/src/burnchains/bitcoin/spv.rs b/src/burnchains/bitcoin/spv.rs index 5f597b64c0..3601f0ffff 100644 --- a/src/burnchains/bitcoin/spv.rs +++ b/src/burnchains/bitcoin/spv.rs @@ -42,8 +42,8 @@ use rusqlite::Transaction; use rusqlite::{Connection, OpenFlags, NO_PARAMS}; use util::db::{ - query_row, query_rows, tx_begin_immediate, tx_busy_handler, u64_to_sql, DBConn, DBTx, - Error as db_error, FromColumn, FromRow, + query_row, query_rows, sqlite_open, tx_begin_immediate, tx_busy_handler, u64_to_sql, DBConn, + DBTx, Error as db_error, FromColumn, FromRow, }; use util::get_epoch_time_secs; use util::hash::{hex_bytes, to_hex}; @@ -212,11 +212,8 @@ impl SpvClient { } }; - let mut conn = - Connection::open_with_flags(headers_path, open_flags).map_err(db_error::SqliteError)?; - - conn.busy_handler(Some(tx_busy_handler)) - .map_err(db_error::SqliteError)?; + let mut conn = sqlite_open(headers_path, open_flags, false) + .map_err(|e| btc_error::DBError(db_error::SqliteError(e)))?; if create_flag { SpvClient::db_instantiate(&mut conn)?; diff --git a/src/burnchains/db.rs b/src/burnchains/db.rs index f4e45c2358..f312968524 100644 --- a/src/burnchains/db.rs +++ b/src/burnchains/db.rs @@ -27,8 +27,8 @@ use burnchains::{Burnchain, BurnchainBlock, BurnchainBlockHeader, Error as Burnc use chainstate::burn::operations::BlockstackOperationType; use chainstate::stacks::index::MarfTrieId; use util::db::{ - query_row, query_rows, sql_pragma, tx_begin_immediate, tx_busy_handler, u64_to_sql, - Error as DBError, FromColumn, FromRow, + query_row, query_rows, sql_pragma, sqlite_open, tx_begin_immediate, tx_busy_handler, + u64_to_sql, Error as DBError, FromColumn, FromRow, }; use crate::types::chainstate::BurnchainHeaderHash; @@ -209,16 +209,11 @@ impl BurnchainDB { } }; - let conn = Connection::open_with_flags(path, open_flags) - .expect(&format!("FAILED to open: {}", path)); - - conn.busy_handler(Some(tx_busy_handler))?; - + let conn = sqlite_open(path, open_flags, true)?; let mut db = BurnchainDB { conn }; if create_flag { let db_tx = db.tx_begin()?; - sql_pragma(&db_tx.sql_tx, "PRAGMA journal_mode = WAL;")?; db_tx.sql_tx.execute_batch(BURNCHAIN_DB_INITIAL_SCHEMA)?; db_tx.sql_tx.execute( @@ -247,9 +242,7 @@ impl BurnchainDB { } else { OpenFlags::SQLITE_OPEN_READ_ONLY }; - let conn = Connection::open_with_flags(path, open_flags)?; - conn.busy_handler(Some(tx_busy_handler))?; - + let conn = sqlite_open(path, open_flags, true)?; Ok(BurnchainDB { conn }) } diff --git a/src/chainstate/burn/db/sortdb.rs b/src/chainstate/burn/db/sortdb.rs index db1353e41d..76e13d6cf0 100644 --- a/src/chainstate/burn/db/sortdb.rs +++ b/src/chainstate/burn/db/sortdb.rs @@ -1980,7 +1980,9 @@ impl SortitionDB { fn open_index(index_path: &str) -> Result, db_error> { test_debug!("Open index at {}", index_path); - MARF::from_path(index_path).map_err(|_e| db_error::Corruption) + let marf = MARF::from_path(index_path).map_err(|_e| db_error::Corruption)?; + sql_pragma(marf.sqlite_conn(), "foreign_keys", &true)?; + Ok(marf) } /// Open the database on disk. It must already exist and be instantiated. @@ -2004,6 +2006,7 @@ impl SortitionDB { first_block_height: first_snapshot.block_height, first_burn_header_hash: first_snapshot.burn_header_hash.clone(), }; + Ok(db) } @@ -2101,7 +2104,8 @@ impl SortitionDB { ) -> Result<(), db_error> { debug!("Instantiate SortDB"); - sql_pragma(self.conn(), "PRAGMA journal_mode = WAL;")?; + sql_pragma(self.conn(), "journal_mode", &"WAL")?; + sql_pragma(self.conn(), "foreign_keys", &true)?; let mut db_tx = SortitionHandleTx::begin(self, &SortitionId::sentinel())?; diff --git a/src/chainstate/stacks/index/storage.rs b/src/chainstate/stacks/index/storage.rs index 02080c1273..c3e0d9a268 100644 --- a/src/chainstate/stacks/index/storage.rs +++ b/src/chainstate/stacks/index/storage.rs @@ -45,6 +45,7 @@ use chainstate::stacks::index::node::{ }; use chainstate::stacks::index::Error; use chainstate::stacks::index::{trie_sql, BlockMap, MarfTrieId}; +use util::db::sqlite_open; use util::db::tx_begin_immediate; use util::db::tx_busy_handler; use util::db::Error as db_error; @@ -795,9 +796,7 @@ impl TrieFileStorage { } }; - let mut db = Connection::open_with_flags(db_path, open_flags)?; - db.busy_handler(Some(tx_busy_handler))?; - + let mut db = sqlite_open(db_path, open_flags, false)?; let db_path = db_path.to_string(); if create_flag { @@ -866,8 +865,7 @@ impl TrieFileStorage { } pub fn reopen_readonly(&self) -> Result, Error> { - let db = Connection::open_with_flags(&self.db_path, OpenFlags::SQLITE_OPEN_READ_ONLY)?; - db.busy_handler(Some(tx_busy_handler))?; + let db = sqlite_open(&self.db_path, OpenFlags::SQLITE_OPEN_READ_ONLY, false)?; trace!("Make read-only view of TrieFileStorage: {}", &self.db_path); @@ -911,8 +909,7 @@ impl<'a, T: MarfTrieId> TrieStorageTransaction<'a, T> { /// reopen this transaction as a read-only marf. /// _does not_ preserve the cur_block/open tip pub fn reopen_readonly(&self) -> Result, Error> { - let db = Connection::open_with_flags(&self.db_path, OpenFlags::SQLITE_OPEN_READ_ONLY)?; - db.busy_handler(Some(tx_busy_handler))?; + let db = sqlite_open(&self.db_path, OpenFlags::SQLITE_OPEN_READ_ONLY, false)?; trace!( "Make read-only view of TrieStorageTransaction: {}", @@ -1286,9 +1283,7 @@ impl<'a, T: MarfTrieId> TrieStorageConnection<'a, T> { /// Recover from partially-written state -- i.e. blow it away. /// Doesn't get called automatically. pub fn recover(db_path: &String) -> Result<(), Error> { - let conn = Connection::open(db_path)?; - conn.busy_handler(Some(tx_busy_handler))?; - + let conn = sqlite_open(db_path, OpenFlags::SQLITE_OPEN_READ_WRITE, false)?; trie_sql::clear_lock_data(&conn) } diff --git a/src/chainstate/stacks/index/trie_sql.rs b/src/chainstate/stacks/index/trie_sql.rs index 456c5cb14d..efa598324b 100644 --- a/src/chainstate/stacks/index/trie_sql.rs +++ b/src/chainstate/stacks/index/trie_sql.rs @@ -83,8 +83,6 @@ CREATE TABLE IF NOT EXISTS block_extension_locks (block_hash TEXT PRIMARY KEY); "; pub fn create_tables_if_needed(conn: &mut Connection) -> Result<(), Error> { - sql_pragma(conn, "PRAGMA journal_mode = WAL;")?; - let tx = tx_begin_immediate(conn)?; tx.execute_batch(SQL_MARF_DATA_TABLE)?; diff --git a/src/clarity.rs b/src/clarity.rs index 104b88d869..7539e2140b 100644 --- a/src/clarity.rs +++ b/src/clarity.rs @@ -31,6 +31,7 @@ use rusqlite::{Connection, OpenFlags, NO_PARAMS}; use address::c32::c32_address; use chainstate::stacks::index::{storage::TrieFileStorage, MarfTrieId}; +use util::db::sqlite_open; use util::db::FromColumn; use util::hash::{bytes_to_hex, Sha512Trunc256Sum}; @@ -252,7 +253,7 @@ fn create_or_open_db(path: &String) -> Connection { }; let conn = friendly_expect( - Connection::open_with_flags(path, open_flags), + sqlite_open(path, open_flags, false), &format!("FATAL: failed to open '{}'", path), ); conn diff --git a/src/core/mempool.rs b/src/core/mempool.rs index fcea1a2c2a..98df52add6 100644 --- a/src/core/mempool.rs +++ b/src/core/mempool.rs @@ -45,6 +45,7 @@ use monitoring::increment_stx_mempool_gc; use std::time::Instant; use util::db::query_row_columns; use util::db::query_rows; +use util::db::sqlite_open; use util::db::tx_begin_immediate; use util::db::tx_busy_handler; use util::db::u64_to_sql; @@ -395,8 +396,6 @@ impl MemPoolTxInfo { impl MemPoolDB { fn instantiate_mempool_db(conn: &mut DBConn) -> Result<(), db_error> { - sql_pragma(conn, "PRAGMA journal_mode = WAL;")?; - let tx = tx_begin_immediate(conn)?; for cmd in MEMPOOL_INITIAL_SCHEMA { @@ -464,13 +463,7 @@ impl MemPoolDB { OpenFlags::SQLITE_OPEN_READ_WRITE }; - let mut conn = - DBConn::open_with_flags(&db_path, open_flags).map_err(db_error::SqliteError)?; - conn.busy_handler(Some(tx_busy_handler)) - .map_err(db_error::SqliteError)?; - - conn.execute_batch("PRAGMA foreign_keys = ON;")?; - + let mut conn = sqlite_open(&db_path, open_flags, true)?; if create_flag { // instantiate! MemPoolDB::instantiate_mempool_db(&mut conn)?; diff --git a/src/cost_estimates/fee_scalar.rs b/src/cost_estimates/fee_scalar.rs index 8b239be034..d2181c1542 100644 --- a/src/cost_estimates/fee_scalar.rs +++ b/src/cost_estimates/fee_scalar.rs @@ -10,6 +10,7 @@ use rusqlite::{ use serde_json::Value as JsonValue; use chainstate::stacks::TransactionPayload; +use util::db::sqlite_open; use util::db::tx_begin_immediate_sqlite; use util::db::u64_to_sql; @@ -20,7 +21,6 @@ use core::BLOCK_LIMIT_MAINNET; use chainstate::stacks::db::StacksEpochReceipt; use chainstate::stacks::events::TransactionOrigin; -use crate::util::db::set_wal_mode; use crate::util::db::sql_pragma; use crate::util::db::table_exists; @@ -54,16 +54,16 @@ pub struct ScalarFeeRateEstimator { impl ScalarFeeRateEstimator { /// Open a fee rate estimator at the given db path. Creates if not existent. pub fn open(p: &Path, metric: M) -> Result { - let db = match Connection::open_with_flags(p, rusqlite::OpenFlags::SQLITE_OPEN_READ_WRITE) { - Ok(db) => { - set_wal_mode(&db)?; - Ok(db) - } - Err(e) => { + let db = + sqlite_open(p, rusqlite::OpenFlags::SQLITE_OPEN_READ_WRITE, false).or_else(|e| { if let SqliteError::SqliteFailure(ref internal, _) = e { if let rusqlite::ErrorCode::CannotOpen = internal.code { - let mut db = Connection::open(p)?; - set_wal_mode(&db)?; + let mut db = sqlite_open( + p, + rusqlite::OpenFlags::SQLITE_OPEN_CREATE + | rusqlite::OpenFlags::SQLITE_OPEN_READ_WRITE, + false, + )?; let tx = tx_begin_immediate_sqlite(&mut db)?; Self::instantiate_db(&tx)?; tx.commit()?; @@ -74,8 +74,7 @@ impl ScalarFeeRateEstimator { } else { Err(e) } - } - }?; + })?; Ok(Self { db, diff --git a/src/cost_estimates/pessimistic.rs b/src/cost_estimates/pessimistic.rs index e8ed274007..bc2e2bd60b 100644 --- a/src/cost_estimates/pessimistic.rs +++ b/src/cost_estimates/pessimistic.rs @@ -10,12 +10,12 @@ use rusqlite::{ use serde_json::Value as JsonValue; use chainstate::stacks::TransactionPayload; +use util::db::sqlite_open; use util::db::u64_to_sql; use vm::costs::ExecutionCost; use core::BLOCK_LIMIT_MAINNET; -use crate::util::db::set_wal_mode; use crate::util::db::sql_pragma; use crate::util::db::table_exists; use crate::util::db::tx_begin_immediate_sqlite; @@ -178,16 +178,16 @@ impl Samples { impl PessimisticEstimator { pub fn open(p: &Path, log_error: bool) -> Result { - let db = match Connection::open_with_flags(p, rusqlite::OpenFlags::SQLITE_OPEN_READ_WRITE) { - Ok(db) => { - set_wal_mode(&db)?; - Ok(db) - } - Err(e) => { + let db = + sqlite_open(p, rusqlite::OpenFlags::SQLITE_OPEN_READ_WRITE, false).or_else(|e| { if let SqliteError::SqliteFailure(ref internal, _) = e { if let rusqlite::ErrorCode::CannotOpen = internal.code { - let mut db = Connection::open(p)?; - set_wal_mode(&db)?; + let mut db = sqlite_open( + p, + rusqlite::OpenFlags::SQLITE_OPEN_CREATE + | rusqlite::OpenFlags::SQLITE_OPEN_READ_WRITE, + false, + )?; let tx = tx_begin_immediate_sqlite(&mut db)?; PessimisticEstimator::instantiate_db(&tx)?; tx.commit()?; @@ -198,8 +198,8 @@ impl PessimisticEstimator { } else { Err(e) } - } - }?; + })?; + Ok(PessimisticEstimator { db, log_error }) } diff --git a/src/main.rs b/src/main.rs index e27d65b1b4..0ee4956516 100644 --- a/src/main.rs +++ b/src/main.rs @@ -63,6 +63,7 @@ use blockstack_lib::{ stacks::db::{StacksChainState, StacksHeaderInfo}, }, core::MemPoolDB, + util::db::sqlite_open, util::{hash::Hash160, vrf::VRFProof}, vm::costs::ExecutionCost, }; @@ -608,7 +609,7 @@ simulating a miner. let value_opt = marf.get(&marf_bhh, marf_key).expect("Failed to read MARF"); if let Some(value) = value_opt { - let conn = Connection::open_with_flags(&db_path, OpenFlags::SQLITE_OPEN_READ_ONLY) + let conn = sqlite_open(&db_path, OpenFlags::SQLITE_OPEN_READ_ONLY, false) .expect("Failed to open DB"); let args: &[&dyn ToSql] = &[&value.to_hex()]; let res: Result = conn.query_row_and_then( diff --git a/src/monitoring/mod.rs b/src/monitoring/mod.rs index 8ae6360826..bc38e98fde 100644 --- a/src/monitoring/mod.rs +++ b/src/monitoring/mod.rs @@ -31,6 +31,7 @@ use burnchains::BurnchainSigner; use std::error::Error; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Mutex; +use util::db::sqlite_open; use util::db::Error as DatabaseError; use util::uint::{Uint256, Uint512}; @@ -144,9 +145,7 @@ fn txid_tracking_db(chainstate_root_path: &str) -> Result OpenFlags::SQLITE_OPEN_READ_WRITE }; - let conn = DBConn::open_with_flags(&db_path, open_flags)?; - - conn.busy_handler(Some(tx_busy_handler))?; + let conn = sqlite_open(&db_path, open_flags, false)?; if create_flag { conn.execute( @@ -195,8 +194,7 @@ pub fn log_transaction_processed( #[cfg(feature = "monitoring_prom")] { let mempool_db_path = MemPoolDB::db_path(chainstate_root_path)?; - let mempool_conn = - DBConn::open_with_flags(&mempool_db_path, OpenFlags::SQLITE_OPEN_READ_ONLY)?; + let mempool_conn = sqlite_open(&mempool_db_path, OpenFlags::SQLITE_OPEN_READ_ONLY, false)?; let tracking_db = txid_tracking_db(chainstate_root_path)?; let tx = match MemPoolDB::get_tx(&mempool_conn, txid)? { diff --git a/src/net/atlas/db.rs b/src/net/atlas/db.rs index c839b06055..c145fffb94 100644 --- a/src/net/atlas/db.rs +++ b/src/net/atlas/db.rs @@ -24,6 +24,7 @@ use std::convert::From; use std::convert::TryFrom; use std::fs; +use util::db::sqlite_open; use util::db::tx_begin_immediate; use util::db::DBConn; use util::db::Error as db_error; @@ -197,9 +198,8 @@ impl AtlasDB { OpenFlags::SQLITE_OPEN_READ_ONLY } }; - let conn = - Connection::open_with_flags(path, open_flags).map_err(|e| db_error::SqliteError(e))?; + let conn = sqlite_open(path, open_flags, false)?; let mut db = AtlasDB { atlas_config, conn, diff --git a/src/net/db.rs b/src/net/db.rs index 1876696d92..6f4da80649 100644 --- a/src/net/db.rs +++ b/src/net/db.rs @@ -25,6 +25,7 @@ use std::convert::From; use std::convert::TryFrom; use std::fs; +use util::db::sqlite_open; use util::db::tx_begin_immediate; use util::db::DBConn; use util::db::Error as db_error; @@ -526,10 +527,8 @@ impl PeerDB { } }; - let conn = - Connection::open_with_flags(path, open_flags).map_err(|e| db_error::SqliteError(e))?; + let conn = sqlite_open(path, open_flags, false)?; - conn.busy_handler(Some(tx_busy_handler))?; let mut db = PeerDB { conn: conn, readwrite: readwrite, diff --git a/src/util/db.rs b/src/util/db.rs index 27307da557..3d69d801e2 100644 --- a/src/util/db.rs +++ b/src/util/db.rs @@ -23,6 +23,7 @@ use std::io; use std::io::Error as IOError; use std::ops::Deref; use std::ops::DerefMut; +use std::path::Path; use std::path::PathBuf; use util::hash::to_hex; @@ -37,6 +38,7 @@ use rusqlite::types::{ }; use rusqlite::Connection; use rusqlite::Error as sqlite_error; +use rusqlite::OpenFlags; use rusqlite::OptionalExtension; use rusqlite::Row; use rusqlite::Transaction; @@ -400,14 +402,20 @@ where /// Run a PRAGMA statement. This can't always be done via execute(), because it may return a result (and /// rusqlite does not like this). -pub fn sql_pragma(conn: &Connection, pragma_stmt: &str) -> Result<(), Error> { - conn.query_row_and_then(pragma_stmt, NO_PARAMS, |_row| Ok(())) +pub fn sql_pragma( + conn: &Connection, + pragma_name: &str, + pragma_value: &dyn ToSql, +) -> Result<(), Error> { + inner_sql_pragma(conn, pragma_name, pragma_value).map_err(|e| Error::SqliteError(e)) } -pub fn set_wal_mode(conn: &Connection) -> Result<(), sqlite_error> { - conn.query_row("PRAGMA journal_mode = WAL;", rusqlite::NO_PARAMS, |_row| { - Ok(()) - }) +fn inner_sql_pragma( + conn: &Connection, + pragma_name: &str, + pragma_value: &dyn ToSql, +) -> Result<(), sqlite_error> { + conn.pragma_update(None, pragma_name, pragma_value) } /// Returns true if the database table `table_name` exists in the active @@ -551,6 +559,21 @@ pub fn tx_begin_immediate_sqlite<'a>(conn: &'a mut Connection) -> Result>( + path: P, + flags: OpenFlags, + foreign_keys: bool, +) -> Result { + let db = Connection::open_with_flags(path, flags)?; + db.busy_handler(Some(tx_busy_handler))?; + inner_sql_pragma(&db, "journal_mode", &"WAL")?; + if foreign_keys { + inner_sql_pragma(&db, "foreign_keys", &true)?; + } + Ok(db) +} + /// Get the ancestor block hash of a block of a given height, given a descendent block hash. pub fn get_ancestor_block_hash( index: &MARF, @@ -785,3 +808,41 @@ impl<'a, C: Clone, T: MarfTrieId> Drop for IndexDBTx<'a, C, T> { } } } + +#[cfg(test)] +mod tests { + use super::*; + use std::fs; + + #[test] + fn test_pragma() { + let path = "/tmp/blockstack_db_test_pragma.db"; + if fs::metadata(path).is_ok() { + fs::remove_file(path).unwrap(); + } + + // calls pragma_update with both journal_mode and foreign_keys + let db = sqlite_open( + path, + OpenFlags::SQLITE_OPEN_CREATE | OpenFlags::SQLITE_OPEN_READ_WRITE, + true, + ) + .unwrap(); + + // journal mode must be WAL + db.pragma_query(None, "journal_mode", |row| { + let value: String = row.get(0)?; + assert_eq!(value, "wal"); + Ok(()) + }) + .unwrap(); + + // foreign keys must be on + db.pragma_query(None, "foreign_keys", |row| { + let value: i64 = row.get(0)?; + assert_eq!(value, 1); + Ok(()) + }) + .unwrap(); + } +}