diff --git a/src/bitcoin/poller/looper.rs b/src/bitcoin/poller/looper.rs index 93c85749..8ca1c73e 100644 --- a/src/bitcoin/poller/looper.rs +++ b/src/bitcoin/poller/looper.rs @@ -156,6 +156,40 @@ fn update_coins( } } +// Add new deposit and spend transactions to the database. +fn add_txs_to_db( + bit: &impl BitcoinInterface, + db_conn: &mut Box, + updated_coins: &UpdatedCoins, +) { + let curr_txids: HashSet<_> = db_conn.list_saved_txids().into_iter().collect(); + let mut new_txids = HashSet::new(); + // First get all newly received coins that have not expired. + new_txids.extend(updated_coins.received.iter().filter_map(|c| { + if !updated_coins.expired.contains(&c.outpoint) { + Some(c.outpoint.txid) + } else { + None + } + })); + + // Add spend txid for new & existing coins. + new_txids.extend(updated_coins.spending.iter().map(|(_, txid)| txid)); + + // Remove those txids we already have. + let missing_txids = new_txids.difference(&curr_txids); + log::debug!("Missing txids: {:?}", missing_txids); + + // Now retrieve txs. + let txs: Vec<_> = missing_txids + .map(|txid| bit.wallet_transaction(txid).map(|(tx, _)| tx)) + .collect::>>() + .expect("we must retrieve all txs"); + if !txs.is_empty() { + db_conn.new_txs(&txs); + } +} + #[derive(Debug, Clone, Copy)] enum TipUpdate { // The best block is still the same as in the previous poll. @@ -233,6 +267,8 @@ fn updates( return updates(db_conn, bit, descs, secp); } + // Transactions must be added to the DB before coins due to foreign key constraints. + add_txs_to_db(bit, db_conn, &updated_coins); // The chain tip did not change since we started our updates. Record them and the latest tip. // Having the tip in database means that, as far as the chain is concerned, we've got all // updates up to this block. But not more. diff --git a/src/commands/mod.rs b/src/commands/mod.rs index 276758db..6e2b2063 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -155,27 +155,33 @@ impl fmt::Display for RbfErrorInfo { } } -/// A wallet transaction getter which fetches the transaction from our Bitcoin backend with a cache +/// A wallet transaction getter which fetches the transaction from our database backend with a cache /// to avoid needless redundant calls. Note the cache holds an Option<> so we also avoid redundant -/// calls when the txid isn't known by our Bitcoin backend. -struct BitcoindTxGetter<'a> { - bitcoind: &'a sync::Arc>, +/// calls when the txid isn't known by our database backend. +struct DbTxGetter<'a> { + db: &'a sync::Arc>, cache: HashMap>, } -impl<'a> BitcoindTxGetter<'a> { - pub fn new(bitcoind: &'a sync::Arc>) -> Self { +impl<'a> DbTxGetter<'a> { + pub fn new(db: &'a sync::Arc>) -> Self { Self { - bitcoind, + db, cache: HashMap::new(), } } } -impl<'a> TxGetter for BitcoindTxGetter<'a> { +impl<'a> TxGetter for DbTxGetter<'a> { fn get_tx(&mut self, txid: &bitcoin::Txid) -> Option { if let hash_map::Entry::Vacant(entry) = self.cache.entry(*txid) { - entry.insert(self.bitcoind.wallet_transaction(txid).map(|wtx| wtx.0)); + let tx = self + .db + .connection() + .list_wallet_transactions(&[*txid]) + .pop() + .map(|(tx, _, _)| tx); + entry.insert(tx); } self.cache.get(txid).cloned().flatten() } @@ -457,7 +463,7 @@ impl DaemonControl { return Err(CommandError::InvalidFeerate(feerate_vb)); } let mut db_conn = self.db.connection(); - let mut tx_getter = BitcoindTxGetter::new(&self.bitcoin); + let mut tx_getter = DbTxGetter::new(&self.db); // Prepare the destination addresses. let mut destinations_checked = Vec::with_capacity(destinations.len()); @@ -754,7 +760,7 @@ impl DaemonControl { feerate_vb: Option, ) -> Result { let mut db_conn = self.db.connection(); - let mut tx_getter = BitcoindTxGetter::new(&self.bitcoin); + let mut tx_getter = DbTxGetter::new(&self.db); if is_cancel && feerate_vb.is_some() { return Err(CommandError::RbfError(RbfErrorInfo::SuperfluousFeerate)); @@ -1015,39 +1021,19 @@ impl DaemonControl { limit: u64, ) -> ListTransactionsResult { let mut db_conn = self.db.connection(); + // Note the result could in principle be retrieved in a single database query. let txids = db_conn.list_txids(start, end, limit); - let transactions = txids - .iter() - .filter_map(|txid| { - // TODO: batch those calls to the Bitcoin backend - // so it can in turn optimize its queries. - self.bitcoin - .wallet_transaction(txid) - .map(|(tx, block)| TransactionInfo { - tx, - height: block.map(|b| b.height), - time: block.map(|b| b.time), - }) - }) - .collect(); - ListTransactionsResult { transactions } + self.list_transactions(&txids) } /// list_transactions retrieves the transactions with the given txids. pub fn list_transactions(&self, txids: &[bitcoin::Txid]) -> ListTransactionsResult { - let transactions = txids - .iter() - .filter_map(|txid| { - // TODO: batch those calls to the Bitcoin backend - // so it can in turn optimize its queries. - self.bitcoin - .wallet_transaction(txid) - .map(|(tx, block)| TransactionInfo { - tx, - height: block.map(|b| b.height), - time: block.map(|b| b.time), - }) - }) + let transactions = self + .db + .connection() + .list_wallet_transactions(txids) + .into_iter() + .map(|(tx, height, time)| TransactionInfo { tx, height, time }) .collect(); ListTransactionsResult { transactions } } @@ -1068,7 +1054,7 @@ impl DaemonControl { if feerate_vb < 1 { return Err(CommandError::InvalidFeerate(feerate_vb)); } - let mut tx_getter = BitcoindTxGetter::new(&self.bitcoin); + let mut tx_getter = DbTxGetter::new(&self.db); let mut db_conn = self.db.connection(); let sweep_addr = self.spend_addr(&mut db_conn, self.validate_address(address)?); @@ -1422,25 +1408,17 @@ mod tests { #[test] fn create_spend() { - let dummy_op = bitcoin::OutPoint::from_str( - "3753a1d74c0af8dd0a0f3b763c14faf3bd9ed03cbdf33337a074fb0e9f6c7810:0", - ) - .unwrap(); - let mut dummy_bitcoind = DummyBitcoind::new(); - dummy_bitcoind.txs.insert( - dummy_op.txid, - ( - bitcoin::Transaction { - version: TxVersion::TWO, - lock_time: absolute::LockTime::Blocks(absolute::Height::ZERO), - input: vec![], - output: vec![], - }, - None, - ), - ); - let ms = DummyLiana::new(dummy_bitcoind, DummyDatabase::new()); + let dummy_tx = bitcoin::Transaction { + version: TxVersion::TWO, + lock_time: absolute::LockTime::Blocks(absolute::Height::ZERO), + input: vec![], + output: vec![], + }; + let dummy_op = bitcoin::OutPoint::new(dummy_tx.txid(), 0); + let ms = DummyLiana::new(DummyBitcoind::new(), DummyDatabase::new()); let control = &ms.control(); + let mut db_conn = control.db().lock().unwrap().connection(); + db_conn.new_txs(&[dummy_tx]); // Arguments sanity checking let dummy_addr = @@ -1471,7 +1449,6 @@ mod tests { control.create_spend(&destinations, &[dummy_op], 1, None), Err(CommandError::UnknownOutpoint(dummy_op)) ); - let mut db_conn = control.db().lock().unwrap().connection(); db_conn.new_unspent_coins(&[Coin { outpoint: dummy_op, is_immature: false, @@ -2305,8 +2282,8 @@ mod tests { }, ]); - let mut btc = DummyBitcoind::new(); - btc.txs.insert( + let mut txs_map = HashMap::new(); + txs_map.insert( deposit1.txid(), ( deposit1.clone(), @@ -2320,7 +2297,7 @@ mod tests { }), ), ); - btc.txs.insert( + txs_map.insert( deposit2.txid(), ( deposit2.clone(), @@ -2334,7 +2311,7 @@ mod tests { }), ), ); - btc.txs.insert( + txs_map.insert( spend_tx.txid(), ( spend_tx.clone(), @@ -2348,7 +2325,7 @@ mod tests { }), ), ); - btc.txs.insert( + txs_map.insert( deposit3.txid(), ( deposit3.clone(), @@ -2363,11 +2340,15 @@ mod tests { ), ); - let ms = DummyLiana::new(btc, db); + let ms = DummyLiana::new(DummyBitcoind::new(), db); let control = &ms.control(); + let mut db_conn = control.db.connection(); + let txs: Vec<_> = txs_map.values().map(|(tx, _)| tx.clone()).collect(); + db_conn.new_txs(&txs); - let transactions = control.list_confirmed_transactions(0, 4, 10).transactions; + let mut transactions = control.list_confirmed_transactions(0, 4, 10).transactions; + transactions.sort_by(|tx1, tx2| tx2.height.cmp(&tx1.height)); assert_eq!(transactions.len(), 4); assert_eq!(transactions[0].time, Some(4)); @@ -2382,7 +2363,8 @@ mod tests { assert_eq!(transactions[3].time, Some(1)); assert_eq!(transactions[3].tx, deposit1); - let transactions = control.list_confirmed_transactions(2, 3, 10).transactions; + let mut transactions = control.list_confirmed_transactions(2, 3, 10).transactions; + transactions.sort_by(|tx1, tx2| tx2.height.cmp(&tx1.height)); assert_eq!(transactions.len(), 2); assert_eq!(transactions[0].time, Some(3)); @@ -2451,8 +2433,8 @@ mod tests { }], }; - let mut btc = DummyBitcoind::new(); - btc.txs.insert( + let mut txs_map = HashMap::new(); + txs_map.insert( tx1.txid(), ( tx1.clone(), @@ -2466,7 +2448,7 @@ mod tests { }), ), ); - btc.txs.insert( + txs_map.insert( tx2.txid(), ( tx2.clone(), @@ -2480,7 +2462,7 @@ mod tests { }), ), ); - btc.txs.insert( + txs_map.insert( tx3.txid(), ( tx3.clone(), @@ -2495,9 +2477,31 @@ mod tests { ), ); - let ms = DummyLiana::new(btc, DummyDatabase::new()); - + let ms = DummyLiana::new(DummyBitcoind::new(), DummyDatabase::new()); let control = &ms.control(); + let mut db_conn = control.db.connection(); + let txs: Vec<_> = txs_map.values().map(|(tx, _)| tx.clone()).collect(); + db_conn.new_txs(&txs); + // We need coins in the DB in order to get the block info for the transactions. + for (txid, (_tx, block)) in txs_map { + // Insert more than one coin per transaction to check that the command does not + // return duplicate transactions. + for vout in 0..4 { + db_conn.new_unspent_coins(&[Coin { + outpoint: bitcoin::OutPoint::new(txid, vout), + is_immature: false, + block_info: block.map(|b| BlockInfo { + height: b.height, + time: b.time, + }), + amount: bitcoin::Amount::from_sat(100_000), + derivation_index: bip32::ChildNumber::from(13), + is_change: false, + spend_txid: None, + spend_block: None, + }]); + } + } let transactions = control.list_transactions(&[tx1.txid()]).transactions; assert_eq!(transactions.len(), 1); diff --git a/src/database/mod.rs b/src/database/mod.rs index cf06eaf4..3940b7eb 100644 --- a/src/database/mod.rs +++ b/src/database/mod.rs @@ -146,6 +146,18 @@ pub trait DatabaseConnection { /// Retrieve a limited list of txids that where deposited or spent between the start and end timestamps (inclusive bounds) fn list_txids(&mut self, start: u32, end: u32, limit: u64) -> Vec; + + /// Retrieves all txids from the transactions table whether or not they are referenced by a coin. + fn list_saved_txids(&mut self) -> Vec; + + /// Store transactions in database, ignoring any that already exist. + fn new_txs(&mut self, txs: &[bitcoin::Transaction]); + + /// Retrieve a list of transactions and their corresponding block heights and times. + fn list_wallet_transactions( + &mut self, + txids: &[bitcoin::Txid], + ) -> Vec<(bitcoin::Transaction, Option, Option)>; } impl DatabaseConnection for SqliteConn { @@ -310,6 +322,30 @@ impl DatabaseConnection for SqliteConn { fn list_txids(&mut self, start: u32, end: u32, limit: u64) -> Vec { self.db_list_txids(start, end, limit) } + + fn list_saved_txids(&mut self) -> Vec { + self.db_list_saved_txids() + } + + fn new_txs<'a>(&mut self, txs: &[bitcoin::Transaction]) { + self.new_txs(txs) + } + + fn list_wallet_transactions( + &mut self, + txids: &[bitcoin::Txid], + ) -> Vec<(bitcoin::Transaction, Option, Option)> { + self.list_wallet_transactions(txids) + .into_iter() + .map(|wtx| { + ( + wtx.transaction, + wtx.block_info.map(|b| b.height), + wtx.block_info.map(|b| b.time), + ) + }) + .collect() + } } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] diff --git a/src/database/sqlite/mod.rs b/src/database/sqlite/mod.rs index ad99ac55..fb3b2e1b 100644 --- a/src/database/sqlite/mod.rs +++ b/src/database/sqlite/mod.rs @@ -16,7 +16,7 @@ use crate::{ sqlite::{ schema::{ DbAddress, DbCoin, DbLabel, DbLabelledKind, DbSpendTransaction, DbTip, DbWallet, - SCHEMA, + DbWalletTransaction, SCHEMA, }, utils::{ create_fresh_db, curr_timestamp, db_exec, db_query, db_tx_query, db_version, @@ -43,7 +43,11 @@ use miniscript::bitcoin::{ secp256k1, }; -const DB_VERSION: i64 = 4; +const DB_VERSION: i64 = 5; + +/// Last database version for which Bitcoin transactions were not stored in database. In practice +/// this meant we relied on the bitcoind watchonly wallet to store them for us. +pub const MAX_DB_VERSION_NO_TX_DB: i64 = 4; #[derive(Debug)] pub enum SqliteDbError { @@ -140,6 +144,7 @@ pub struct SqliteDb { impl SqliteDb { /// Instanciate an SQLite database either from an existing database file or by creating a fresh /// one. + /// NOTE: don't forget to apply any migration with `maybe_apply_migration` if necessary. pub fn new( db_path: path::PathBuf, fresh_options: Option, @@ -155,11 +160,20 @@ impl SqliteDb { } log::info!("Checking if the database needs upgrading."); - maybe_apply_migration(&db_path)?; Ok(SqliteDb { db_path }) } + /// If the database version is older than expected, migrate it to the current version. If + /// migrating from a database version 4 or earlier, all the wallet Bitcoin transactions must be + /// passed through the `bitcoin_txs` parameter otherwise the migration will fail. + pub fn maybe_apply_migrations( + &self, + bitcoin_txs: &[bitcoin::Transaction], + ) -> Result<(), SqliteDbError> { + maybe_apply_migration(&self.db_path, bitcoin_txs) + } + /// Get a new connection to the database. pub fn connection(&self) -> Result { let conn = rusqlite::Connection::open(&self.db_path)?; @@ -682,6 +696,83 @@ impl SqliteConn { .expect("Db must not fail") } + /// Retrieves all txids from the transactions table whether or not they are referenced by a coin. + pub fn db_list_saved_txids(&mut self) -> Vec { + db_query( + &mut self.conn, + "SELECT txid FROM transactions", + rusqlite::params![], + |row| { + let txid: Vec = row.get(0)?; + let txid: bitcoin::Txid = + encode::deserialize(&txid).expect("We only store valid txids"); + Ok(txid) + }, + ) + .expect("Db must not fail") + } + + /// Store transactions in database, ignoring any that already exist. + pub fn new_txs(&mut self, txs: &[bitcoin::Transaction]) { + db_exec(&mut self.conn, |db_tx| { + for tx in txs { + let txid = &tx.txid()[..].to_vec(); + let tx_ser = bitcoin::consensus::serialize(tx); + db_tx.execute( + "INSERT INTO transactions (txid, tx) VALUES (?1, ?2) \ + ON CONFLICT DO NOTHING", + rusqlite::params![txid, tx_ser,], + )?; + } + Ok(()) + }) + .expect("Database must be available") + } + + pub fn list_wallet_transactions( + &mut self, + txids: &[bitcoin::Txid], + ) -> Vec { + // The UNION will remove duplicates. + // We assume that a transaction's block info is the same in every coins row + // it appears in. + let query = format!( + "SELECT t.tx, c.blockheight, c.blocktime \ + FROM transactions t \ + INNER JOIN ( \ + SELECT txid, blockheight, blocktime \ + FROM coins \ + WHERE wallet_id = {WALLET_ID} \ + UNION \ + SELECT spend_txid, spend_block_height, spend_block_time \ + FROM coins \ + WHERE wallet_id = {WALLET_ID} \ + AND spend_txid IS NOT NULL \ + ) c ON t.txid = c.txid \ + WHERE t.txid in ({})", + txids + .iter() + .map(|txid| format!("x'{}'", FrontwardHexTxid(*txid))) + .collect::>() + .join(",") + ); + let w_txs: Vec = + db_query(&mut self.conn, &query, rusqlite::params![], |row| { + row.try_into() + }) + .expect("Db must not fail"); + debug_assert_eq!( + w_txs.len(), + w_txs + .iter() + .map(|t| t.transaction.txid()) + .collect::>() + .len(), + "database must not contain inconsistent block info for the same txid" + ); + w_txs + } + pub fn delete_spend(&mut self, txid: &bitcoin::Txid) { db_exec(&mut self.conn, |db_tx| { db_tx.execute( @@ -735,7 +826,7 @@ mod tests { str::FromStr, }; - use bitcoin::{bip32, hashes::Hash}; + use bitcoin::bip32; // The database schema used by the first versions of Liana (database version 0). Used to test // migrations starting from the first version. @@ -885,6 +976,91 @@ CREATE TABLE spend_transactions ( updated_at INTEGER ); +/* Labels applied on addresses (0), outpoints (1), txids (2) */ +CREATE TABLE labels ( + id INTEGER PRIMARY KEY NOT NULL, + wallet_id INTEGER NOT NULL, + item_kind INTEGER NOT NULL CHECK (item_kind IN (0,1,2)), + item TEXT UNIQUE NOT NULL, + value TEXT NOT NULL +); +"; + + const V4_SCHEMA: &str = " +CREATE TABLE version ( + version INTEGER NOT NULL +); + +/* About the Bitcoin network. */ +CREATE TABLE tip ( + network TEXT NOT NULL, + blockheight INTEGER, + blockhash BLOB +); + +/* This stores metadata about our wallet. We only support single wallet for + * now (and the foreseeable future). + * + * The 'timestamp' field is the creation date of the wallet. We guarantee to have seen all + * information related to our descriptor(s) that occured after this date. + * The optional 'rescan_timestamp' field is a the timestamp we need to rescan the chain + * for events related to our descriptor(s) from. + */ +CREATE TABLE wallets ( + id INTEGER PRIMARY KEY NOT NULL, + timestamp INTEGER NOT NULL, + main_descriptor TEXT NOT NULL, + deposit_derivation_index INTEGER NOT NULL, + change_derivation_index INTEGER NOT NULL, + rescan_timestamp INTEGER +); + +/* Our (U)TxOs. + * + * The 'spend_block_height' and 'spend_block.time' are only present if the spending + * transaction for this coin exists and was confirmed. + * + * The 'is_immature' field is for coinbase deposits that are not yet buried under 100 + * blocks. Note coinbase deposits can't technically be unconfirmed but we keep them + * as such until they become mature. + */ +CREATE TABLE coins ( + id INTEGER PRIMARY KEY NOT NULL, + wallet_id INTEGER NOT NULL, + blockheight INTEGER, + blocktime INTEGER, + txid BLOB NOT NULL, + vout INTEGER NOT NULL, + amount_sat INTEGER NOT NULL, + derivation_index INTEGER NOT NULL, + is_change BOOLEAN NOT NULL CHECK (is_change IN (0,1)), + spend_txid BLOB, + spend_block_height INTEGER, + spend_block_time INTEGER, + is_immature BOOLEAN NOT NULL CHECK (is_immature IN (0,1)), + UNIQUE (txid, vout), + FOREIGN KEY (wallet_id) REFERENCES wallets (id) + ON UPDATE RESTRICT + ON DELETE RESTRICT +); + +/* A mapping from descriptor address to derivation index. Necessary until + * we can get the derivation index from the parent descriptor from bitcoind. + */ +CREATE TABLE addresses ( + receive_address TEXT NOT NULL UNIQUE, + change_address TEXT NOT NULL UNIQUE, + derivation_index INTEGER NOT NULL UNIQUE +); + +/* Transactions we created that spend some of our coins. */ +CREATE TABLE spend_transactions ( + id INTEGER PRIMARY KEY NOT NULL, + psbt BLOB UNIQUE NOT NULL, + txid BLOB UNIQUE NOT NULL, + updated_at INTEGER +); + /* Labels applied on addresses (0), outpoints (1), txids (2) */ CREATE TABLE labels ( id INTEGER PRIMARY KEY NOT NULL, @@ -957,9 +1133,13 @@ CREATE TABLE labels ( // TODO: version check let db = SqliteDb::new(db_path.clone(), Some(options.clone()), &secp).unwrap(); + db.sanity_check(bitcoin::Network::Bitcoin, &options.main_descriptor) + .unwrap(); + let db = SqliteDb::new(db_path.clone(), None, &secp).unwrap(); db.sanity_check(bitcoin::Network::Bitcoin, &options.main_descriptor) .unwrap(); let db = SqliteDb::new(db_path, None, &secp).unwrap(); + db.maybe_apply_migrations(&[]).unwrap(); db.sanity_check(bitcoin::Network::Bitcoin, &options.main_descriptor) .unwrap(); @@ -1042,11 +1222,18 @@ CREATE TABLE labels ( // Necessarily empty at first. assert!(conn.coins(&[], &[]).is_empty()); + let txs: Vec<_> = (0..6) + .map(|i| bitcoin::Transaction { + version: bitcoin::transaction::Version::TWO, + lock_time: bitcoin::absolute::LockTime::from_height(i).unwrap(), + input: Vec::new(), + output: Vec::new(), + }) + .collect(); + conn.new_txs(&txs); + // Add one unconfirmed coin. - let outpoint_a = bitcoin::OutPoint::from_str( - "6f0dc85a369b44458eba3a1f0ea5b5935d563afb6994f70f5b0094e05be1676c:1", - ) - .unwrap(); + let outpoint_a = bitcoin::OutPoint::new(txs.first().unwrap().txid(), 1); let coin_a = Coin { outpoint: outpoint_a, is_immature: false, @@ -1092,10 +1279,7 @@ CREATE TABLE labels ( .is_empty()); // Add a second coin. - let outpoint_b = bitcoin::OutPoint::from_str( - "61db3e276b095e5b05f1849dd6bfffb4e7e5ec1c4a4210099b98fce01571936f:12", - ) - .unwrap(); + let outpoint_b = bitcoin::OutPoint::new(txs.get(1).unwrap().txid(), 12); let coin_b = Coin { outpoint: outpoint_b, is_immature: false, @@ -1173,10 +1357,7 @@ CREATE TABLE labels ( && c[1].outpoint == coin_b.outpoint)); // Now if we spend one, it'll be marked as such. - conn.spend_coins(&[( - coin_a.outpoint, - bitcoin::Txid::from_slice(&[0; 32][..]).unwrap(), - )]); + conn.spend_coins(&[(coin_a.outpoint, txs.get(2).unwrap().txid())]); assert!([ conn.coins(&[CoinStatus::Spending], &[]), conn.coins(&[CoinStatus::Spending], &[outpoint_a]), @@ -1199,7 +1380,7 @@ CREATE TABLE labels ( // Now we confirm the spend. conn.confirm_spend(&[( coin_a.outpoint, - bitcoin::Txid::from_slice(&[0; 32][..]).unwrap(), + txs.get(2).unwrap().txid(), 128_097, 3_000_000, )]); @@ -1229,10 +1410,7 @@ CREATE TABLE labels ( && c[1].outpoint == coin_b.outpoint)); // Add a third and fourth coin. - let outpoint_c = bitcoin::OutPoint::from_str( - "61db3e276b095e5b05f1849dd6bfffb4e7e5ec1c4a4210099b98fce01571937a:42", - ) - .unwrap(); + let outpoint_c = bitcoin::OutPoint::new(txs.get(3).unwrap().txid(), 42); let coin_c = Coin { outpoint: outpoint_c, is_immature: false, @@ -1243,10 +1421,7 @@ CREATE TABLE labels ( spend_txid: None, spend_block: None, }; - let outpoint_d = bitcoin::OutPoint::from_str( - "61db3e276b095e5b05f1849dd6bfffb4e7e5ec1c4a4210099b98fce01571937a:43", - ) - .unwrap(); + let outpoint_d = bitcoin::OutPoint::new(txs.get(4).unwrap().txid(), 43); let coin_d = Coin { outpoint: outpoint_d, is_immature: false, @@ -1287,10 +1462,7 @@ CREATE TABLE labels ( && coin[1].outpoint == coin_c.outpoint)); // Now spend second coin, even though it is still unconfirmed. - conn.spend_coins(&[( - coin_b.outpoint, - bitcoin::Txid::from_slice(&[1; 32][..]).unwrap(), - )]); + conn.spend_coins(&[(coin_b.outpoint, txs.get(5).unwrap().txid())]); // The coin shows as spending. assert!([ conn.coins(&[CoinStatus::Spending], &[]), @@ -1348,15 +1520,22 @@ CREATE TABLE labels ( { let mut conn = db.connection().unwrap(); + let txs: Vec<_> = (0..4) + .map(|i| bitcoin::Transaction { + version: bitcoin::transaction::Version::TWO, + lock_time: bitcoin::absolute::LockTime::from_height(i).unwrap(), + input: Vec::new(), + output: Vec::new(), + }) + .collect(); + conn.new_txs(&txs); + // Necessarily empty at first. assert!(conn.coins(&[], &[]).is_empty()); // Add one, we'll get it. let coin_a = Coin { - outpoint: bitcoin::OutPoint::from_str( - "6f0dc85a369b44458eba3a1f0ea5b5935d563afb6994f70f5b0094e05be1676c:1", - ) - .unwrap(), + outpoint: bitcoin::OutPoint::new(txs.first().unwrap().txid(), 1), is_immature: false, block_info: None, amount: bitcoin::Amount::from_sat(98765), @@ -1398,10 +1577,7 @@ CREATE TABLE labels ( // Add a second one (this one is change), we'll get both. let coin_b = Coin { - outpoint: bitcoin::OutPoint::from_str( - "61db3e276b095e5b05f1849dd6bfffb4e7e5ec1c4a4210099b98fce01571936f:12", - ) - .unwrap(), + outpoint: bitcoin::OutPoint::new(txs.get(1).unwrap().txid(), 12), is_immature: false, block_info: None, amount: bitcoin::Amount::from_sat(1111), @@ -1453,10 +1629,7 @@ CREATE TABLE labels ( assert!(coins[1].block_info.is_none()); // Now if we spend one, it'll be marked as such. - conn.spend_coins(&[( - coin_a.outpoint, - bitcoin::Txid::from_slice(&[0; 32][..]).unwrap(), - )]); + conn.spend_coins(&[(coin_a.outpoint, txs.get(2).unwrap().txid())]); let coin = conn .coins(&[], &[coin_a.outpoint]) .into_iter() @@ -1474,10 +1647,7 @@ CREATE TABLE labels ( assert!(coin.spend_txid.is_none()); // Spend it back. We will see it as 'spending' - conn.spend_coins(&[( - coin_a.outpoint, - bitcoin::Txid::from_slice(&[0; 32][..]).unwrap(), - )]); + conn.spend_coins(&[(coin_a.outpoint, txs.get(2).unwrap().txid())]); let outpoints: HashSet = conn .list_spending_coins() .into_iter() @@ -1498,12 +1668,7 @@ CREATE TABLE labels ( // Now if we confirm the spend. let height = 128_097; let time = 3_000_000; - conn.confirm_spend(&[( - coin_a.outpoint, - bitcoin::Txid::from_slice(&[0; 32][..]).unwrap(), - height, - time, - )]); + conn.confirm_spend(&[(coin_a.outpoint, txs.get(2).unwrap().txid(), height, time)]); // the coin is not in a spending state. let outpoints: HashSet = conn .list_spending_coins() @@ -1535,10 +1700,7 @@ CREATE TABLE labels ( // Add an immature coin. As all coins it's first registered as unconfirmed (even though // it's not). let coin_imma = Coin { - outpoint: bitcoin::OutPoint::from_str( - "61db3e276b095e5b05f1849dd6bfffb4e7e5ec1c4a4210099b98fce01571937a:42", - ) - .unwrap(), + outpoint: bitcoin::OutPoint::new(txs.get(3).unwrap().txid(), 42), is_immature: true, block_info: None, amount: bitcoin::Amount::from_sat(424242), @@ -1691,6 +1853,15 @@ CREATE TABLE labels ( }; conn.update_tip(&old_tip); + let txs: Vec<_> = (0..7) + .map(|i| bitcoin::Transaction { + version: bitcoin::transaction::Version::TWO, + lock_time: bitcoin::absolute::LockTime::from_height(i).unwrap(), + input: Vec::new(), + output: Vec::new(), + }) + .collect(); + conn.new_txs(&txs); // 5 coins: // - One unconfirmed // - One confirmed before the rollback height @@ -1700,10 +1871,7 @@ CREATE TABLE labels ( // TODO: immature deposits let coins = [ Coin { - outpoint: bitcoin::OutPoint::from_str( - "6f0dc85a369b44458eba3a1f0ea5b5935d563afb6994f70f5b0094e05be1676c:1", - ) - .unwrap(), + outpoint: bitcoin::OutPoint::new(txs.first().unwrap().txid(), 1), is_immature: false, block_info: None, amount: bitcoin::Amount::from_sat(98765), @@ -1713,10 +1881,7 @@ CREATE TABLE labels ( spend_block: None, }, Coin { - outpoint: bitcoin::OutPoint::from_str( - "c449539458c60bee6c0d8905ba1dadb20b9187b82045d306a408b894cea492b0:2", - ) - .unwrap(), + outpoint: bitcoin::OutPoint::new(txs.get(1).unwrap().txid(), 2), is_immature: false, block_info: Some(BlockInfo { height: 101_095, @@ -1729,10 +1894,7 @@ CREATE TABLE labels ( spend_block: None, }, Coin { - outpoint: bitcoin::OutPoint::from_str( - "f0801fd9ca8bca0624c230ab422b2e2c4c8dc995e4e1dbc6412510959cce1e4f:3", - ) - .unwrap(), + outpoint: bitcoin::OutPoint::new(txs.get(2).unwrap().txid(), 3), is_immature: false, block_info: Some(BlockInfo { height: 101_099, @@ -1741,22 +1903,14 @@ CREATE TABLE labels ( amount: bitcoin::Amount::from_sat(98765), derivation_index: bip32::ChildNumber::from_normal_idx(1000).unwrap(), is_change: false, - spend_txid: Some( - bitcoin::Txid::from_str( - "0c62a990d20d54429e70859292e82374ba6b1b951a3ab60f26bb65fee5724ff7", - ) - .unwrap(), - ), + spend_txid: Some(txs.get(3).unwrap().txid()), spend_block: Some(BlockInfo { height: 101_199, time: 1_231_678, }), }, Coin { - outpoint: bitcoin::OutPoint::from_str( - "19f56e65069f0a7a3bfb00c6a7085cc0669e03e91befeca1ee9891c9e737b2fb:4", - ) - .unwrap(), + outpoint: bitcoin::OutPoint::new(txs.get(4).unwrap().txid(), 4), is_immature: false, block_info: Some(BlockInfo { height: 101_100, @@ -1769,10 +1923,7 @@ CREATE TABLE labels ( spend_block: None, }, Coin { - outpoint: bitcoin::OutPoint::from_str( - "ed6c8f1af9325f84de521e785e7ddfd33dc28c9ada4d687dcd3850100bde54e9:5", - ) - .unwrap(), + outpoint: bitcoin::OutPoint::new(txs.get(5).unwrap().txid(), 5), is_immature: false, block_info: Some(BlockInfo { height: 101_102, @@ -1781,12 +1932,7 @@ CREATE TABLE labels ( amount: bitcoin::Amount::from_sat(98765), derivation_index: bip32::ChildNumber::from_normal_idx(100000).unwrap(), is_change: false, - spend_txid: Some( - bitcoin::Txid::from_str( - "7477017f992cdc7ba08acafb77cb3b5bc0f42ac340d3e1e1da0785bdda20d5f6", - ) - .unwrap(), - ), + spend_txid: Some(txs.get(6).unwrap().txid()), spend_block: Some(BlockInfo { height: 101_105, time: 1_201_678, @@ -1916,12 +2062,19 @@ CREATE TABLE labels ( { let mut conn = db.connection().unwrap(); + let txs: Vec<_> = (0..7) + .map(|i| bitcoin::Transaction { + version: bitcoin::transaction::Version::TWO, + lock_time: bitcoin::absolute::LockTime::from_height(i).unwrap(), + input: Vec::new(), + output: Vec::new(), + }) + .collect(); + conn.new_txs(&txs); + let coins = [ Coin { - outpoint: bitcoin::OutPoint::from_str( - "6f0dc85a369b44458eba3a1f0ea5b5935d563afb6994f70f5b0094e05be1676c:1", - ) - .unwrap(), + outpoint: bitcoin::OutPoint::new(txs.first().unwrap().txid(), 1), is_immature: false, block_info: None, amount: bitcoin::Amount::from_sat(98765), @@ -1931,10 +2084,7 @@ CREATE TABLE labels ( spend_block: None, }, Coin { - outpoint: bitcoin::OutPoint::from_str( - "c449539458c60bee6c0d8905ba1dadb20b9187b82045d306a408b894cea492b0:2", - ) - .unwrap(), + outpoint: bitcoin::OutPoint::new(txs.get(1).unwrap().txid(), 2), is_immature: false, block_info: Some(BlockInfo { height: 101_095, @@ -1947,10 +2097,7 @@ CREATE TABLE labels ( spend_block: None, }, Coin { - outpoint: bitcoin::OutPoint::from_str( - "f0801fd9ca8bca0624c230ab422b2e2c4c8dc995e4e1dbc6412510959cce1e4f:3", - ) - .unwrap(), + outpoint: bitcoin::OutPoint::new(txs.get(2).unwrap().txid(), 3), is_immature: false, block_info: Some(BlockInfo { height: 101_099, @@ -1959,22 +2106,14 @@ CREATE TABLE labels ( amount: bitcoin::Amount::from_sat(98765), derivation_index: bip32::ChildNumber::from_normal_idx(1000).unwrap(), is_change: false, - spend_txid: Some( - bitcoin::Txid::from_str( - "0c62a990d20d54429e70859292e82374ba6b1b951a3ab60f26bb65fee5724ff7", - ) - .unwrap(), - ), + spend_txid: Some(txs.get(3).unwrap().txid()), spend_block: Some(BlockInfo { height: 101_199, time: 1_123_000, }), }, Coin { - outpoint: bitcoin::OutPoint::from_str( - "19f56e65069f0a7a3bfb00c6a7085cc0669e03e91befeca1ee9891c9e737b2fb:4", - ) - .unwrap(), + outpoint: bitcoin::OutPoint::new(txs.get(4).unwrap().txid(), 4), is_immature: true, block_info: Some(BlockInfo { height: 101_100, @@ -1987,10 +2126,7 @@ CREATE TABLE labels ( spend_block: None, }, Coin { - outpoint: bitcoin::OutPoint::from_str( - "ed6c8f1af9325f84de521e785e7ddfd33dc28c9ada4d687dcd3850100bde54e9:5", - ) - .unwrap(), + outpoint: bitcoin::OutPoint::new(txs.get(5).unwrap().txid(), 5), is_immature: false, block_info: Some(BlockInfo { height: 101_102, @@ -1999,12 +2135,7 @@ CREATE TABLE labels ( amount: bitcoin::Amount::from_sat(98765), derivation_index: bip32::ChildNumber::from_normal_idx(100000).unwrap(), is_change: false, - spend_txid: Some( - bitcoin::Txid::from_str( - "7477017f992cdc7ba08acafb77cb3b5bc0f42ac340d3e1e1da0785bdda20d5f6", - ) - .unwrap(), - ), + spend_txid: Some(txs.get(6).unwrap().txid()), spend_block: Some(BlockInfo { height: 101_105, time: 1_126_000, @@ -2030,54 +2161,234 @@ CREATE TABLE labels ( ); let db_txids = conn.db_list_txids(1_123_000, 1_127_000, 10); - assert_eq!( - &db_txids[..], - &[ - bitcoin::Txid::from_str( - "7477017f992cdc7ba08acafb77cb3b5bc0f42ac340d3e1e1da0785bdda20d5f6" - ) - .unwrap(), - bitcoin::Txid::from_str( - "ed6c8f1af9325f84de521e785e7ddfd33dc28c9ada4d687dcd3850100bde54e9" - ) - .unwrap(), - bitcoin::Txid::from_str( - "19f56e65069f0a7a3bfb00c6a7085cc0669e03e91befeca1ee9891c9e737b2fb" - ) - .unwrap(), - bitcoin::Txid::from_str( - "0c62a990d20d54429e70859292e82374ba6b1b951a3ab60f26bb65fee5724ff7" - ) - .unwrap() - ] - ); + // Ordered by desc block time. + let expected_txids = [6, 5, 4, 3].map(|i| txs.get(i).unwrap().txid()); + assert_eq!(&db_txids[..], &expected_txids,); let db_txids = conn.db_list_txids(1_123_000, 1_127_000, 2); - assert_eq!( - &db_txids[..], - &[ - bitcoin::Txid::from_str( - "7477017f992cdc7ba08acafb77cb3b5bc0f42ac340d3e1e1da0785bdda20d5f6" - ) - .unwrap(), - bitcoin::Txid::from_str( - "ed6c8f1af9325f84de521e785e7ddfd33dc28c9ada4d687dcd3850100bde54e9" - ) - .unwrap(), - ] - ); + // Ordered by desc block time. + let expected_txids = [6, 5].map(|i| txs.get(i).unwrap().txid()); + assert_eq!(&db_txids[..], &expected_txids,); } fs::remove_dir_all(tmp_dir).unwrap(); } #[test] - fn v0_to_v2_migration() { + fn sqlite_list_saved_txids() { + let (tmp_dir, _, _, db) = dummy_db(); + + { + let mut conn = db.connection().unwrap(); + + let txs: Vec<_> = (0..7) + .map(|i| bitcoin::Transaction { + version: bitcoin::transaction::Version::TWO, + lock_time: bitcoin::absolute::LockTime::from_height(i).unwrap(), + input: Vec::new(), + output: Vec::new(), + }) + .collect(); + conn.new_txs(&txs); + + let mut db_txids = conn.db_list_saved_txids(); + db_txids.sort(); + let mut expected_txids: Vec<_> = txs.iter().map(|tx| tx.txid()).collect(); + expected_txids.sort(); + assert_eq!(&db_txids[..], &expected_txids,); + } + + fs::remove_dir_all(tmp_dir).unwrap(); + } + + #[test] + fn sqlite_list_wallet_transactions() { + let (tmp_dir, _, _, db) = dummy_db(); + + { + let mut conn = db.connection().unwrap(); + + // The following is based on the `v4_to_v5_migration` test. + let mut bitcoin_txs: Vec<_> = (0..100) + .map(|i| bitcoin::Transaction { + version: bitcoin::transaction::Version::TWO, + lock_time: bitcoin::absolute::LockTime::from_height(i).unwrap(), + input: Vec::new(), + output: Vec::new(), + }) + .collect(); + let spend_txs: Vec<_> = (0..10) + .map(|i| { + ( + bitcoin::Transaction { + version: bitcoin::transaction::Version::TWO, + lock_time: bitcoin::absolute::LockTime::from_height(1_234 + i).unwrap(), + input: Vec::new(), + output: Vec::new(), + }, + if i % 2 == 0 { + Some(BlockInfo { + height: (i % 5) as i32 * 2_000, + time: 1722488619 + (i % 5) * 84_999, + }) + } else { + None + }, + ) + }) + .collect(); + let coins: Vec = bitcoin_txs + .iter() + .chain(bitcoin_txs.iter()) // We do this to have coins which originate from the same tx. + .enumerate() + .map(|(i, tx)| Coin { + outpoint: bitcoin::OutPoint { + txid: tx.txid(), + vout: i as u32, + }, + is_immature: (i % 10) == 0, + amount: bitcoin::Amount::from_sat(i as u64 * 3473), + derivation_index: bip32::ChildNumber::from_normal_idx(i as u32 * 100).unwrap(), + is_change: (i % 4) == 0, + block_info: if i & 2 == 0 { + Some(BlockInfo { + height: (i % 100) as i32 * 1_000, + time: 1722408619 + (i % 100) as u32 * 42_000, + }) + } else { + None + }, + spend_txid: if i % 20 == 0 { + Some(spend_txs[i / 20].0.txid()) + } else { + None + }, + spend_block: if i % 20 == 0 { + spend_txs[i / 20].1 + } else { + None + }, + }) + .collect(); + + bitcoin_txs.extend(spend_txs.into_iter().map(|(tx, _)| tx)); + conn.new_txs(&bitcoin_txs); + + // Insert all these coins into database. + conn.new_unspent_coins(&coins); + + // Confirm those which are supposed to be. + let confirmed_coins: Vec<_> = coins + .iter() + .filter_map(|coin| { + coin.block_info + .map(|blk| (coin.outpoint, blk.height, blk.time)) + }) + .collect(); + conn.confirm_coins(&confirmed_coins); + + // Spend those which are supposed to be. + let spent_coins: Vec<_> = coins + .iter() + .filter_map(|coin| coin.spend_txid.map(|txid| (coin.outpoint, txid))) + .collect(); + conn.spend_coins(&spent_coins); + + // Mark the spend as confirmed for those which are supposed to be. + let confirmed_spent_coins: Vec<_> = coins + .iter() + .filter_map(|coin| { + coin.spend_block.map(|blk| { + ( + coin.outpoint, + coin.spend_txid.expect("always set when spend block is"), + blk.height, + blk.time, + ) + }) + }) + .collect(); + conn.confirm_spend(&confirmed_spent_coins); + + // For easy lookup, map each tx to its txid. + let bitcoin_txs: HashMap<_, _> = + bitcoin_txs.into_iter().map(|tx| (tx.txid(), tx)).collect(); + + let block_info_from_coins: HashSet<_> = coins + .iter() + .map(|c| (c.outpoint.txid, c.block_info)) + .chain( + coins + .iter() + .filter_map(|c| c.spend_txid.map(|txid| (txid, c.spend_block))), + ) + .collect(); + + // Make sure each txid only has one block info. + assert_eq!(bitcoin_txs.len(), block_info_from_coins.len()); + + // For each txid, determine its wallet transaction based on the coins defined above. + let wallet_txs_from_coins: HashMap<_, _> = block_info_from_coins + .into_iter() + .map(|(txid, block_info)| { + let tx = bitcoin_txs.get(&txid).unwrap(); + + ( + txid, + DbWalletTransaction { + transaction: tx.clone(), + block_info: block_info.map(|info| DbBlockInfo { + height: info.height, + time: info.time, + }), + }, + ) + }) + .collect(); + + let all_txids: Vec<_> = bitcoin_txs.keys().cloned().collect(); + + for indices in [ + (0..all_txids.len()).collect(), + (0..all_txids.len() / 2).collect(), + (all_txids.len() / 5..all_txids.len() / 2).collect(), + vec![3, 4, 5, 6], + vec![4, 5], + vec![1, 3, 5], + vec![1], + vec![1, 1, 3, 4, 3], // we can pass duplicate txids + vec![], // can pass empty slice + ] { + let txids: Vec<_> = indices + .iter() + .map(|i| *all_txids.get(*i).unwrap()) + .collect(); + + // Make sure we have the expected number of txids. + assert_eq!(txids.len(), indices.len()); + + let mut db_txs = conn.list_wallet_transactions(&txids); + db_txs.sort_by(|a, b| a.transaction.txid().cmp(&b.transaction.txid())); + let mut expected_txs: Vec<_> = txids + .iter() + .collect::>() // remove duplicates + .into_iter() + .map(|txid| wallet_txs_from_coins.get(txid).unwrap().clone()) + .collect(); + expected_txs.sort_by(|a, b| a.transaction.txid().cmp(&b.transaction.txid())); + assert_eq!(&db_txs[..], &expected_txs[..],); + } + } + + fs::remove_dir_all(tmp_dir).unwrap(); + } + + #[test] + fn v0_to_v5_migration() { let secp = secp256k1::Secp256k1::verification_only(); // Create a database with version 0, using the old schema. let tmp_dir = tmp_dir(); - eprintln!("{}", tmp_dir.as_path().to_string_lossy()); fs::create_dir_all(&tmp_dir).unwrap(); let db_path: path::PathBuf = [tmp_dir.as_path(), path::Path::new("lianad_v0.sqlite3")] .iter() @@ -2092,6 +2403,14 @@ CREATE TABLE labels ( let first_psbt = psbt_from_str("cHNidP8BAIkCAAAAAWi3OFgkj1CqCDT3Swm8kbxZS9lxz4L3i4W2v9KGC7nqAQAAAAD9////AkANAwAAAAAAIgAg27lNc1rog+dOq80ohRuds4Hgg/RcpxVun2XwgpuLSrFYMwwAAAAAACIAIDyWveqaElWmFGkTbFojg1zXWHODtiipSNjfgi2DqBy9AAAAAAABAOoCAAAAAAEBsRWl70USoAFFozxc86pC7Dovttdg4kvja//3WMEJskEBAAAAAP7///8CWKmCIk4GAAAWABRKBWYWkCNS46jgF0r69Ehdnq+7T0BCDwAAAAAAIgAgTt5fs+CiB+FRzNC8lHcgWLH205sNjz1pT59ghXlG5tQCRzBEAiBXK9MF8z3bX/VnY2aefgBBmiAHPL4tyDbUOe7+KpYA4AIgL5kU0DFG8szKd+szRzz/OTUWJ0tZqij41h2eU9rSe1IBIQNBB1hy+jKsg1TihMT0dXw7etpu9TkO3NuvhBDFJlBj1cP2AQABAStAQg8AAAAAACIAIE7eX7PgogfhUczQvJR3IFix9tObDY89aU+fYIV5RubUIgICSKJsNs0zFJN58yd2aYQ+C3vhMbi0x7k0FV3wBhR4THlIMEUCIQCPWWWOhs2lThxOq/G8X2fYBRvM9MXSm7qPH+dRVYQZEwIgfut2vx3RvwZWcgEj4ohQJD5lNJlwOkA4PAiN1fjx6dABIgID3mvj1zerZKohOVhKCiskYk+3qrCum6PIwDhQ16ePACpHMEQCICZNR+0/1hPkrDQwPFmg5VjUHkh6aK9cXUu3kPbM8hirAiAyE/5NUXKfmFKij30isuyysJbq8HrURjivd+S9vdRGKQEBBZNSIQJIomw2zTMUk3nzJ3ZphD4Le+ExuLTHuTQVXfAGFHhMeSEC9OfCXl+sJOrxUFLBuMV4ZUlJYjuzNGZSld5ioY14y8FSrnNkUSED3mvj1zerZKohOVhKCiskYk+3qrCum6PIwDhQ16ePACohA+ECH+HlR+8Sf3pumaXH3IwSsoqSLCH7H1THiBP93z3ZUq9SsmgiBgJIomw2zTMUk3nzJ3ZphD4Le+ExuLTHuTQVXfAGFHhMeRxjat8/MAAAgAEAAIAAAACAAgAAgAAAAAABAAAAIgYC9OfCXl+sJOrxUFLBuMV4ZUlJYjuzNGZSld5ioY14y8Ec/9Y8jTAAAIABAACAAAAAgAIAAIAAAAAAAQAAACIGA95r49c3q2SqITlYSgorJGJPt6qwrpujyMA4UNenjwAqHGNq3z8wAACAAQAAgAEAAIACAACAAAAAAAEAAAAiBgPhAh/h5UfvEn96bpmlx9yMErKKkiwh+x9Ux4gT/d892Rz/1jyNMAAAgAEAAIABAACAAgAAgAAAAAABAAAAACICAlBQ7gGocg7eF3sXrCio+zusAC9+xfoyIV95AeR69DWvHGNq3z8wAACAAQAAgAEAAIACAACAAAAAAAMAAAAiAgMvVy984eg8Kgvj058PBHetFayWbRGb7L0DMnS9KHSJzBxjat8/MAAAgAEAAIAAAACAAgAAgAAAAAADAAAAIgIDSRIG1dn6njdjsDXenHa2lUvQHWGPLKBVrSzbQOhiIxgc/9Y8jTAAAIABAACAAAAAgAIAAIAAAAAAAwAAACICA0/epE59sVEj7Et0I4R9qJQNuX23RNvDZKCRL7eUps9FHP/WPI0wAACAAQAAgAEAAIACAACAAAAAAAMAAAAAIgICgldCOK6iHscv//2NipgaMABLV5TICU/zlP7HlQmlg08cY2rfPzAAAIABAACAAQAAgAIAAIABAAAAAQAAACICApb0p9rfpJshB3J186PGWrvzQdixcwQZWmebOUMdkquZHP/WPI0wAACAAQAAgAAAAIACAACAAQAAAAEAAAAiAgLY5q+unoDxC/HI5BaNiPq12ei1REZIcUAN304JfKXUwxz/1jyNMAAAgAEAAIABAACAAgAAgAEAAAABAAAAIgIDg6cUVCJB79cMcofiURHojxFARWyS4YEhJNRixuOZZRgcY2rfPzAAAIABAACAAAAAgAIAAIABAAAAAQAAAAA="); let second_psbt = psbt_from_str("cHNidP8BAP0fAQIAAAAGAGo6V8K5MtKcQ8vRFedf5oJiOREiH4JJcEniyRv2800BAAAAAP3///9e3dVLjWKPAGwDeuUOmKFzOYEP5Ipu4LWdOPA+lITrRgAAAAAA/f///7cl9oeu9ssBXKnkWMCUnlgZPXhb+qQO2+OPeLEsbdGkAQAAAAD9////idkxRErbs34vsHUZ7QCYaiVaAFDV9gxNvvtwQLozwHsAAAAAAP3///9EakyJhd2PjwYh1I7zT2cmcTFI5g1nBd3srLeL7wKEewIAAAAA/f///7BcaP77nMaA2NjT/hyI6zueB/2jU/jK4oxmSqMaFkAzAQAAAAD9////AUAfAAAAAAAAFgAUqo7zdMr638p2kC3bXPYcYLv9nYUAAAAAAAEA/X4BAgAAAAABApEoe5xCmSi8hNTtIFwsy46aj3hlcLrtFrug39v5wy+EAQAAAGpHMEQCIDeI8JTWCTyX6opCCJBhWc4FytH8g6fxDaH+Wa/QqUoMAiAgbITpz8TBhwxhv/W4xEXzehZpOjOTjKnPw36GIy6SHAEhA6QnYCHUbU045FVh6ZwRwYTVineqRrB9tbqagxjaaBKh/v///+v1seDE9gGsZiWwewQs3TKuh0KSBIHiEtG8ABbz2DpAAQAAAAD+////Aqhaex4AAAAAFgAUkcVOEjVMct0jyCzhZN6zBT+lvTQvIAAAAAAAACIAIKKDUd/GWjAnwU99llS9TAK2dK80/nSRNLjmrhj0odUEAAJHMEQCICSn+boh4ItAa3/b4gRUpdfblKdcWtMLKZrgSEFFrC+zAiBtXCx/Dq0NutLSu1qmzFF1lpwSCB3w3MAxp5W90z7b/QEhA51S2ERUi0bg+l+bnJMJeAfDknaetMTagfQR9+AOrVKlxdMkAAEBKy8gAAAAAAAAIgAgooNR38ZaMCfBT32WVL1MArZ0rzT+dJE0uOauGPSh1QQiAgN+zbSfdr8oJBtlKomnQTHynF2b/UhovAwf0eS8awRSqUgwRQIhAJhm6xQvxt2LY+eNZqjhsgMOAxD0OPYty6nf9WaQZtgkAiBf/AXkeyq6ALknO9TZwY6ZRa0evY+DQ3j3XaqiBiAMfgEBBUEhA37NtJ92vygkG2UqiadBMfKcXZv9SGi8DB/R5LxrBFKprHNkdqkUxttmGj2sqzzaxSaacJTnJPDCbY6IrVqyaCIGAv9qeBDEB+5kvM/sZ8jQ7QApfZcDrqtq5OAe2gQ1V+pmDIpk8qkAAAAA0AAAACIGA37NtJ92vygkG2UqiadBMfKcXZv9SGi8DB/R5LxrBFKpDPWswv0AAAAA0AAAAAABAOoCAAAAAAEB0OPoVJs9ihvnAwjO16k/wGJuEus1IEE1Yo2KBjC2NSEAAAAAAP7///8C6AMAAAAAAAAiACBfeUS9jQv6O1a96Aw/mPV6gHxHl3mfj+f0frfAs2sMpP1QGgAAAAAAFgAUDS4UAIpdm1RlFYmg0OoCxW0yBT4CRzBEAiAPvbNlnhiUxLNshxN83AuK/lGWwlpXOvmcqoxsMLzIKwIgWwATJuYPf9buLe9z5SnXVnPVL0q6UZaWE5mjCvEl1RUBIQI54LFZmq9Lw0pxKpEGeqI74NnIfQmLMDcv5ySplUS1/wDMJAABASvoAwAAAAAAACIAIF95RL2NC/o7Vr3oDD+Y9XqAfEeXeZ+P5/R+t8CzawykIgICYn4eZbb6KGoxB1PEv/XPiujZFDhfoi/rJPtfHPVML2lHMEQCIDOHEqKdBozXIPLVgtBj3eWC1MeIxcKYDADe4zw0DbcMAiAq4+dbkTNCAjyCxJi0TKz5DWrPulxrqOdjMRHWngXHsQEBBUEhAmJ+HmW2+ihqMQdTxL/1z4ro2RQ4X6Iv6yT7Xxz1TC9prHNkdqkUzc/gCLoe6rQw63CGXhIR3YRz1qCIrVqyaCIGAmJ+HmW2+ihqMQdTxL/1z4ro2RQ4X6Iv6yT7Xxz1TC9pDPWswv0AAAAAqgAAACIGA8JCTIzdSoTJhiKN1pn+NnlkyuKOndiTgH2NIX+yNsYqDIpk8qkAAAAAqgAAAAABAOoCAAAAAAEBRGpMiYXdj48GIdSO809nJnExSOYNZwXd7Ky3i+8ChHsAAAAAAP7///8COMMQAAAAAAAWABQ5rnyuG5T8iuhqfaGAmpzlybo3t+gDAAAAAAAAIgAg7Kz3CX1RBjIvbK9LBYztmi7F1XIxQpX6mtCUkflvvl8CRzBEAiBaYx4sOHckEZwDnSrbb1ivc6seX4Puasm1PBGnBWgSTQIgCeUiXvd90ajI3F4/BHifLUI4fVIgVQFCqLTbbeXQD5oBIQOmGm+gTRx1slzF+wn8NhZoR1xfSYgoKX6bpRSVRjLcEXrOJAABASvoAwAAAAAAACIAIOys9wl9UQYyL2yvSwWM7ZouxdVyMUKV+prQlJH5b75fIgID0X2UJhC5+2jgJqUrihxZxDZHK7jgPFlrUYzoSHQTmP9HMEQCIEM4K8lVACvE2oSMZHDJiOeD81qsYgAvgpRgcSYgKc3AAiAQjdDr2COBea69W+2iVbnODuH3QwacgShW3dS4yeggJAEBBUEhA9F9lCYQufto4CalK4ocWcQ2Ryu44DxZa1GM6Eh0E5j/rHNkdqkU0DTexcgOQQ+BFjgS031OTxcWiH2IrVqyaCIGA9F9lCYQufto4CalK4ocWcQ2Ryu44DxZa1GM6Eh0E5j/DPWswv0AAAAAvwAAACIGA/xg4Uvem3JHVPpyTLP5JWiUH/yk3Y/uUI6JkZasCmHhDIpk8qkAAAAAvwAAAAABAOoCAAAAAAEBmG+mPq0O6QSWEMctsMjvv5LzWHGoT8wsA9Oa05kxIxsBAAAAAP7///8C6AMAAAAAAAAiACDUvIILFr0OxybADV3fB7ms7+ufnFZgicHR0nbI+LFCw1UoGwAAAAAAFgAUC+1ZjCC1lmMcvJ/4JkevqoZF4igCRzBEAiA3d8o96CNgNWHUkaINWHTvAUinjUINvXq0KBeWcsSWuwIgKfzRNWFR2LDbnB/fMBsBY/ylVXcSYwLs8YC+kmko1zIBIQOpEfsLv0htuertA1sgzCwGvHB0vE4zFO69wWEoHClKmAfMJAABASvoAwAAAAAAACIAINS8ggsWvQ7HJsANXd8Huazv65+cVmCJwdHSdsj4sULDIgID96jZc0sCi0IIXf2CpfE7tY+9LRmMsOdSTTHelFxfCwJHMEQCIHlaiMMznx8Cag8Y3X2gXi9Qtg0ZuyHEC6DsOzipSGOKAiAV2eC+S3Mbq6ig5QtRvTBsq5M3hCBdEJQlOrLVhWWt6AEBBUEhA/eo2XNLAotCCF39gqXxO7WPvS0ZjLDnUk0x3pRcXwsCrHNkdqkUyJ+Cbx7vYVY665yjJnMNODyYrAuIrVqyaCIGAt8UyDXk+mW3Y6IZNIBuDJHkdOaZi/UEShkN5L3GiHR5DIpk8qkAAAAAuAAAACIGA/eo2XNLAotCCF39gqXxO7WPvS0ZjLDnUk0x3pRcXwsCDPWswv0AAAAAuAAAAAABAP0JAQIAAAAAAQG7Zoy4I3J9x+OybAlIhxVKcYRuPFrkDFJfxMiC3kIqIAEAAAAA/v///wO5xxAAAAAAABYAFHgBzs9wJNVk6YwR81IMKmckTmC56AMAAAAAAAAWABTQ/LmJix5JoHBOr8LcgEChXHdLROgDAAAAAAAAIgAg7Kz3CX1RBjIvbK9LBYztmi7F1XIxQpX6mtCUkflvvl8CRzBEAiA+sIKnWVE3SmngjUgJdu1K2teW6eqeolfGe0d11b+irAIgL20zSabXaFRNM8dqVlcFsfNJ0exukzvxEOKl/OcF8VsBIQJrUspHq45AMSwbm24//2a9JM8XHFWbOKpyV+gNCtW71nrOJAABASvoAwAAAAAAACIAIOys9wl9UQYyL2yvSwWM7ZouxdVyMUKV+prQlJH5b75fIgID0X2UJhC5+2jgJqUrihxZxDZHK7jgPFlrUYzoSHQTmP9IMEUCIQCmDhJ9fyhlQwPruoOUemDuldtRu3ZkiTM3DA0OhkguSQIgYerNaYdP43DcqI5tnnL3n4jEeMHFCs+TBkOd6hDnqAkBAQVBIQPRfZQmELn7aOAmpSuKHFnENkcruOA8WWtRjOhIdBOY/6xzZHapFNA03sXIDkEPgRY4EtN9Tk8XFoh9iK1asmgiBgPRfZQmELn7aOAmpSuKHFnENkcruOA8WWtRjOhIdBOY/wz1rML9AAAAAL8AAAAiBgP8YOFL3ptyR1T6ckyz+SVolB/8pN2P7lCOiZGWrAph4QyKZPKpAAAAAL8AAAAAAQDqAgAAAAABAT6/vc6qBRzhQyjVtkC25NS2BvGyl2XjjEsw3e8vAesjAAAAAAD+////AgPBAO4HAAAAFgAUEwiWd/qI1ergMUw0F1+qLys5G/foAwAAAAAAACIAIOOPEiwmp2ZXR7ciyrveITXw0tn6zbQUA1Eikd9QlHRhAkcwRAIgJMZdO5A5u2UIMrAOgrR4NcxfNgZI6OfY7GKlZP0O8yUCIDFujbBRnamLEbf0887qidnXo6UgQA9IwTx6Zomd4RvJASEDoNmR2/XcqSyCWrE1tjGJ1oLWlKt4zsFekK9oyB4Hl0HF0yQAAQEr6AMAAAAAAAAiACDjjxIsJqdmV0e3Isq73iE18NLZ+s20FANRIpHfUJR0YSICAo3uyJxKHR9Z8fwvU7cywQCnZyPvtMl3nv54wPW1GSGqSDBFAiEAlLY98zqEL/xTUvm9ZKy5kBa4UWfr4Ryu6BmSZjseXPQCIGy7efKbZLQSDq8RhgNNjl1384gWFTN7nPwWV//SGriyAQEFQSECje7InEodH1nx/C9TtzLBAKdnI++0yXee/njA9bUZIaqsc2R2qRQhPRlaLsh/M/K/9fvbjxF/M20cNoitWrJoIgYCF7Rj5jFhe5L6VDzP5m2BeaG0mA9e7+6fMeWkWxLwpbAMimTyqQAAAADNAAAAIgYCje7InEodH1nx/C9TtzLBAKdnI++0yXee/njA9bUZIaoM9azC/QAAAADNAAAAAAA="); + let bitcoin_txs: Vec<_> = (0..2) + .map(|i| bitcoin::Transaction { + version: bitcoin::transaction::Version::TWO, + lock_time: bitcoin::absolute::LockTime::from_height(i).unwrap(), + input: Vec::new(), + output: Vec::new(), + }) + .collect(); // The helper that was used to store Spend transaction in previous versions of the software // when there was no associated timestamp. fn store_spend_old(conn: &mut rusqlite::Connection, psbt: &Psbt) { @@ -2147,20 +2466,14 @@ CREATE TABLE labels ( let mut conn = rusqlite::Connection::open(&db_path).unwrap(); store_coin_old( &mut conn, - &bitcoin::OutPoint::from_str( - "ed6c8f1af9325f84de521e785e7ddfd33dc28c9ada4d687dcd3850100bde54e9:5", - ) - .unwrap(), + &bitcoin::OutPoint::new(bitcoin_txs.first().unwrap().txid(), 5), bitcoin::Amount::from_sat(14_000), 24.into(), true, ); store_coin_old( &mut conn, - &bitcoin::OutPoint::from_str( - "81b2f327d4c1fd67afd039374f8798fd9ff37932c6f5c221c1c569350eac5ac8:2", - ) - .unwrap(), + &bitcoin::OutPoint::new(bitcoin_txs.get(1).unwrap().txid(), 2), bitcoin::Amount::from_sat(392_093_123), 24_567.into(), false, @@ -2168,10 +2481,17 @@ CREATE TABLE labels ( } // Migrate the DB. - maybe_apply_migration(&db_path).unwrap(); - maybe_apply_migration(&db_path).unwrap(); // Migrating twice will be a no-op. + maybe_apply_migration(&db_path, &bitcoin_txs).unwrap(); + // Migrating twice will be a no-op. No need to pass `bitcoin_txs` second time. + maybe_apply_migration(&db_path, &[]).unwrap(); let db = SqliteDb::new(db_path, None, &secp).unwrap(); + // The DB version has been updated. + { + let mut conn = db.connection().unwrap(); + let version = conn.db_version(); + assert_eq!(version, 5); + } // We should now be able to insert another PSBT, to query both, and the first PSBT must // have no associated timestamp. { @@ -2194,11 +2514,15 @@ CREATE TABLE labels ( // should not be immature. { let mut conn = db.connection().unwrap(); + let tx = bitcoin::Transaction { + version: bitcoin::transaction::Version::TWO, + lock_time: bitcoin::absolute::LockTime::from_height(2).unwrap(), + input: Vec::new(), + output: Vec::new(), + }; + conn.new_txs(&[tx.clone()]); conn.new_unspent_coins(&[Coin { - outpoint: bitcoin::OutPoint::from_str( - "6f0dc85a369b44458eba3a1f0ea5b5935d563afb6994f70f5b0094e05be1676c:1", - ) - .unwrap(), + outpoint: bitcoin::OutPoint::new(tx.txid(), 1), is_immature: true, block_info: None, amount: bitcoin::Amount::from_sat(98765), @@ -2212,11 +2536,27 @@ CREATE TABLE labels ( assert_eq!(coins.iter().filter(|c| !c.is_immature).count(), 2); } + // We can insert labels. + { + let mut conn = db.connection().unwrap(); + + let txid_str = "0c62a990d20d54429e70859292e82374ba6b1b951a3ab60f26bb65fee5724ff7"; + let txid = LabelItem::from_str(txid_str, bitcoin::Network::Bitcoin).unwrap(); + let mut txids_labels = HashMap::new(); + txids_labels.insert(txid.clone(), Some("hello".to_string())); + conn.update_labels(&txids_labels); + + let mut items = HashSet::new(); + items.insert(txid); + let db_labels = conn.db_labels(&items); + assert_eq!(db_labels[0].value, "hello"); + } + fs::remove_dir_all(tmp_dir).unwrap(); } #[test] - fn v3_to_v4_migration() { + fn v3_to_v5_migration() { let secp = secp256k1::Secp256k1::verification_only(); // Create a database with version 3, using the old schema. @@ -2231,23 +2571,26 @@ CREATE TABLE labels ( create_fresh_db(&db_path, options, &secp).unwrap(); { - // Don't use SqliteDb::new() in order not to apply migration. - let db = SqliteDb { - db_path: db_path.clone(), - }; + let db = SqliteDb::new(db_path.clone(), None, &secp).unwrap(); let mut conn = db.connection().unwrap(); assert!(conn.db_version() == 3); + let bitcoin_txs: Vec<_> = (0..8) + .map(|i| bitcoin::Transaction { + version: bitcoin::transaction::Version::TWO, + lock_time: bitcoin::absolute::LockTime::from_height(i).unwrap(), + input: Vec::new(), + output: Vec::new(), + }) + .collect(); + // The following coins will be inserted into the DB as unconfirmed and then // some of them will be subsequently confirmed and spent. // Note that `block_info`, `spend_txid` and `spend_block` will all be set to // NULL in the DB by the `new_unspent_coins` method, but are set to `None` // here anyway. let coin_a = Coin { - outpoint: bitcoin::OutPoint::from_str( - "6f0dc85a369b44458eba3a1f0ea5b5935d563afb6994f70f5b0094e05be1676c:1", - ) - .unwrap(), + outpoint: bitcoin::OutPoint::new(bitcoin_txs.first().unwrap().txid(), 1), is_immature: false, amount: bitcoin::Amount::from_sat(1231001), derivation_index: bip32::ChildNumber::from_normal_idx(101).unwrap(), @@ -2257,10 +2600,7 @@ CREATE TABLE labels ( spend_block: None, }; let coin_b = Coin { - outpoint: bitcoin::OutPoint::from_str( - "81b2f327d4c1fd67afd039374f8798fd9ff37932c6f5c221c1c569350eac5ac8:19234", - ) - .unwrap(), + outpoint: bitcoin::OutPoint::new(bitcoin_txs.get(1).unwrap().txid(), 19234), is_immature: false, amount: bitcoin::Amount::from_sat(23145), derivation_index: bip32::ChildNumber::from_normal_idx(10).unwrap(), @@ -2270,10 +2610,7 @@ CREATE TABLE labels ( spend_block: None, }; let coin_c = Coin { - outpoint: bitcoin::OutPoint::from_str( - "7477017f992cdc7ba08acafb77cb3b5bc0f42ac340d3e1e1da0785bdda20d5f6:932", - ) - .unwrap(), + outpoint: bitcoin::OutPoint::new(bitcoin_txs.get(2).unwrap().txid(), 932), is_immature: false, amount: bitcoin::Amount::from_sat(354764), derivation_index: bip32::ChildNumber::from_normal_idx(3401).unwrap(), @@ -2283,10 +2620,7 @@ CREATE TABLE labels ( spend_block: None, }; let coin_d = Coin { - outpoint: bitcoin::OutPoint::from_str( - "ed6c8f1af9325f84de521e785e7ddfd33dc28c9ada4d687dcd3850100bde54e9:1456", - ) - .unwrap(), + outpoint: bitcoin::OutPoint::new(bitcoin_txs.get(3).unwrap().txid(), 1456), is_immature: false, amount: bitcoin::Amount::from_sat(23200), derivation_index: bip32::ChildNumber::from_normal_idx(4793235).unwrap(), @@ -2296,10 +2630,7 @@ CREATE TABLE labels ( spend_block: None, }; let coin_e = Coin { - outpoint: bitcoin::OutPoint::from_str( - "4753a1d74c0af8dd0a0f3b763c14faf3bd9ed03cbdf33337a074fb0e9f6c7810:4633", - ) - .unwrap(), + outpoint: bitcoin::OutPoint::new(bitcoin_txs.get(4).unwrap().txid(), 4633), is_immature: false, amount: bitcoin::Amount::from_sat(675000), derivation_index: bip32::ChildNumber::from_normal_idx(3).unwrap(), @@ -2309,10 +2640,7 @@ CREATE TABLE labels ( spend_block: None, }; let coin_imma_a = Coin { - outpoint: bitcoin::OutPoint::from_str( - "c449539458c60bee6c0d8905ba1dadb20b9187b82045d306a408b894cea492b0:5", - ) - .unwrap(), + outpoint: bitcoin::OutPoint::new(bitcoin_txs.get(5).unwrap().txid(), 5), is_immature: true, amount: bitcoin::Amount::from_sat(4564347), derivation_index: bip32::ChildNumber::from_normal_idx(453).unwrap(), @@ -2322,10 +2650,7 @@ CREATE TABLE labels ( spend_block: None, }; let coin_imma_b = Coin { - outpoint: bitcoin::OutPoint::from_str( - "f0801fd9ca8bca0624c230ab422b2e2c4c8dc995e4e1dbc6412510959cce1e4f:19234", - ) - .unwrap(), + outpoint: bitcoin::OutPoint::new(bitcoin_txs.get(6).unwrap().txid(), 19234), is_immature: true, amount: bitcoin::Amount::from_sat(731453), derivation_index: bip32::ChildNumber::from_normal_idx(98).unwrap(), @@ -2358,16 +2683,13 @@ CREATE TABLE labels ( (coin_imma_a.outpoint, 176001, 1755001004), ]); conn.spend_coins(&[ - ( - coin_a.outpoint, - bitcoin::Txid::from_slice(&[1; 32][..]).unwrap(), - ), + (coin_a.outpoint, bitcoin_txs.get(7).unwrap().txid()), (coin_b.outpoint, coin_d.outpoint.txid), (coin_d.outpoint, coin_e.outpoint.txid), ]); conn.confirm_spend(&[( coin_a.outpoint, - bitcoin::Txid::from_slice(&[1; 32][..]).unwrap(), + bitcoin_txs.get(7).unwrap().txid(), 245500, 1755003000, )]); @@ -2395,10 +2717,11 @@ CREATE TABLE labels ( ); // Migrate the DB. - maybe_apply_migration(&db_path).unwrap(); - assert!(conn.db_version() == 4); - maybe_apply_migration(&db_path).unwrap(); // Migrating twice will be a no-op. - assert!(conn.db_version() == 4); + maybe_apply_migration(&db_path, &bitcoin_txs).unwrap(); + assert_eq!(conn.db_version(), 5); + // Migrating twice will be a no-op. No need to pass `bitcoin_txs` second time. + maybe_apply_migration(&db_path, &[]).unwrap(); + assert!(conn.db_version() == 5); let coins_post = conn.coins(&[], &[]); assert_eq!(coins_pre, coins_post); } @@ -2407,38 +2730,155 @@ CREATE TABLE labels ( } #[test] - fn v0_to_v4_migration() { + fn v4_to_v5_migration() { let secp = secp256k1::Secp256k1::verification_only(); - // Create a database with version 0, using the old schema. + // Create a database with version 3, using the old schema. let tmp_dir = tmp_dir(); fs::create_dir_all(&tmp_dir).unwrap(); - let db_path: path::PathBuf = [tmp_dir.as_path(), path::Path::new("lianad_v0.sqlite3")] + let db_path: path::PathBuf = [tmp_dir.as_path(), path::Path::new("lianad_v4.sqlite3")] .iter() .collect(); let mut options = dummy_options(); - options.schema = V0_SCHEMA; - options.version = 0; - create_fresh_db(&db_path, options, &secp).unwrap(); + options.schema = V4_SCHEMA; + options.version = 4; - // SqliteDb new is doing the migration. - let db = SqliteDb::new(db_path, None, &secp).unwrap(); + // Create a hundred different transactions, from which originate two hundred + // pseudo-random coins. + let mut bitcoin_txs: Vec<_> = (0..100) + .map(|i| bitcoin::Transaction { + version: bitcoin::transaction::Version::TWO, + lock_time: bitcoin::absolute::LockTime::from_height(i).unwrap(), + input: Vec::new(), + output: Vec::new(), + }) + .collect(); + let spend_txs: Vec<_> = (0..10) + .map(|i| { + ( + bitcoin::Transaction { + version: bitcoin::transaction::Version::TWO, + lock_time: bitcoin::absolute::LockTime::from_height(1_234 + i).unwrap(), + input: Vec::new(), + output: Vec::new(), + }, + if i % 2 == 0 { + Some(BlockInfo { + height: (i % 5) as i32 * 2_000, + time: 1722488619 + (i % 5) * 84_999, + }) + } else { + None + }, + ) + }) + .collect(); + let coins: Vec = bitcoin_txs + .iter() + .chain(bitcoin_txs.iter()) // We do this to have coins which originate from the same tx. + .enumerate() + .map(|(i, tx)| Coin { + outpoint: bitcoin::OutPoint { + txid: tx.txid(), + vout: i as u32, + }, + is_immature: (i % 10) == 0, + amount: bitcoin::Amount::from_sat(i as u64 * 3473), + derivation_index: bip32::ChildNumber::from_normal_idx(i as u32 * 100).unwrap(), + is_change: (i % 4) == 0, + block_info: if i & 2 == 0 { + Some(BlockInfo { + height: (i % 100) as i32 * 1_000, + time: 1722408619 + (i % 100) as u32 * 42_000, + }) + } else { + None + }, + spend_txid: if i % 20 == 0 { + Some(spend_txs[i / 20].0.txid()) + } else { + None + }, + spend_block: if i % 20 == 0 { + spend_txs[i / 20].1 + } else { + None + }, + }) + .collect(); { + let db = SqliteDb::new(db_path.clone(), Some(options), &secp).unwrap(); let mut conn = db.connection().unwrap(); - let version = conn.db_version(); - assert_eq!(version, 4); - let txid_str = "0c62a990d20d54429e70859292e82374ba6b1b951a3ab60f26bb65fee5724ff7"; - let txid = LabelItem::from_str(txid_str, bitcoin::Network::Bitcoin).unwrap(); - let mut txids_labels = HashMap::new(); - txids_labels.insert(txid.clone(), Some("hello".to_string())); - conn.update_labels(&txids_labels); + // Insert all these coins into database. + conn.new_unspent_coins(&coins); - let mut items = HashSet::new(); - items.insert(txid); - let db_labels = conn.db_labels(&items); - assert_eq!(db_labels[0].value, "hello"); + // Confirm those which are supposed to be. + let confirmed_coins: Vec<_> = coins + .iter() + .filter_map(|coin| { + coin.block_info + .map(|blk| (coin.outpoint, blk.height, blk.time)) + }) + .collect(); + conn.confirm_coins(&confirmed_coins); + + // Spend those which are supposed to be. + let spent_coins: Vec<_> = coins + .iter() + .filter_map(|coin| coin.spend_txid.map(|txid| (coin.outpoint, txid))) + .collect(); + conn.spend_coins(&spent_coins); + + // Mark the spend as confirmed for those which are supposed to be. + let confirmed_spent_coins: Vec<_> = coins + .iter() + .filter_map(|coin| { + coin.spend_block.map(|blk| { + ( + coin.outpoint, + coin.spend_txid.expect("always set when spend block is"), + blk.height, + blk.time, + ) + }) + }) + .collect(); + conn.confirm_spend(&confirmed_spent_coins); + } + + // Trying to migrate without specifying the transactions will fail. + assert!(maybe_apply_migration(&db_path, &[]) + .unwrap_err() + .to_string() + .contains("FOREIGN KEY constraint failed")); + + // Trying to migrate without specifying ALL the transactions will fail. (Missing the spend + // tx here.) + assert!(maybe_apply_migration(&db_path, &[]) + .unwrap_err() + .to_string() + .contains("FOREIGN KEY constraint failed")); + + // Migration with all txs will succeed. + bitcoin_txs.extend(spend_txs.iter().map(|(tx, _)| tx.clone())); + maybe_apply_migration(&db_path, &bitcoin_txs).unwrap(); + + // Make sure all the transactions are indeed in DB. + { + let db = SqliteDb::new(db_path.clone(), None, &secp).unwrap(); + let mut conn = db.connection().unwrap(); + + let txids: Vec<_> = bitcoin_txs.iter().map(|tx| tx.txid()).collect(); + let bitcoin_txs_in_db: HashSet<_> = conn + .list_wallet_transactions(&txids) + .into_iter() + .map(|tx| tx.transaction) + .collect(); + let bitcoin_txs: HashSet<_> = bitcoin_txs.into_iter().collect(); + assert_eq!(bitcoin_txs.len(), bitcoin_txs_in_db.len()); + assert_eq!(bitcoin_txs, bitcoin_txs_in_db); } fs::remove_dir_all(tmp_dir).unwrap(); diff --git a/src/database/sqlite/schema.rs b/src/database/sqlite/schema.rs index ac101bcd..456e45cb 100644 --- a/src/database/sqlite/schema.rs +++ b/src/database/sqlite/schema.rs @@ -58,6 +58,12 @@ CREATE TABLE coins ( is_immature BOOLEAN NOT NULL CHECK (is_immature IN (0,1)), UNIQUE (txid, vout), FOREIGN KEY (wallet_id) REFERENCES wallets (id) + ON UPDATE RESTRICT + ON DELETE RESTRICT, + FOREIGN KEY (txid) REFERENCES transactions (txid) + ON UPDATE RESTRICT + ON DELETE RESTRICT, + FOREIGN KEY (spend_txid) REFERENCES transactions (txid) ON UPDATE RESTRICT ON DELETE RESTRICT ); @@ -71,6 +77,13 @@ CREATE TABLE addresses ( derivation_index INTEGER NOT NULL UNIQUE ); +/* Transactions for all wallets. */ +CREATE TABLE transactions ( + id INTEGER PRIMARY KEY NOT NULL, + txid BLOB UNIQUE NOT NULL, + tx BLOB UNIQUE NOT NULL +); + /* Transactions we created that spend some of our coins. */ CREATE TABLE spend_transactions ( id INTEGER PRIMARY KEY NOT NULL, @@ -346,3 +359,32 @@ impl TryFrom<&rusqlite::Row<'_>> for DbLabel { }) } } + +/// A transaction together with its block info. +#[derive(Clone, Debug, PartialEq)] +pub struct DbWalletTransaction { + pub transaction: bitcoin::Transaction, + pub block_info: Option, +} + +impl TryFrom<&rusqlite::Row<'_>> for DbWalletTransaction { + type Error = rusqlite::Error; + + fn try_from(row: &rusqlite::Row) -> Result { + let transaction: Vec = row.get(0)?; + let transaction: bitcoin::Transaction = + bitcoin::consensus::deserialize(&transaction).expect("We only store valid txs"); + let block_height: Option = row.get(1)?; + let block_time: Option = row.get(2)?; + assert_eq!(block_height.is_none(), block_time.is_none()); + let block_info = block_height.map(|height| DbBlockInfo { + height, + time: block_time.expect("Must be there if height is"), + }); + + Ok(DbWalletTransaction { + transaction, + block_info, + }) + } +} diff --git a/src/database/sqlite/utils.rs b/src/database/sqlite/utils.rs index e1981ee4..c3426cc1 100644 --- a/src/database/sqlite/utils.rs +++ b/src/database/sqlite/utils.rs @@ -2,7 +2,7 @@ use crate::database::sqlite::{FreshDbOptions, SqliteDbError, DB_VERSION}; use std::{convert::TryInto, fs, path, time}; -use miniscript::bitcoin::secp256k1; +use miniscript::bitcoin::{self, secp256k1}; pub const LOOK_AHEAD_LIMIT: u32 = 200; @@ -229,9 +229,80 @@ fn migrate_v3_to_v4(conn: &mut rusqlite::Connection) -> Result<(), SqliteDbError Ok(()) } +fn migrate_v4_to_v5( + conn: &mut rusqlite::Connection, + bitcoin_txs: &[bitcoin::Transaction], +) -> Result<(), SqliteDbError> { + db_exec(conn, |db_tx| { + db_tx.execute( + " + CREATE TABLE transactions ( + id INTEGER PRIMARY KEY NOT NULL, + txid BLOB UNIQUE NOT NULL, + tx BLOB UNIQUE NOT NULL + );", + rusqlite::params![], + )?; + + for bitcoin_tx in bitcoin_txs { + let txid = &bitcoin_tx.txid()[..].to_vec(); + let bitcoin_tx_ser = bitcoin::consensus::serialize(bitcoin_tx); + db_tx.execute( + "INSERT INTO transactions (txid, tx) VALUES (?1, ?2);", + rusqlite::params![txid, bitcoin_tx_ser,], + )?; + } + + // Create new coins table with foreign key constraints on transactions table. + db_tx.execute_batch( + " + CREATE TABLE coins_new ( + id INTEGER PRIMARY KEY NOT NULL, + wallet_id INTEGER NOT NULL, + blockheight INTEGER, + blocktime INTEGER, + txid BLOB NOT NULL, + vout INTEGER NOT NULL, + amount_sat INTEGER NOT NULL, + derivation_index INTEGER NOT NULL, + is_change BOOLEAN NOT NULL CHECK (is_change IN (0,1)), + spend_txid BLOB, + spend_block_height INTEGER, + spend_block_time INTEGER, + is_immature BOOLEAN NOT NULL CHECK (is_immature IN (0,1)), + UNIQUE (txid, vout), + FOREIGN KEY (wallet_id) REFERENCES wallets (id) + ON UPDATE RESTRICT + ON DELETE RESTRICT, + FOREIGN KEY (txid) REFERENCES transactions (txid) + ON UPDATE RESTRICT + ON DELETE RESTRICT, + FOREIGN KEY (spend_txid) REFERENCES transactions (txid) + ON UPDATE RESTRICT + ON DELETE RESTRICT + ); + + INSERT INTO coins_new SELECT * FROM coins; + + DROP TABLE coins; + + ALTER TABLE coins_new RENAME TO coins; + + UPDATE version SET version = 5;", + ) + })?; + Ok(()) +} + /// Check the database version and if necessary apply the migrations to upgrade it to the current -/// one. -pub fn maybe_apply_migration(db_path: &path::Path) -> Result<(), SqliteDbError> { +/// one. The `bitcoin_txs` parameter is here for the migration from versions 4 and earlier, which +/// did not store the Bitcoin transactions in database, to versions 5 and later, which do. For a +/// migration from v4 or earlier to v5 or later it is assumed the caller passes *all* necessary +/// transactions, otherwise the migration will fail. +pub fn maybe_apply_migration( + db_path: &path::Path, + bitcoin_txs: &[bitcoin::Transaction], +) -> Result<(), SqliteDbError> { let mut conn = rusqlite::Connection::open(db_path)?; // Iteratively apply the database migrations necessary. @@ -262,6 +333,15 @@ pub fn maybe_apply_migration(db_path: &path::Path) -> Result<(), SqliteDbError> migrate_v3_to_v4(&mut conn)?; log::warn!("Migration from database version 3 to version 4 successful."); } + 4 => { + log::warn!("Upgrading database from version 4 to version 5."); + log::warn!( + "Number of bitcoin transactions to be inserted: {}.", + bitcoin_txs.len() + ); + migrate_v4_to_v5(&mut conn, bitcoin_txs)?; + log::warn!("Migration from database version 4 to version 5 successful."); + } _ => return Err(SqliteDbError::UnsupportedVersion(version)), } } diff --git a/src/lib.rs b/src/lib.rs index b6064291..584a3f3c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -23,13 +23,13 @@ use crate::{ bitcoin::{poller, BitcoinInterface}, config::Config, database::{ - sqlite::{FreshDbOptions, SqliteDb, SqliteDbError}, + sqlite::{FreshDbOptions, SqliteDb, SqliteDbError, MAX_DB_VERSION_NO_TX_DB}, DatabaseInterface, }, }; use std::{ - error, fmt, fs, io, path, + collections, error, fmt, fs, io, path, sync::{self, mpsc}, thread, }; @@ -92,6 +92,7 @@ pub enum StartupError { DefaultDataDirNotFound, DatadirCreation(path::PathBuf, io::Error), MissingBitcoindConfig, + DbMigrateBitcoinTxs(&'static str), Database(SqliteDbError), Bitcoind(BitcoindError), #[cfg(unix)] @@ -116,6 +117,10 @@ impl fmt::Display for StartupError { f, "Our Bitcoin interface is bitcoind but we have no 'bitcoind_config' entry in the configuration." ), + Self::DbMigrateBitcoinTxs(msg) => write!( + f, + "Error when migrating Bitcoin transaction from Bitcoin backend to database: {}.", msg + ), Self::Database(e) => write!(f, "Error initializing database: '{}'.", e), Self::Bitcoind(e) => write!(f, "Error setting up bitcoind interface: '{}'.", e), #[cfg(unix)] @@ -184,6 +189,7 @@ fn setup_sqlite( data_dir: &path::Path, fresh_data_dir: bool, secp: &secp256k1::Secp256k1, + bitcoind: &Option, ) -> Result { let db_path: path::PathBuf = [data_dir, path::Path::new("lianad.sqlite3")] .iter() @@ -196,7 +202,35 @@ fn setup_sqlite( } else { None }; + + // If opening an existing wallet whose database does not yet store the wallet transactions, + // query them from the Bitcoin backend before proceeding to the migration. let sqlite = SqliteDb::new(db_path, options, secp)?; + if !fresh_data_dir { + let mut conn = sqlite.connection()?; + let wallet_txs = if conn.db_version() <= MAX_DB_VERSION_NO_TX_DB { + let bit = bitcoind.as_ref().ok_or(StartupError::DbMigrateBitcoinTxs( + "a connection to a Bitcoin backend is required", + ))?; + let coins = conn.db_coins(&[]); + let coins_txids = coins + .iter() + .map(|c| c.outpoint.txid) + .chain(coins.iter().filter_map(|c| c.spend_txid)) + .collect::>(); + coins_txids + .into_iter() + .map(|txid| bit.get_transaction(&txid).map(|res| res.tx)) + .collect::>>() + .ok_or(StartupError::DbMigrateBitcoinTxs( + "missing transaction in Bitcoin backend", + ))? + } else { + Vec::new() + }; + sqlite.maybe_apply_migrations(&wallet_txs)?; + } + sqlite.sanity_check(config.bitcoin_config.network, &config.main_descriptor)?; log::info!("Database initialized and checked."); @@ -337,7 +371,15 @@ impl DaemonHandle { log::info!("Created a new data directory at '{}'", data_dir.display()); } - // Then set up the database + // Set up the connection to bitcoind (if using it) first as we may need it for the database + // migration when setting up SQLite below. + let bitcoind = if bitcoin.is_none() { + Some(setup_bitcoind(&config, &data_dir, fresh_data_dir)?) + } else { + None + }; + + // Then set up the database backend. let db = match db { Some(db) => sync::Arc::from(sync::Mutex::from(db)), None => sync::Arc::from(sync::Mutex::from(setup_sqlite( @@ -345,17 +387,16 @@ impl DaemonHandle { &data_dir, fresh_data_dir, &secp, + &bitcoind, )?)) as sync::Arc>, }; - // Now, set up the Bitcoin interface. - let bit = match bitcoin { - Some(bit) => sync::Arc::from(sync::Mutex::from(bit)), - None => sync::Arc::from(sync::Mutex::from(setup_bitcoind( - &config, - &data_dir, - fresh_data_dir, - )?)) as sync::Arc>, + // Finally set up the Bitcoin backend. + let bit = match (bitcoin, bitcoind) { + (Some(bit), None) => sync::Arc::from(sync::Mutex::from(bit)), + (None, Some(bit)) => sync::Arc::from(sync::Mutex::from(bit)) + as sync::Arc>, + _ => unreachable!("Either bitcoind or bitcoin interface is always set."), }; // If we are on a UNIX system and they told us to daemonize, do it now. diff --git a/src/testutils.rs b/src/testutils.rs index 0d4b028e..3669b5fc 100644 --- a/src/testutils.rs +++ b/src/testutils.rs @@ -138,6 +138,7 @@ struct DummyDbState { change_index: bip32::ChildNumber, curr_tip: Option, coins: HashMap, + txs: HashMap, spend_txs: HashMap)>, timestamp: u32, } @@ -169,6 +170,7 @@ impl DummyDatabase { change_index: 0.into(), curr_tip: None, coins: HashMap::new(), + txs: HashMap::new(), spend_txs: HashMap::new(), timestamp: now, })), @@ -437,6 +439,48 @@ impl DatabaseConnection for DummyDatabase { txids_and_time.truncate(limit as usize); txids_and_time.into_iter().map(|(txid, _)| txid).collect() } + + fn list_saved_txids(&mut self) -> Vec { + self.db.read().unwrap().txs.keys().cloned().collect() + } + + fn new_txs(&mut self, txs: &[bitcoin::Transaction]) { + for tx in txs { + self.db.write().unwrap().txs.insert(tx.txid(), tx.clone()); + } + } + + fn list_wallet_transactions( + &mut self, + txids: &[bitcoin::Txid], + ) -> Vec<(bitcoin::Transaction, Option, Option)> { + let txs: HashMap<_, _> = self + .db + .read() + .unwrap() + .txs + .clone() + .into_iter() + .filter(|(txid, _tx)| txids.contains(txid)) + .collect(); + let coins = self.coins(&[], &[]); + let mut wallet_txs = Vec::with_capacity(txs.len()); + for (txid, tx) in txs { + let first_block_info = coins.values().find_map(|c| { + if c.outpoint.txid == txid { + Some(c.block_info) + } else if c.spend_txid == Some(txid) { + Some(c.spend_block) + } else { + None + } + }); + if let Some(block_info) = first_block_info { + wallet_txs.push((tx, block_info.map(|b| b.height), block_info.map(|b| b.time))); + } + } + wallet_txs + } } pub struct DummyLiana {