From a86d12d629c0197b9611b36a1b133cf3f7d41dc8 Mon Sep 17 00:00:00 2001 From: jp1ac4 <121959000+jp1ac4@users.noreply.github.com> Date: Mon, 8 Jul 2024 12:54:33 +0100 Subject: [PATCH] commands: get wallet transactions from db --- src/commands/mod.rs | 91 +++++++------- src/database/mod.rs | 22 ++++ src/database/sqlite/mod.rs | 228 +++++++++++++++++++++++++++++++++- src/database/sqlite/schema.rs | 29 +++++ src/testutils.rs | 32 +++++ 5 files changed, 359 insertions(+), 43 deletions(-) diff --git a/src/commands/mod.rs b/src/commands/mod.rs index 714cd1b3..6588aa67 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -990,39 +990,19 @@ impl DaemonControl { limit: u64, ) -> ListTransactionsResult { let mut db_conn = self.db.connection(); + // Note the result could in principle be retrieved in a single database query. let txids = db_conn.list_txids(start, end, limit); - let transactions = txids - .iter() - .filter_map(|txid| { - // TODO: batch those calls to the Bitcoin backend - // so it can in turn optimize its queries. - self.bitcoin - .wallet_transaction(txid) - .map(|(tx, block)| TransactionInfo { - tx, - height: block.map(|b| b.height), - time: block.map(|b| b.time), - }) - }) - .collect(); - ListTransactionsResult { transactions } + self.list_transactions(&txids) } /// list_transactions retrieves the transactions with the given txids. pub fn list_transactions(&self, txids: &[bitcoin::Txid]) -> ListTransactionsResult { - let transactions = txids - .iter() - .filter_map(|txid| { - // TODO: batch those calls to the Bitcoin backend - // so it can in turn optimize its queries. - self.bitcoin - .wallet_transaction(txid) - .map(|(tx, block)| TransactionInfo { - tx, - height: block.map(|b| b.height), - time: block.map(|b| b.time), - }) - }) + let transactions = self + .db + .connection() + .list_wallet_transactions(txids) + .into_iter() + .map(|(tx, height, time)| TransactionInfo { tx, height, time }) .collect(); ListTransactionsResult { transactions } } @@ -2278,8 +2258,8 @@ mod tests { }, ]); - let mut btc = DummyBitcoind::new(); - btc.txs.insert( + let mut txs_map = HashMap::new(); + txs_map.insert( deposit1.txid(), ( deposit1.clone(), @@ -2293,7 +2273,7 @@ mod tests { }), ), ); - btc.txs.insert( + txs_map.insert( deposit2.txid(), ( deposit2.clone(), @@ -2307,7 +2287,7 @@ mod tests { }), ), ); - btc.txs.insert( + txs_map.insert( spend_tx.txid(), ( spend_tx.clone(), @@ -2321,7 +2301,7 @@ mod tests { }), ), ); - btc.txs.insert( + txs_map.insert( deposit3.txid(), ( deposit3.clone(), @@ -2336,11 +2316,15 @@ mod tests { ), ); - let ms = DummyLiana::new(btc, db); + let ms = DummyLiana::new(DummyBitcoind::new(), db); let control = &ms.control(); + let mut db_conn = control.db.connection(); + let txs: Vec<_> = txs_map.values().map(|(tx, _)| tx.clone()).collect(); + db_conn.new_txs(&txs); - let transactions = control.list_confirmed_transactions(0, 4, 10).transactions; + let mut transactions = control.list_confirmed_transactions(0, 4, 10).transactions; + transactions.sort_by(|tx1, tx2| tx2.height.cmp(&tx1.height)); assert_eq!(transactions.len(), 4); assert_eq!(transactions[0].time, Some(4)); @@ -2355,7 +2339,8 @@ mod tests { assert_eq!(transactions[3].time, Some(1)); assert_eq!(transactions[3].tx, deposit1); - let transactions = control.list_confirmed_transactions(2, 3, 10).transactions; + let mut transactions = control.list_confirmed_transactions(2, 3, 10).transactions; + transactions.sort_by(|tx1, tx2| tx2.height.cmp(&tx1.height)); assert_eq!(transactions.len(), 2); assert_eq!(transactions[0].time, Some(3)); @@ -2424,8 +2409,8 @@ mod tests { }], }; - let mut btc = DummyBitcoind::new(); - btc.txs.insert( + let mut txs_map = HashMap::new(); + txs_map.insert( tx1.txid(), ( tx1.clone(), @@ -2439,7 +2424,7 @@ mod tests { }), ), ); - btc.txs.insert( + txs_map.insert( tx2.txid(), ( tx2.clone(), @@ -2453,7 +2438,7 @@ mod tests { }), ), ); - btc.txs.insert( + txs_map.insert( tx3.txid(), ( tx3.clone(), @@ -2468,9 +2453,31 @@ mod tests { ), ); - let ms = DummyLiana::new(btc, DummyDatabase::new()); - + let ms = DummyLiana::new(DummyBitcoind::new(), DummyDatabase::new()); let control = &ms.control(); + let mut db_conn = control.db.connection(); + let txs: Vec<_> = txs_map.values().map(|(tx, _)| tx.clone()).collect(); + db_conn.new_txs(&txs); + // We need coins in the DB in order to get the block info for the transactions. + for (txid, (_tx, block)) in txs_map { + // Insert more than one coin per transaction to check that the command does not + // return duplicate transactions. + for vout in 0..4 { + db_conn.new_unspent_coins(&[Coin { + outpoint: bitcoin::OutPoint::new(txid, vout), + is_immature: false, + block_info: block.map(|b| BlockInfo { + height: b.height, + time: b.time, + }), + amount: bitcoin::Amount::from_sat(100_000), + derivation_index: bip32::ChildNumber::from(13), + is_change: false, + spend_txid: None, + spend_block: None, + }]); + } + } let transactions = control.list_transactions(&[tx1.txid()]).transactions; assert_eq!(transactions.len(), 1); diff --git a/src/database/mod.rs b/src/database/mod.rs index 9388821d..3940b7eb 100644 --- a/src/database/mod.rs +++ b/src/database/mod.rs @@ -152,6 +152,12 @@ pub trait DatabaseConnection { /// Store transactions in database, ignoring any that already exist. fn new_txs(&mut self, txs: &[bitcoin::Transaction]); + + /// Retrieve a list of transactions and their corresponding block heights and times. + fn list_wallet_transactions( + &mut self, + txids: &[bitcoin::Txid], + ) -> Vec<(bitcoin::Transaction, Option, Option)>; } impl DatabaseConnection for SqliteConn { @@ -324,6 +330,22 @@ impl DatabaseConnection for SqliteConn { fn new_txs<'a>(&mut self, txs: &[bitcoin::Transaction]) { self.new_txs(txs) } + + fn list_wallet_transactions( + &mut self, + txids: &[bitcoin::Txid], + ) -> Vec<(bitcoin::Transaction, Option, Option)> { + self.list_wallet_transactions(txids) + .into_iter() + .map(|wtx| { + ( + wtx.transaction, + wtx.block_info.map(|b| b.height), + wtx.block_info.map(|b| b.time), + ) + }) + .collect() + } } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] diff --git a/src/database/sqlite/mod.rs b/src/database/sqlite/mod.rs index 7a624b2d..9cd5e037 100644 --- a/src/database/sqlite/mod.rs +++ b/src/database/sqlite/mod.rs @@ -16,7 +16,7 @@ use crate::{ sqlite::{ schema::{ DbAddress, DbCoin, DbLabel, DbLabelledKind, DbSpendTransaction, DbTip, DbWallet, - SCHEMA, + DbWalletTransaction, SCHEMA, }, utils::{ create_fresh_db, curr_timestamp, db_exec, db_query, db_tx_query, db_version, @@ -729,6 +729,50 @@ impl SqliteConn { .expect("Database must be available") } + pub fn list_wallet_transactions( + &mut self, + txids: &[bitcoin::Txid], + ) -> Vec { + // The UNION will remove duplicates. + // We assume that a transaction's block info is the same in every coins row + // it appears in. + let query = format!( + "SELECT t.tx, c.blockheight, c.blocktime \ + FROM transactions t \ + INNER JOIN ( \ + SELECT txid, blockheight, blocktime \ + FROM coins \ + WHERE wallet_id = {WALLET_ID} \ + UNION \ + SELECT spend_txid, spend_block_height, spend_block_time \ + FROM coins \ + WHERE wallet_id = {WALLET_ID} \ + AND spend_txid IS NOT NULL \ + ) c ON t.txid = c.txid \ + WHERE t.txid in ({})", + txids + .iter() + .map(|txid| format!("x'{}'", FrontwardHexTxid(*txid))) + .collect::>() + .join(",") + ); + let w_txs: Vec = + db_query(&mut self.conn, &query, rusqlite::params![], |row| { + row.try_into() + }) + .expect("Db must not fail"); + debug_assert_eq!( + w_txs.len(), + w_txs + .iter() + .map(|t| t.transaction.txid()) + .collect::>() + .len(), + "database must not contain inconsistent block info for the same txid" + ); + w_txs + } + pub fn delete_spend(&mut self, txid: &bitcoin::Txid) { db_exec(&mut self.conn, |db_tx| { db_tx.execute( @@ -2072,6 +2116,188 @@ CREATE TABLE labels ( fs::remove_dir_all(tmp_dir).unwrap(); } + #[test] + fn sqlite_list_wallet_transactions() { + let (tmp_dir, _, _, db) = dummy_db(); + + { + let mut conn = db.connection().unwrap(); + + // The following is based on the `v4_to_v5_migration` test. + let mut bitcoin_txs: Vec<_> = (0..100) + .map(|i| bitcoin::Transaction { + version: bitcoin::transaction::Version::TWO, + lock_time: bitcoin::absolute::LockTime::from_height(i).unwrap(), + input: Vec::new(), + output: Vec::new(), + }) + .collect(); + let spend_txs: Vec<_> = (0..10) + .map(|i| { + ( + bitcoin::Transaction { + version: bitcoin::transaction::Version::TWO, + lock_time: bitcoin::absolute::LockTime::from_height(1_234 + i).unwrap(), + input: Vec::new(), + output: Vec::new(), + }, + if i % 2 == 0 { + Some(BlockInfo { + height: (i % 5) as i32 * 2_000, + time: 1722488619 + (i % 5) * 84_999, + }) + } else { + None + }, + ) + }) + .collect(); + let coins: Vec = bitcoin_txs + .iter() + .chain(bitcoin_txs.iter()) // We do this to have coins which originate from the same tx. + .enumerate() + .map(|(i, tx)| Coin { + outpoint: bitcoin::OutPoint { + txid: tx.txid(), + vout: i as u32, + }, + is_immature: (i % 10) == 0, + amount: bitcoin::Amount::from_sat(i as u64 * 3473), + derivation_index: bip32::ChildNumber::from_normal_idx(i as u32 * 100).unwrap(), + is_change: (i % 4) == 0, + block_info: if i & 2 == 0 { + Some(BlockInfo { + height: (i % 100) as i32 * 1_000, + time: 1722408619 + (i % 100) as u32 * 42_000, + }) + } else { + None + }, + spend_txid: if i % 20 == 0 { + Some(spend_txs[i / 20].0.txid()) + } else { + None + }, + spend_block: if i % 20 == 0 { + spend_txs[i / 20].1 + } else { + None + }, + }) + .collect(); + + bitcoin_txs.extend(spend_txs.into_iter().map(|(tx, _)| tx)); + conn.new_txs(&bitcoin_txs); + + // Insert all these coins into database. + conn.new_unspent_coins(&coins); + + // Confirm those which are supposed to be. + let confirmed_coins: Vec<_> = coins + .iter() + .filter_map(|coin| { + coin.block_info + .map(|blk| (coin.outpoint, blk.height, blk.time)) + }) + .collect(); + conn.confirm_coins(&confirmed_coins); + + // Spend those which are supposed to be. + let spent_coins: Vec<_> = coins + .iter() + .filter_map(|coin| coin.spend_txid.map(|txid| (coin.outpoint, txid))) + .collect(); + conn.spend_coins(&spent_coins); + + // Mark the spend as confirmed for those which are supposed to be. + let confirmed_spent_coins: Vec<_> = coins + .iter() + .filter_map(|coin| { + coin.spend_block.map(|blk| { + ( + coin.outpoint, + coin.spend_txid.expect("always set when spend block is"), + blk.height, + blk.time, + ) + }) + }) + .collect(); + conn.confirm_spend(&confirmed_spent_coins); + + // For easy lookup, map each tx to its txid. + let bitcoin_txs: HashMap<_, _> = + bitcoin_txs.into_iter().map(|tx| (tx.txid(), tx)).collect(); + + let block_info_from_coins: HashSet<_> = coins + .iter() + .map(|c| (c.outpoint.txid, c.block_info)) + .chain( + coins + .iter() + .filter_map(|c| c.spend_txid.map(|txid| (txid, c.spend_block))), + ) + .collect(); + + // Make sure each txid only has one block info. + assert_eq!(bitcoin_txs.len(), block_info_from_coins.len()); + + // For each txid, determine its wallet transaction based on the coins defined above. + let wallet_txs_from_coins: HashMap<_, _> = block_info_from_coins + .into_iter() + .map(|(txid, block_info)| { + let tx = bitcoin_txs.get(&txid).unwrap(); + + ( + txid, + DbWalletTransaction { + transaction: tx.clone(), + block_info: block_info.map(|info| DbBlockInfo { + height: info.height, + time: info.time, + }), + }, + ) + }) + .collect(); + + let all_txids: Vec<_> = bitcoin_txs.keys().cloned().collect(); + + for indices in [ + (0..all_txids.len()).collect(), + (0..all_txids.len() / 2).collect(), + (all_txids.len() / 5..all_txids.len() / 2).collect(), + vec![3, 4, 5, 6], + vec![4, 5], + vec![1, 3, 5], + vec![1], + vec![1, 1, 3, 4, 3], // we can pass duplicate txids + vec![], // can pass empty slice + ] { + let txids: Vec<_> = indices + .iter() + .map(|i| *all_txids.get(*i).unwrap()) + .collect(); + + // Make sure we have the expected number of txids. + assert_eq!(txids.len(), indices.len()); + + let mut db_txs = conn.list_wallet_transactions(&txids); + db_txs.sort_by(|a, b| a.transaction.txid().cmp(&b.transaction.txid())); + let mut expected_txs: Vec<_> = txids + .iter() + .collect::>() // remove duplicates + .into_iter() + .map(|txid| wallet_txs_from_coins.get(txid).unwrap().clone()) + .collect(); + expected_txs.sort_by(|a, b| a.transaction.txid().cmp(&b.transaction.txid())); + assert_eq!(&db_txs[..], &expected_txs[..],); + } + } + + fs::remove_dir_all(tmp_dir).unwrap(); + } + #[test] fn v0_to_v2_migration() { let secp = secp256k1::Secp256k1::verification_only(); diff --git a/src/database/sqlite/schema.rs b/src/database/sqlite/schema.rs index 4ffc2979..456e45cb 100644 --- a/src/database/sqlite/schema.rs +++ b/src/database/sqlite/schema.rs @@ -359,3 +359,32 @@ impl TryFrom<&rusqlite::Row<'_>> for DbLabel { }) } } + +/// A transaction together with its block info. +#[derive(Clone, Debug, PartialEq)] +pub struct DbWalletTransaction { + pub transaction: bitcoin::Transaction, + pub block_info: Option, +} + +impl TryFrom<&rusqlite::Row<'_>> for DbWalletTransaction { + type Error = rusqlite::Error; + + fn try_from(row: &rusqlite::Row) -> Result { + let transaction: Vec = row.get(0)?; + let transaction: bitcoin::Transaction = + bitcoin::consensus::deserialize(&transaction).expect("We only store valid txs"); + let block_height: Option = row.get(1)?; + let block_time: Option = row.get(2)?; + assert_eq!(block_height.is_none(), block_time.is_none()); + let block_info = block_height.map(|height| DbBlockInfo { + height, + time: block_time.expect("Must be there if height is"), + }); + + Ok(DbWalletTransaction { + transaction, + block_info, + }) + } +} diff --git a/src/testutils.rs b/src/testutils.rs index c9f1f776..aee756aa 100644 --- a/src/testutils.rs +++ b/src/testutils.rs @@ -449,6 +449,38 @@ impl DatabaseConnection for DummyDatabase { self.db.write().unwrap().txs.insert(tx.txid(), tx.clone()); } } + + fn list_wallet_transactions( + &mut self, + txids: &[bitcoin::Txid], + ) -> Vec<(bitcoin::Transaction, Option, Option)> { + let txs: HashMap<_, _> = self + .db + .read() + .unwrap() + .txs + .clone() + .into_iter() + .filter(|(txid, _tx)| txids.contains(txid)) + .collect(); + let coins = self.coins(&[], &[]); + let mut wallet_txs = Vec::with_capacity(txs.len()); + for (txid, tx) in txs { + let first_block_info = coins.values().find_map(|c| { + if c.outpoint.txid == txid { + Some(c.block_info) + } else if c.spend_txid == Some(txid) { + Some(c.spend_block) + } else { + None + } + }); + if let Some(block_info) = first_block_info { + wallet_txs.push((tx, block_info.map(|b| b.height), block_info.map(|b| b.time))); + } + } + wallet_txs + } } pub struct DummyLiana {