diff --git a/tests/test_chain.py b/tests/test_chain.py index 96eb06ca..e23987d8 100644 --- a/tests/test_chain.py +++ b/tests/test_chain.py @@ -1,3 +1,5 @@ +import time + from fixtures import * from test_framework.utils import wait_for, get_txid, spend_coins @@ -166,3 +168,91 @@ def test_reorg_status_recovery(minisafed, bitcoind): new_coin_b = get_coin(minisafed, coin_b["outpoint"]) coin_b["spend_info"]["height"] = initial_height assert new_coin_b == coin_b + + +def test_rescan_edge_cases(minisafed, bitcoind): + """Test some specific cases that could arise when rescanning the chain.""" + initial_tip = bitcoind.rpc.getblockheader(bitcoind.rpc.getbestblockhash()) + + # Some helpers + list_coins = lambda: minisafed.rpc.listcoins()["coins"] + sorted_coins = lambda: sorted(list_coins(), key=lambda c: c["outpoint"]) + wait_synced = lambda: wait_for( + lambda: minisafed.rpc.getinfo()["blockheight"] == bitcoind.rpc.getblockcount() + ) + + def reorg_shift(height, txs): + """Remine the chain from given height, shifting the txs by one block.""" + delta = bitcoind.rpc.getblockcount() - height + 1 + assert delta > 2 + h = bitcoind.rpc.getblockhash(initial_tip["height"]) + bitcoind.rpc.invalidateblock(h) + bitcoind.generate_block(1) + for tx in txs: + bitcoind.rpc.sendrawtransaction(tx) + bitcoind.generate_block(delta - 1, wait_for_mempool=len(txs)) + + # Create 3 coins and spend 2 of them. Keep the transactions in memory to + # rebroadcast them on reorgs. + txs = [] + for _ in range(3): + addr = minisafed.rpc.getnewaddress()["address"] + amount = 0.356 + txid = bitcoind.rpc.sendtoaddress(addr, amount) + txs.append(bitcoind.rpc.gettransaction(txid)["hex"]) + wait_for(lambda: len(list_coins()) == 3) + txs.append(spend_coins(minisafed, bitcoind, list_coins()[:2])) + bitcoind.generate_block(1, wait_for_mempool=4) + wait_synced() + + # Advance the blocktime by >2h in the future for the importdescriptors rescan + added_time = 60 * 60 * 3 + bitcoind.rpc.setmocktime(initial_tip["time"] + added_time) + bitcoind.generate_block(12) + + # Lose our state + coins_before = sorted_coins() + outpoints_before = set(c["outpoint"] for c in coins_before) + bitcoind.generate_block(1) + minisafed.restart_fresh(bitcoind) + assert len(list_coins()) == 0 + + # We can be stopped while we are rescanning + minisafed.rpc.startrescan(initial_tip["time"]) + minisafed.stop() + minisafed.start() + wait_for(lambda: minisafed.rpc.getinfo()["rescan_progress"] is None) + assert coins_before == sorted_coins() + + # Lose our state again + bitcoind.generate_block(1) + minisafed.restart_fresh(bitcoind) + wait_for(lambda: minisafed.rpc.getinfo()["rescan_progress"] is None) + assert len(list_coins()) == 0 + + # There can be a reorg when we start rescanning + reorg_shift(initial_tip["height"], txs) + minisafed.rpc.startrescan(initial_tip["time"]) + wait_synced() + wait_for(lambda: minisafed.rpc.getinfo()["rescan_progress"] is None) + assert len(sorted_coins()) == len(coins_before) + assert all(c["outpoint"] in outpoints_before for c in list_coins()) + + # Advance the blocktime again + bitcoind.rpc.setmocktime(initial_tip["time"] + added_time * 2) + bitcoind.generate_block(12) + + # Lose our state again + bitcoind.generate_block(1) + minisafed.restart_fresh(bitcoind) + wait_synced() + wait_for(lambda: minisafed.rpc.getinfo()["rescan_progress"] is None) + assert len(list_coins()) == 0 + + # We can be rescanning when a reorg happens + minisafed.rpc.startrescan(initial_tip["time"]) + reorg_shift(initial_tip["height"] + 1, txs) + wait_synced() + wait_for(lambda: minisafed.rpc.getinfo()["rescan_progress"] is None) + assert len(sorted_coins()) == len(coins_before) + assert all(c["outpoint"] in outpoints_before for c in list_coins()) diff --git a/tests/test_framework/bitcoind.py b/tests/test_framework/bitcoind.py index 0bb82136..fd1588bf 100644 --- a/tests/test_framework/bitcoind.py +++ b/tests/test_framework/bitcoind.py @@ -8,19 +8,19 @@ from test_framework.utils import TailableProc, wait_for, TIMEOUT, BITCOIND_PATH, class BitcoindRpcInterface: - def __init__(self, data_dir, network, rpc_port): + def __init__(self, data_dir, network, rpc_port, wallet=None): self.cookie_path = os.path.join(data_dir, network, ".cookie") self.rpc_port = rpc_port - self.wallet_name = "minisafed-tests" + self.wallet_name = wallet def __getattr__(self, name): assert not (name.startswith("__") and name.endswith("__")), "Python internals" with open(self.cookie_path) as fd: authpair = fd.read() - service_url = ( - f"http://{authpair}@localhost:{self.rpc_port}/wallet/{self.wallet_name}" - ) + service_url = f"http://{authpair}@localhost:{self.rpc_port}" + if self.wallet_name is not None: + service_url += f"/wallet/{self.wallet_name}" proxy = AuthServiceProxy(service_url, name) def f(*args): @@ -68,7 +68,12 @@ class Bitcoind(TailableProc): for k, v in bitcoind_conf.items(): f.write(f"{k}={v}\n") - self.rpc = BitcoindRpcInterface(bitcoin_dir, "regtest", rpcport) + # An RPC interface with our internal wallet, and an RPC interface with no + # wallet to be able to call 'unloadwallet' on any wallet. + self.rpc = BitcoindRpcInterface( + bitcoin_dir, "regtest", rpcport, wallet="minisafed-tests" + ) + self.node_rpc = BitcoindRpcInterface(bitcoin_dir, "regtest", rpcport) def start(self): TailableProc.start(self) diff --git a/tests/test_framework/minisafed.py b/tests/test_framework/minisafed.py index 085e07e6..8cfc137f 100644 --- a/tests/test_framework/minisafed.py +++ b/tests/test_framework/minisafed.py @@ -1,5 +1,6 @@ import logging import os +import shutil from bip32.utils import coincurve from bip380.descriptors import Descriptor @@ -10,6 +11,7 @@ from test_framework.utils import ( VERBOSE, LOG_LEVEL, MINISAFED_PATH, + wait_for, ) from test_framework.serializations import ( PSBT, @@ -34,6 +36,7 @@ class Minisafed(TailableProc): ): TailableProc.__init__(self, datadir, verbose=VERBOSE) + self.datadir = datadir self.prefix = os.path.split(datadir)[-1] self.owner_hd = owner_hd @@ -145,6 +148,18 @@ class Minisafed(TailableProc): return psbt + def restart_fresh(self, bitcoind): + """Delete the internal state of the wallet and restart.""" + self.stop() + dir_path = os.path.join(self.datadir, "regtest") + shutil.rmtree(dir_path) + wallet_path = os.path.join(dir_path, "minisafed_watchonly_wallet") + bitcoind.node_rpc.unloadwallet(wallet_path) + self.start() + wait_for( + lambda: self.rpc.getinfo()["blockheight"] == bitcoind.rpc.getblockcount() + ) + def start(self): TailableProc.start(self) self.wait_for_logs( diff --git a/tests/test_rpc.py b/tests/test_rpc.py index b8044aeb..2bc0aa73 100644 --- a/tests/test_rpc.py +++ b/tests/test_rpc.py @@ -1,4 +1,8 @@ +import os import pytest +import random +import shutil +import time from fixtures import * from test_framework.serializations import PSBT, PSBT_IN_PARTIAL_SIG @@ -12,6 +16,7 @@ def test_getinfo(minisafed): wait_for(lambda: res["blockheight"] == 101) assert res["sync"] == 1.0 assert "main" in res["descriptors"] + assert res["rescan_progress"] is None def test_getaddress(minisafed): @@ -253,3 +258,87 @@ def test_broadcast_spend(minisafed, bitcoind): # Now we've signed and stored it, the daemon will take care of finalizing # the PSBT before broadcasting the transaction. minisafed.rpc.broadcastspend(txid) + + +def test_start_rescan(minisafed, bitcoind): + """Test we successfully retrieve all our transactions after losing state by rescanning.""" + initial_timestamp = int(time.time()) + + # Some utility functions to DRY + list_coins = lambda: minisafed.rpc.listcoins()["coins"] + unspent_coins = lambda: ( + c for c in minisafed.rpc.listcoins()["coins"] if c["spend_info"] is None + ) + sorted_coins = lambda: sorted(list_coins(), key=lambda c: c["outpoint"]) + + def all_spent(coins): + unspent = set(c["outpoint"] for c in unspent_coins()) + for c in coins: + if c["outpoint"] in unspent: + return False + return True + + # We can rescan from one second before the tip timestamp, that's almost a no-op. + tip_timestamp = bitcoind.rpc.getblockheader(bitcoind.rpc.getbestblockhash())["time"] + minisafed.rpc.startrescan(tip_timestamp - 1) + wait_for(lambda: minisafed.rpc.getinfo()["rescan_progress"] is None) + # We can't rescan from an insane timestamp though. + with pytest.raises(RpcError, match="Insane timestamp.*"): + minisafed.rpc.startrescan(tip_timestamp) + assert minisafed.rpc.getinfo()["rescan_progress"] is None + future_timestamp = tip_timestamp + 60 * 60 + with pytest.raises(RpcError, match="Insane timestamp.*"): + minisafed.rpc.startrescan(future_timestamp) + assert minisafed.rpc.getinfo()["rescan_progress"] is None + prebitcoin_timestamp = 1231006505 - 1 + with pytest.raises(RpcError, match="Insane timestamp."): + minisafed.rpc.startrescan(prebitcoin_timestamp) + assert minisafed.rpc.getinfo()["rescan_progress"] is None + + # First, get some coins + for _ in range(10): + addr = minisafed.rpc.getnewaddress()["address"] + amount = random.randint(1, COIN * 10) / COIN + txid = bitcoind.rpc.sendtoaddress(addr, amount) + bitcoind.generate_block(random.randint(1, 10), wait_for_mempool=txid) + wait_for(lambda: len(list_coins()) == 10) + + # Then simulate some regular activity (spend and receive) + # TODO: instead of having randomness we should lay down all different cases (with or + # without change, single or multiple inputs, sending externally or to self). + for _ in range(5): + addr = minisafed.rpc.getnewaddress()["address"] + amount = random.randint(1, COIN * 10) / COIN + txid = bitcoind.rpc.sendtoaddress(addr, amount) + avail = list(unspent_coins()) + to_spend = random.sample(avail, random.randint(1, len(avail))) + spend_coins(minisafed, bitcoind, to_spend) + bitcoind.generate_block(random.randint(1, 5), wait_for_mempool=2) + wait_for(lambda: all_spent(to_spend)) + wait_for( + lambda: minisafed.rpc.getinfo()["blockheight"] == bitcoind.rpc.getblockcount() + ) + + # Move time forward one day as bitcoind will rescan the last 2 hours of block upon + # importing a descriptor. + now = int(time.time()) + added_time = 60 * 60 * 24 + bitcoind.rpc.setmocktime(now + added_time) + bitcoind.generate_block(10) + + # Now delete the wallet state. When starting up we'll re-create a fresh database + # and watchonly wallet. Those won't be aware of past coins for the configured + # descriptor. + coins_before = sorted_coins() + minisafed.restart_fresh(bitcoind) + assert len(list_coins()) == 0 + + # Once the rescan is done, we must have detected all previous transactions. + minisafed.rpc.startrescan(initial_timestamp) + rescan_progress = minisafed.rpc.getinfo()["rescan_progress"] + assert rescan_progress is None or 0 <= rescan_progress <= 1 + wait_for(lambda: minisafed.rpc.getinfo()["rescan_progress"] is None) + wait_for( + lambda: minisafed.rpc.getinfo()["blockheight"] == bitcoind.rpc.getblockcount() + ) + assert coins_before == sorted_coins()