From b14bc602d45cac8191059d8ed6a5391ca7cfc68f Mon Sep 17 00:00:00 2001 From: Antoine Poinsot Date: Mon, 17 Oct 2022 15:48:12 +0200 Subject: [PATCH 1/5] bitcoin: interface for broadcasting a transaction --- src/bitcoin/d/mod.rs | 8 ++++++++ src/bitcoin/mod.rs | 41 +++++++++++++++++++++++++++++++++++++++-- src/testutils.rs | 6 +++++- 3 files changed, 52 insertions(+), 3 deletions(-) diff --git a/src/bitcoin/d/mod.rs b/src/bitcoin/d/mod.rs index 2bfe3a5e..c3db160e 100644 --- a/src/bitcoin/d/mod.rs +++ b/src/bitcoin/d/mod.rs @@ -679,6 +679,14 @@ impl BitcoinD { blockhash, } } + + pub fn broadcast_tx(&self, tx: &bitcoin::Transaction) -> Result<(), BitcoindError> { + self.make_fallible_node_request( + "sendrawtransaction", + ¶ms!(bitcoin::consensus::encode::serialize_hex(tx)), + )?; + Ok(()) + } } // Bitcoind uses a guess for the value of verificationprogress. It will eventually get to // be 1, and we want to be less conservative. diff --git a/src/bitcoin/mod.rs b/src/bitcoin/mod.rs index 5f9a7289..ec7cf141 100644 --- a/src/bitcoin/mod.rs +++ b/src/bitcoin/mod.rs @@ -4,12 +4,30 @@ pub mod d; pub mod poller; -use d::LSBlockEntry; +use d::{BitcoindError, LSBlockEntry}; -use std::{collections::HashMap, fmt, sync}; +use std::{collections::HashMap, error, fmt, sync}; use miniscript::bitcoin; +/// Error occuring when querying our Bitcoin backend. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum BitcoinError { + Broadcast(String), +} + +impl fmt::Display for BitcoinError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + BitcoinError::Broadcast(reason) => { + write!(f, "Failed to broadcast transaction: '{}'", reason) + } + } + } +} + +impl error::Error for BitcoinError {} + /// Information about the best block in the chain #[derive(Debug, Clone, Eq, PartialEq, Copy)] pub struct BlockChainTip { @@ -60,6 +78,9 @@ pub trait BitcoinInterface: Send { /// Get the common ancestor between the Bitcoin backend's tip and the given tip. fn common_ancestor(&self, tip: &BlockChainTip) -> BlockChainTip; + + /// Broadcast this transaction to the Bitcoin P2P network + fn broadcast_tx(&self, tx: &bitcoin::Transaction) -> Result<(), BitcoinError>; } impl BitcoinInterface for d::BitcoinD { @@ -234,6 +255,18 @@ impl BitcoinInterface for d::BitcoinD { ancestor } + + fn broadcast_tx(&self, tx: &bitcoin::Transaction) -> Result<(), BitcoinError> { + match self.broadcast_tx(tx) { + Ok(()) => Ok(()), + Err(BitcoindError::Server(e)) => Err(BitcoinError::Broadcast(e.to_string())), + // We assume the Bitcoin backend doesn't fail, so it must be a JSONRPC error. + Err(e) => panic!( + "Unexpected Bitcoin error when broadcast transaction: '{}'.", + e + ), + } + } } // FIXME: do we need to repeat the entire trait implemenation? Isn't there a nicer way? @@ -282,6 +315,10 @@ impl BitcoinInterface for sync::Arc> fn common_ancestor(&self, tip: &BlockChainTip) -> BlockChainTip { self.lock().unwrap().common_ancestor(tip) } + + fn broadcast_tx(&self, tx: &bitcoin::Transaction) -> Result<(), BitcoinError> { + self.lock().unwrap().broadcast_tx(tx) + } } // FIXME: We could avoid this type (and all the conversions entailing allocations) if bitcoind diff --git a/src/testutils.rs b/src/testutils.rs index 523dc3de..3f5b1e72 100644 --- a/src/testutils.rs +++ b/src/testutils.rs @@ -1,5 +1,5 @@ use crate::{ - bitcoin::{BitcoinInterface, BlockChainTip, UTxO}, + bitcoin::{BitcoinError, BitcoinInterface, BlockChainTip, UTxO}, config::{BitcoinConfig, Config}, database::{Coin, DatabaseConnection, DatabaseInterface, SpendBlock}, DaemonHandle, @@ -66,6 +66,10 @@ impl BitcoinInterface for DummyBitcoind { fn common_ancestor(&self, _: &BlockChainTip) -> BlockChainTip { todo!() } + + fn broadcast_tx(&self, _: &bitcoin::Transaction) -> Result<(), BitcoinError> { + todo!() + } } pub struct DummyDb { From 37ee93a1e6ddd5089c1cc3d2cfa6f263b691cf32 Mon Sep 17 00:00:00 2001 From: Antoine Poinsot Date: Mon, 17 Oct 2022 15:49:11 +0200 Subject: [PATCH 2/5] commands: a new command for broadcasting a Spend transaction --- src/commands/mod.rs | 33 ++++++++++++++++++++++++++++++++- src/jsonrpc/mod.rs | 11 ++++++++++- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/src/commands/mod.rs b/src/commands/mod.rs index a52e85f4..040382d0 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -5,7 +5,7 @@ mod utils; use crate::{ - bitcoin::BitcoinInterface, + bitcoin::{BitcoinError, BitcoinInterface}, database::{Coin, DatabaseInterface}, descriptors, DaemonControl, VERSION, }; @@ -50,6 +50,10 @@ pub enum CommandError { /* target feerate */ u64, ), SanityCheckFailure(Psbt), + UnknownSpend(bitcoin::Txid), + // FIXME: when upgrading Miniscript put the actual error there + SpendFinalization(String), + TxBroadcast(String), } impl fmt::Display for CommandError { @@ -71,6 +75,11 @@ impl fmt::Display for CommandError { "BUG! Please report this. Failed sanity checks for PSBT '{:?}'.", psbt ), + Self::UnknownSpend(txid) => write!(f, "Unknown spend transaction '{}'.", txid), + Self::SpendFinalization(e) => { + write!(f, "Failed to finalize the spend transaction PSBT: '{}'.", e) + } + Self::TxBroadcast(e) => write!(f, "Failed to broadcast transaction: '{}'.", e), } } } @@ -446,6 +455,28 @@ impl DaemonControl { let mut db_conn = self.db.connection(); db_conn.delete_spend(txid); } + + /// Finalize and broadcast this stored Spend transaction. + pub fn broadcast_spend(&self, txid: &bitcoin::Txid) -> Result<(), CommandError> { + let mut db_conn = self.db.connection(); + + // First, try to finalize the spending transaction with the elements contained + // in the PSBT. + let mut spend_psbt = db_conn + .spend_tx(txid) + .ok_or(CommandError::UnknownSpend(*txid))?; + log::debug!("B"); + miniscript::psbt::finalize(&mut spend_psbt, &self.secp) + .map_err(|e| CommandError::SpendFinalization(e.to_string()))?; + + // Then, broadcast it (or try to, we never know if we are not going to hit an + // error at broadcast time). + let final_tx = spend_psbt.extract_tx(); + match self.bitcoin.broadcast_tx(&final_tx) { + Ok(()) => Ok(()), + Err(BitcoinError::Broadcast(e)) => Err(CommandError::TxBroadcast(e)), + } + } } #[derive(Debug, Clone, Serialize, Deserialize)] diff --git a/src/jsonrpc/mod.rs b/src/jsonrpc/mod.rs index 7bde56f0..db721891 100644 --- a/src/jsonrpc/mod.rs +++ b/src/jsonrpc/mod.rs @@ -51,6 +51,9 @@ pub struct Request { pub id: ReqId, } +/// A failure to broadcast a transaction to the P2P network. +const BROADCAST_ERROR: i64 = 1_000; + /// JSONRPC2 error codes. See https://www.jsonrpc.org/specification#error_object. #[derive(Debug, PartialEq, Eq, Clone)] pub enum ErrorCode { @@ -80,6 +83,7 @@ impl From for ErrorCode { match code { -32601 => ErrorCode::MethodNotFound, -32602 => ErrorCode::InvalidParams, + -32603 => ErrorCode::InternalError, code => ErrorCode::ServerError(code), } } @@ -153,12 +157,17 @@ impl From for Error { | commands::CommandError::InvalidFeerate(..) | commands::CommandError::AlreadySpent(..) | commands::CommandError::InvalidOutputValue(..) - | commands::CommandError::InsufficientFunds(..) => { + | commands::CommandError::InsufficientFunds(..) + | commands::CommandError::UnknownSpend(..) + | commands::CommandError::SpendFinalization(..) => { Error::new(ErrorCode::InvalidParams, e.to_string()) } commands::CommandError::SanityCheckFailure(_) => { Error::new(ErrorCode::InternalError, e.to_string()) } + commands::CommandError::TxBroadcast(_) => { + Error::new(ErrorCode::ServerError(BROADCAST_ERROR), e.to_string()) + } } } } From eff39ee35a4b4d9133586ab5200689da40ef254f Mon Sep 17 00:00:00 2001 From: Antoine Poinsot Date: Mon, 17 Oct 2022 15:49:40 +0200 Subject: [PATCH 3/5] rpc: a new 'broadcastspend' command --- doc/API.md | 17 +++++++++++++ src/jsonrpc/api.rs | 18 ++++++++++++++ tests/test_framework/minisafed.py | 31 +++++++++++++++++------- tests/test_framework/utils.py | 5 +++- tests/test_rpc.py | 40 ++++++++++++++++++++++++++++--- 5 files changed, 99 insertions(+), 12 deletions(-) diff --git a/doc/API.md b/doc/API.md index a9e089bc..75b18a4a 100644 --- a/doc/API.md +++ b/doc/API.md @@ -12,6 +12,7 @@ Commands must be sent as valid JSONRPC 2.0 requests, ending with a `\n`. | [`getnewaddress`](#getnewaddress) | Get a new receiving address | | [`listspendtxs`](#listspendtxs) | List all stored Spend transactions | | [`delspendtx`](#delspendtx) | Delete a stored Spend transaction | +| [`broadcastspend`](#broadcastspend) | Finalize a stored Spend PSBT, and broadcast it | # Reference @@ -182,3 +183,19 @@ This command does not return anything for now. | Field | Type | Description | | -------------- | --------- | ---------------------------------------------------- | + + +### `broadcastspend` + +#### Request + +| Field | Type | Description | +| -------- | ------ | ------------------------------------------------------ | +| `txid` | string | Hex encoded txid of the Spend transaction to broadcast | + +#### Response + +This command does not return anything for now. + +| Field | Type | Description | +| -------------- | --------- | ---------------------------------------------------- | diff --git a/src/jsonrpc/api.rs b/src/jsonrpc/api.rs index 740ed36a..dd0e3a8a 100644 --- a/src/jsonrpc/api.rs +++ b/src/jsonrpc/api.rs @@ -72,9 +72,27 @@ fn delete_spend(control: &DaemonControl, params: Params) -> Result Result { + let txid = params + .get(0, "txid") + .ok_or_else(|| Error::invalid_params("Missing 'txid' parameter."))? + .as_str() + .and_then(|s| bitcoin::Txid::from_str(s).ok()) + .ok_or_else(|| Error::invalid_params("Invalid 'txid' parameter."))?; + control.broadcast_spend(&txid)?; + + Ok(serde_json::json!({})) +} + /// Handle an incoming JSONRPC2 request. pub fn handle_request(control: &DaemonControl, req: Request) -> Result { let result = match req.method.as_str() { + "broadcastspend" => { + let params = req + .params + .ok_or_else(|| Error::invalid_params("Missing 'txid' parameter."))?; + broadcast_spend(control, params)? + } "createspend" => { let params = req.params.ok_or_else(|| { Error::invalid_params( diff --git a/tests/test_framework/minisafed.py b/tests/test_framework/minisafed.py index 7568c118..014fc990 100644 --- a/tests/test_framework/minisafed.py +++ b/tests/test_framework/minisafed.py @@ -57,15 +57,14 @@ class Minisafed(TailableProc): def sign_psbt(self, psbt): """Sign a transaction using the owner's key. - This creates a valid witness for all inputs in the transaction using the - information contained in the PSBT. + This will fill the 'partial_sigs' field of all inputs. :param psbt: PSBT of the transaction to be signed. - :returns: the serialized valid transaction, as hex. + :returns: PSBT with a signature in each input for the owner's key. """ assert isinstance(psbt, PSBT) - # Create a witness for each input of the transaction. + # Sign each input. for i, psbt_in in enumerate(psbt.inputs): # First, gather the needed information from the PSBT input. # 'hd_keypaths' is of the form {pubkey: (fingerprint, derivation index)} @@ -82,13 +81,31 @@ class Minisafed(TailableProc): assert pubkey in psbt_in.hd_keypaths.keys() sig = privkey.sign(sighash, hasher=None) + b"\x01" logging.debug(f"Adding signature {sig.hex()} for pubkey {pubkey.hex()}") + psbt_in.partial_sigs[pubkey] = sig + + return psbt + + def finalize_psbt(self, psbt): + """Create a valid witness for all inputs in the PSBT. + This will fail if the PSBT input does not contain enough material. + + :param psbt: PSBT of the transaction to be finalized. + :returns: PSBT with finalized inputs. + """ + assert isinstance(psbt, PSBT) + + # Create a witness for each input of the transaction. + for i, psbt_in in enumerate(psbt.inputs): + # First, gather the needed information from the PSBT input. + # 'hd_keypaths' is of the form {pubkey: (fingerprint, derivation index)} + der_index = next(iter(psbt_in.hd_keypaths.values()))[1] # Create a copy of the descriptor to derive it at the index used in this input. # Then create a satisfaction for it using the signature we just created. desc = Descriptor.from_str(str(self.main_desc)) desc.derive(der_index) sat_material = SatisfactionMaterial( - signatures={pubkey: sig}, + signatures=psbt_in.partial_sigs, ) stack = desc.satisfy(sat_material) logging.debug(f"Satisfaction for {desc} is {[e.hex() for e in stack]}") @@ -98,9 +115,7 @@ class Minisafed(TailableProc): psbt_in.final_script_witness = CTxInWitness(CScriptWitness(stack)) psbt.tx.wit.vtxinwit.append(psbt_in.final_script_witness) - tx = psbt.tx.serialize_with_witness().hex() - logging.debug(f"Final transaction: {tx}") - return tx + return psbt def start(self): TailableProc.start(self) diff --git a/tests/test_framework/utils.py b/tests/test_framework/utils.py index c48b0d58..0afbbd51 100644 --- a/tests/test_framework/utils.py +++ b/tests/test_framework/utils.py @@ -69,7 +69,9 @@ def spend_coins(minisafed, bitcoind, coins): psbt = PSBT() psbt.deserialize(res["psbt"]) - tx = minisafed.sign_psbt(psbt) + signed_psbt = minisafed.sign_psbt(psbt) + finalized_psbt = minisafed.finalize_psbt(signed_psbt) + tx = finalized_psbt.tx.serialize_with_witness().hex() bitcoind.rpc.sendrawtransaction(tx) return tx @@ -184,6 +186,7 @@ class UnixDomainSocketRpc(object): We might still want to define the actual methods in the subclasses for documentation purposes. """ + def wrapper(*args, **kwargs): if len(args) != 0 and len(kwargs) != 0: raise RpcError( diff --git a/tests/test_rpc.py b/tests/test_rpc.py index 44e0c06e..087a7ce4 100644 --- a/tests/test_rpc.py +++ b/tests/test_rpc.py @@ -1,6 +1,8 @@ +import pytest + from fixtures import * from test_framework.serializations import PSBT -from test_framework.utils import wait_for, COIN, get_txid, spend_coins +from test_framework.utils import wait_for, COIN, RpcError, get_txid, spend_coins def test_getinfo(minisafed): @@ -108,8 +110,10 @@ def test_create_spend(minisafed, bitcoind): assert len(spend_psbt.tx.vout) == 4 # We can sign it and broadcast it. - signed_tx_hex = minisafed.sign_psbt(spend_psbt) - bitcoind.rpc.sendrawtransaction(signed_tx_hex) + signed_psbt = minisafed.sign_psbt(spend_psbt) + finalized_psbt = minisafed.finalize_psbt(signed_psbt) + tx = finalized_psbt.tx.serialize_with_witness().hex() + bitcoind.rpc.sendrawtransaction(tx) def test_list_spend(minisafed, bitcoind): @@ -229,3 +233,33 @@ def test_update_spend(minisafed, bitcoind): assert len(psbt_merged.inputs[0].partial_sigs) == 2 assert psbt_merged.inputs[0].partial_sigs[dummy_pk_a] == dummy_sig_a assert psbt_merged.inputs[0].partial_sigs[dummy_pk_b] == dummy_sig_b + + +def test_broadcast_spend(minisafed, bitcoind): + # Create a new coin and a spending tx for it. + addr = minisafed.rpc.getnewaddress()["address"] + bitcoind.rpc.sendtoaddress(addr, 0.2567) + wait_for(lambda: len(minisafed.rpc.listcoins()["coins"]) > 0) + outpoints = [c["outpoint"] for c in minisafed.rpc.listcoins()["coins"]] + destinations = { + bitcoind.rpc.getnewaddress(): 200_000, + } + res = minisafed.rpc.createspend(outpoints, destinations, 6) + psbt = PSBT() + psbt.deserialize(res["psbt"]) + txid = psbt.tx.txid().hex() + + # We can't broadcast an unknown Spend + with pytest.raises(RpcError, match="Unknown spend transaction.*"): + minisafed.rpc.broadcastspend(txid) + minisafed.rpc.updatespend(res["psbt"]) + + # We can't broadcast an unsigned transaction + with pytest.raises(RpcError, match="Failed to finalize the spend transaction.*"): + minisafed.rpc.broadcastspend(txid) + signed_psbt = minisafed.sign_psbt(psbt) + minisafed.rpc.updatespend(signed_psbt.serialize()) + + # Now we've signed and stored it, the daemon will take care of finalizing + # the PSBT before broadcasting the transaction. + minisafed.rpc.broadcastspend(txid) From af9f0aeaed0f5607db94348c136325cf5fe9e387 Mon Sep 17 00:00:00 2001 From: Antoine Poinsot Date: Wed, 19 Oct 2022 12:00:54 +0200 Subject: [PATCH 4/5] qa: replace our PSBT implementation with a tweaked version of Bitcoin Core's This replaces our existing implementation of PSBTs with a more straightforward one, adapted from the Bitcoin Core functional tests framework. This fixes a few flakes that occured because the previous implementation could produce invalid PSBTs. The Bitcoin Core implementation is pretty low level and was adapted to treat mappings as such (the value in the PSBTMap can itself be a mapping, like for partial signatures or BIP32 derivation paths). The rest of the diff is adapting the users of PSBT to use the new implementation and the clearly superior interface (yay!). --- tests/test_framework/minisafed.py | 34 +- tests/test_framework/serializations.py | 578 +++++++------------------ tests/test_framework/utils.py | 4 +- tests/test_rpc.py | 48 +- 4 files changed, 190 insertions(+), 474 deletions(-) diff --git a/tests/test_framework/minisafed.py b/tests/test_framework/minisafed.py index 014fc990..8600a7de 100644 --- a/tests/test_framework/minisafed.py +++ b/tests/test_framework/minisafed.py @@ -16,6 +16,10 @@ from test_framework.serializations import ( sighash_all_witness, CTxInWitness, CScriptWitness, + PSBT_IN_BIP32_DERIVATION, + PSBT_IN_WITNESS_SCRIPT, + PSBT_IN_PARTIAL_SIG, + PSBT_IN_FINAL_SCRIPTWITNESS, ) @@ -65,11 +69,12 @@ class Minisafed(TailableProc): assert isinstance(psbt, PSBT) # Sign each input. - for i, psbt_in in enumerate(psbt.inputs): + for i, psbt_in in enumerate(psbt.i): # First, gather the needed information from the PSBT input. - # 'hd_keypaths' is of the form {pubkey: (fingerprint, derivation index)} - der_index = next(iter(psbt_in.hd_keypaths.values()))[1] - script_code = psbt_in.witness_script + # 'hd_keypaths' is of the form {pubkey: (fingerprint (4 bytes), derivation index (4 bytes))} + fing_der = next(iter(psbt_in.map[PSBT_IN_BIP32_DERIVATION].values())) + der_index = int.from_bytes(fing_der[4:], byteorder="little", signed=True) + script_code = psbt_in.map[PSBT_IN_WITNESS_SCRIPT] # Now sign the transaction with the key of the "owner" (the participant that # can sign immediately without a timelock) @@ -78,10 +83,14 @@ class Minisafed(TailableProc): self.owner_hd.get_privkey_from_path([der_index]) ) pubkey = privkey.public_key.format() - assert pubkey in psbt_in.hd_keypaths.keys() + assert pubkey in psbt_in.map[PSBT_IN_BIP32_DERIVATION].keys(), ( + pubkey, + psbt_in.map[PSBT_IN_BIP32_DERIVATION].keys(), + ) sig = privkey.sign(sighash, hasher=None) + b"\x01" logging.debug(f"Adding signature {sig.hex()} for pubkey {pubkey.hex()}") - psbt_in.partial_sigs[pubkey] = sig + assert PSBT_IN_PARTIAL_SIG not in psbt_in.map + psbt_in.map[PSBT_IN_PARTIAL_SIG] = {pubkey: sig} return psbt @@ -95,25 +104,28 @@ class Minisafed(TailableProc): assert isinstance(psbt, PSBT) # Create a witness for each input of the transaction. - for i, psbt_in in enumerate(psbt.inputs): + for i, psbt_in in enumerate(psbt.i): # First, gather the needed information from the PSBT input. # 'hd_keypaths' is of the form {pubkey: (fingerprint, derivation index)} - der_index = next(iter(psbt_in.hd_keypaths.values()))[1] + fing_der = next(iter(psbt_in.map[PSBT_IN_BIP32_DERIVATION].values())) + der_index = int.from_bytes(fing_der[4:], byteorder="little", signed=True) # Create a copy of the descriptor to derive it at the index used in this input. # Then create a satisfaction for it using the signature we just created. desc = Descriptor.from_str(str(self.main_desc)) desc.derive(der_index) sat_material = SatisfactionMaterial( - signatures=psbt_in.partial_sigs, + signatures=psbt_in.map[PSBT_IN_PARTIAL_SIG], ) stack = desc.satisfy(sat_material) logging.debug(f"Satisfaction for {desc} is {[e.hex() for e in stack]}") # Update the transaction inside the PSBT directly. assert stack is not None - psbt_in.final_script_witness = CTxInWitness(CScriptWitness(stack)) - psbt.tx.wit.vtxinwit.append(psbt_in.final_script_witness) + psbt_in.map[PSBT_IN_FINAL_SCRIPTWITNESS] = CTxInWitness( + CScriptWitness(stack) + ) + psbt.tx.wit.vtxinwit.append(psbt_in.map[PSBT_IN_FINAL_SCRIPTWITNESS]) return psbt diff --git a/tests/test_framework/serializations.py b/tests/test_framework/serializations.py index c6961877..dec5d336 100644 --- a/tests/test_framework/serializations.py +++ b/tests/test_framework/serializations.py @@ -1,9 +1,13 @@ #!/usr/bin/env python3 -# Stolen from https://github.com/achow101/psbt-simple-signer/blob/5def3622a09f5bcb76ae79707f0790d050291474/serializations.py -# PSBT serialization was authored by Andrew Chow (achow101) +# +# Taken then adapted from: +# - Initially https://github.com/achow101/psbt-simple-signer/blob/5def3622a09f5bcb76ae79707f0790d050291474/serializations.py +# - Then from the October 2022 Bitcoin Core functional test for the new PSBTMap class +# # Copyright (c) 2010 ArtForz -- public domain half-a-node # Copyright (c) 2012 Jeff Garzik # Copyright (c) 2010-2016 The Bitcoin Core developers +# Copyright (c) 2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Bitcoin Object Python Serializations @@ -17,7 +21,7 @@ CTransaction,CTxIn, CTxOut, etc....: ser_*, deser_*: functions that handle serialization/deserialization """ -from io import BytesIO, BufferedReader +from io import BytesIO from codecs import encode import struct import binascii @@ -185,25 +189,20 @@ def bytes_to_hex_str(s): return binascii.hexlify(s) -# Deserialize from a hex string representation (eg from RPC) -def FromHex(obj, hex_string): - obj.deserialize(BytesIO(hex_str_to_bytes(hex_string))) +# like from_hex, but without the hex part +def from_binary(cls, stream): + """deserialize a binary stream (or bytes object) into an object""" + # handle bytes object by turning it into a stream + was_bytes = isinstance(stream, bytes) + if was_bytes: + stream = BytesIO(stream) + obj = cls() + obj.deserialize(stream) + if was_bytes: + assert len(stream.read()) == 0 return obj -# Convert a binary-serializable object to hex (eg for submission via RPC) -def ToHex(obj): - return bytes_to_hex_str(obj.serialize()) - - -def Base64ToHex(s): - return binascii.hexlify(base64.b64decode(s)) - - -def HexToBase64(s): - return base64.b64encode(binascii.unhexlify(s)) - - def ser_sig_der(r, s): sig = b"\x30" @@ -576,433 +575,147 @@ class CTransaction(object): ) -def DeserializeHDKeypath(f, key, hd_keypaths): - if len(key) != 34 and len(key) != 66: - raise IOError( - "Size of key was not the expected size for the type partial signature pubkey" - ) - pubkey = key[1:] - if pubkey in hd_keypaths: - raise IOError( - "Duplicate key, input partial signature for pubkey already provided" - ) +# global types +PSBT_GLOBAL_UNSIGNED_TX = 0x00 +PSBT_GLOBAL_XPUB = 0x01 +PSBT_GLOBAL_TX_VERSION = 0x02 +PSBT_GLOBAL_FALLBACK_LOCKTIME = 0x03 +PSBT_GLOBAL_INPUT_COUNT = 0x04 +PSBT_GLOBAL_OUTPUT_COUNT = 0x05 +PSBT_GLOBAL_TX_MODIFIABLE = 0x06 +PSBT_GLOBAL_VERSION = 0xFB +PSBT_GLOBAL_PROPRIETARY = 0xFC - value = deser_string(f) - hd_keypaths[pubkey] = struct.unpack("<" + "I" * (len(value) // 4), value) +# per-input types +PSBT_IN_NON_WITNESS_UTXO = 0x00 +PSBT_IN_WITNESS_UTXO = 0x01 +PSBT_IN_PARTIAL_SIG = 0x02 +PSBT_IN_SIGHASH_TYPE = 0x03 +PSBT_IN_REDEEM_SCRIPT = 0x04 +PSBT_IN_WITNESS_SCRIPT = 0x05 +PSBT_IN_BIP32_DERIVATION = 0x06 +PSBT_IN_FINAL_SCRIPTSIG = 0x07 +PSBT_IN_FINAL_SCRIPTWITNESS = 0x08 +PSBT_IN_POR_COMMITMENT = 0x09 +PSBT_IN_RIPEMD160 = 0x0A +PSBT_IN_SHA256 = 0x0B +PSBT_IN_HASH160 = 0x0C +PSBT_IN_HASH256 = 0x0D +PSBT_IN_PREVIOUS_TXID = 0x0E +PSBT_IN_OUTPUT_INDEX = 0x0F +PSBT_IN_SEQUENCE = 0x10 +PSBT_IN_REQUIRED_TIME_LOCKTIME = 0x11 +PSBT_IN_REQUIRED_HEIGHT_LOCKTIME = 0x12 +PSBT_IN_TAP_KEY_SIG = 0x13 +PSBT_IN_TAP_SCRIPT_SIG = 0x14 +PSBT_IN_TAP_LEAF_SCRIPT = 0x15 +PSBT_IN_TAP_BIP32_DERIVATION = 0x16 +PSBT_IN_TAP_INTERNAL_KEY = 0x17 +PSBT_IN_TAP_MERKLE_ROOT = 0x18 +PSBT_IN_PROPRIETARY = 0xFC + +# per-output types +PSBT_OUT_REDEEM_SCRIPT = 0x00 +PSBT_OUT_WITNESS_SCRIPT = 0x01 +PSBT_OUT_BIP32_DERIVATION = 0x02 +PSBT_OUT_AMOUNT = 0x03 +PSBT_OUT_SCRIPT = 0x04 +PSBT_OUT_TAP_INTERNAL_KEY = 0x05 +PSBT_OUT_TAP_TREE = 0x06 +PSBT_OUT_TAP_BIP32_DERIVATION = 0x07 +PSBT_OUT_PROPRIETARY = 0xFC -def SerializeHDKeypath(hd_keypaths, type): - r = b"" - for pubkey, path in hd_keypaths.items(): - r += ser_string(type + pubkey) - packed = struct.pack("<" + "I" * len(path), *path) - r += ser_string(packed) - return r +class PSBTMap: + """Class for serializing and deserializing PSBT maps""" + + def __init__(self, map=None): + self.map = map if map is not None else {} + + # NOTE: this implementation assumes that the keytype from bip174 is always 1 byte, + # as it detects mappings (like bip32 derivations, partial sigs, ..) based on this. + def deserialize(self, f): + m = {} + while True: + k = deser_string(f) + if len(k) == 0: + break + v = deser_string(f) + if len(k) == 1: + k = k[0] + assert k not in m + m[k] = v + else: + typ, k = k[0], k[1:] + if typ not in m: + m[typ] = {k: v} + else: + m[typ][k] = v + self.map = m + + def serialize(self): + m = b"" + for key_type in sorted(self.map): + psbt_val = self.map[key_type] + if isinstance(key_type, int) and 0 <= key_type and key_type <= 255: + key_type = bytes([key_type]) + if isinstance(psbt_val, dict): + for key_data, val_data in psbt_val.items(): + k = key_type + key_data + m += ser_compact_size(len(k)) + k + m += ser_compact_size(len(val_data)) + val_data + else: + m += ser_compact_size(len(key_type)) + key_type + m += ser_compact_size(len(psbt_val)) + psbt_val + m += b"\x00" + return m -class PartiallySignedInput: - def __init__(self): - self.non_witness_utxo = None - self.witness_utxo = None - self.partial_sigs = {} - self.sighash = 0 - self.redeem_script = b"" - self.witness_script = b"" - self.hd_keypaths = {} - self.final_script_sig = b"" - self.final_script_witness = CTxInWitness() - self.unknown = {} +class PSBT: + """Class for serializing and deserializing PSBTs""" - def set_null(self): - self.non_witness_utxo = None - self.witness_utxo = None - self.partial_sigs.clear() - self.sighash = 0 - self.redeem_script = b"" - self.witness_script = b"" - self.hd_keypaths.clear() - self.final_script_sig = b"" - self.final_script_witness = CTxInWitness() - self.unknown.clear() + def __init__(self, *, g=None, i=None, o=None): + self.g = g if g is not None else PSBTMap() + self.i = i if i is not None else [] + self.o = o if o is not None else [] + self.tx = None def deserialize(self, f): - while True: - # read the key - try: - key = deser_string(f) - except Exception: - break - - # Check for separator - if len(key) == 0: - break - - # First byte of key is the type - key_type = struct.unpack("b", bytearray([key[0]]))[0] - - if key_type == 0: - if self.non_witness_utxo: - raise IOError( - "Duplicate Key, input non witness utxo already provided" - ) - elif len(key) != 1: - raise IOError("non witness utxo key is more than one byte type") - self.non_witness_utxo = CTransaction() - value = BufferedReader(BytesIO(deser_string(f))) - self.non_witness_utxo.deserialize(value) - self.non_witness_utxo.rehash() - - elif key_type == 1: - if self.witness_utxo: - raise IOError("Duplicate Key, input witness utxo already provided") - elif len(key) != 1: - raise IOError("witness utxo key is more than one byte type") - self.witness_utxo = CTxOut() - value = BufferedReader(BytesIO(deser_string(f))) - self.witness_utxo.deserialize(value) - - elif key_type == 2: - if len(key) != 34 and len(key) != 66: - raise IOError( - "Size of key was not the expected size for the type partial signature pubkey" - ) - pubkey = key[1:] - if pubkey in self.partial_sigs: - raise IOError( - "Duplicate key, input partial signature for pubkey already provided" - ) - - sig = deser_string(f) - self.partial_sigs[pubkey] = sig - - elif key_type == 3: - if self.sighash > 0: - raise IOError("Duplicate key, input sighash type already provided") - elif len(key) != 1: - raise IOError("sighash key is more than one byte type") - value = deser_string(f) - self.sighash = struct.unpack(" 0: - r += ser_string(b"\x03") - r += ser_string(struct.pack(" 1: - raise IOError("Global unsigned tx key is more than one byte type") - - # read in value - value = BufferedReader(BytesIO(deser_string(f))) - self.tx.deserialize(value) - - # Make sure that all scriptSigs and scriptWitnesses are empty - for txin in self.tx.vin: - if len(txin.scriptSig) != 0 or not self.tx.wit.is_null(): - raise IOError( - "Unsigned tx does not have empty scriptSigs and scriptWitnesses" - ) - - else: - if key in self.unknown: - raise IOError( - "Duplicate key, key for unknown value already provided" - ) - value = deser_string(f) - self.unknown[key] = value - - # make sure that we got an unsigned tx - if self.tx.is_null(): - raise IOError("No unsigned trasaction was provided") - - # Read input data - for txin in self.tx.vin: - input = PartiallySignedInput() - input.deserialize(f) - self.inputs.append(input) - - if ( - input.non_witness_utxo - and input.non_witness_utxo.rehash() - and input.non_witness_utxo.sha256 != txin.prevout.sha256 - ): - raise IOError("Non-witness UTXO does not match outpoint hash") - - if len(self.inputs) != len(self.tx.vin): - raise IOError( - "Inputs provided does not match the number of inputs in transaction" - ) - - # Read output data - for txout in self.tx.vout: - output = PartiallySignedOutput() - output.deserialize(f) - self.outputs.append(output) - - if len(self.outputs) != len(self.tx.vout): - raise IOError( - "Outputs provided does not match the number of outputs in transaction" - ) - - if not self.is_sane(): - raise IOError("PSBT is not sane") - - def serialize(self): - r = b"" - - # magic bytes - r += b"psbt\xff" - - # unsigned tx flag - r += b"\x01\x00" - - # write serialized tx - tx = self.tx.serialize_with_witness() - r += ser_compact_size(len(tx)) - r += tx - - # separator - r += b"\x00" - - # unknowns - for key, value in self.unknown: - r += ser_string(key) - r += ser_string(value) - - # inputs - for input in self.inputs: - r += input.serialize() - - # outputs - for output in self.outputs: - r += output.serialize() - - # return hex string - return HexToBase64(binascii.hexlify(r)).decode() - - def is_sane(self): - for input in self.inputs: - if not input.is_sane(): - return False - return True + @classmethod + def from_base64(cls, b64psbt): + return from_binary(cls, base64.b64decode(b64psbt)) # Sighash serializations @@ -1034,13 +747,14 @@ def sighash_all_witness(script_code, psbt, i, acp=False): sighash_type = b"\x01\x00\x00\x00" if not acp else b"\x81\x00\x00\x00" # Make sighash preimage + prev_txo = from_binary(CTxOut, psbt.i[i].map[PSBT_IN_WITNESS_UTXO]) preimage = b"" preimage += struct.pack(" Date: Wed, 19 Oct 2022 12:05:31 +0200 Subject: [PATCH 5/5] qa: remove an unused variable in test_update_spend The new PSBT implementation brought it to light. --- tests/test_rpc.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/test_rpc.py b/tests/test_rpc.py index e3892dbb..935f5e2c 100644 --- a/tests/test_rpc.py +++ b/tests/test_rpc.py @@ -187,9 +187,6 @@ def test_update_spend(minisafed, bitcoind): assert len(list_res) == 1 assert list_res[0]["psbt"] == res["psbt"] - # Keep a copy for later. - psbt_no_sig = PSBT.from_base64(res["psbt"]) - # We can add a signature and update it psbt_sig_a = PSBT.from_base64(res["psbt"]) dummy_pk_a = bytes.fromhex(