Merge #749: Revert PRs #722 and #742: roll back to including all BIP32 derivations in the PSBT input

17ca01322e7badd37fd4120e0aeec8d834d0d6df Revert "Merge #722: Only include BIP32 derivations for a single spending path when creating PSBTs" (Antoine Poinsot)
6f5b053ea5af4737885a25573c82d627c299236e Revert "Merge #742: commands: don't add derivation paths for keys from different path but same signer" (Antoine Poinsot)

Pull request description:

  #722 pruned the BIP32 derivation paths in PSBT inputs for other spending paths than the one this PSBT was created for. This was to make it possible to use them with the Bitbox (see #722 OP for more details).

  These were merged without testing at all whether it would break signing with a Ledger Nano. Turns out it did. Our bad. Thanks to Pythcoiner for finding this bug.

  A follow-up PR will add a helper to prune the derivation separately so that it can only be applied to some signers and not all of them.

ACKs for top commit:
  edouardparis:
    utACK 17ca01322e7badd37fd4120e0aeec8d834d0d6df

Tree-SHA512: 99764fc5c68bc66e8415d72bb9963822ea05f8447c1ec71b50db992f3bf456370e00ae59055c939df1817c0031cdfbbfddbcdae9c448790fdcd67df10589b43d

doc/API.md

@@ -287,8 +287,8 @@ Create a transaction that sweeps all coins for which a timelocked recovery path
 currently available to a provided address with the provided feerate.
 
 The `timelock` parameter can be used to specify which recovery path to use. By default,
-we'll use the first recovery path available. The provided timelock must match the value of one
+we'll use the first recovery path available. If created for a later timelock a recovery
-of the recovery path.
+transaction may be satisfied using an earlier timelock but not the opposite.
 
 Due to the fact coins are generally received at different block heights, not all coins may be
 spendable through a single recovery path at the same time.
@@ -296,14 +296,6 @@ spendable through a single recovery path at the same time.
 This command will error if no such coins are available or the sum of their value is not enough to
 cover the requested feerate.
 
-Note that although a transaction created for a later available timelock can be satisfied using the
-keys for an earlier timelock, a PSBT created with this command will only be usable for a single
-recovery path. If the user creates a PSBT for recovery path "B" and then ends up willing to satisfy
-it using signatures for earlier recovery path "A", they will have to create a new recovery PSBT for
-path "A" using `createrecovery`. This is because we do not include the information necessary to sign
-using other recovery paths to workaround some signing devices' surprising behaviour. (See
-https://github.com/wizardsardine/liana/pull/706#issuecomment-1744705808 for more details.)
-
 #### Request
 
 | Field      | Type              | Description                                                                               |

src/commands/mod.rs

@@ -7,8 +7,7 @@ mod utils;
 use crate::{
     bitcoin::BitcoinInterface,
     database::{Coin, DatabaseInterface},
-    descriptors::{self, keys},
+    descriptors, DaemonControl, VERSION,
-    DaemonControl, VERSION,
 };
 
 pub use crate::database::{CoinStatus, LabelItem};
@@ -26,7 +25,7 @@ use std::{
 
 use miniscript::{
     bitcoin::{
-        self, address, bip32,
+        self, address,
         locktime::absolute,
         psbt::{Input as PsbtIn, Output as PsbtOut, PartiallySignedTransaction as Psbt},
     },
@@ -72,7 +71,6 @@ pub enum CommandError {
     InsaneRescanTimestamp(u32),
     /// An error that might occur in the racy rescan triggering logic.
     RescanTrigger(String),
-    UnknownRecoveryTimelock(u16),
     RecoveryNotAvailable,
 }
 
@@ -134,7 +132,6 @@ impl fmt::Display for CommandError {
             ),
             Self::InsaneRescanTimestamp(t) => write!(f, "Insane timestamp '{}'.", t),
             Self::RescanTrigger(s) => write!(f, "Error while starting rescan: '{}'", s),
-            Self::UnknownRecoveryTimelock(tl) => write!(f, "Provided timelock does not correspond to any recovery path: '{}'.", tl),
             Self::RecoveryNotAvailable => write!(
                 f,
                 "No coin currently spendable through this timelocked recovery path."
@@ -237,25 +234,6 @@ fn serializable_size<T: bitcoin::consensus::Encodable + ?Sized>(t: &T) -> u64 {
     bitcoin::consensus::serialize(t).len().try_into().unwrap()
 }
 
-// Whether a given key was derived from the keys of a specific spending path.
-fn is_key_for_spending_path(
-    pubkey: &descriptors::keys::DerivedPublicKey,
-    path_origins: &HashMap<bip32::Fingerprint, HashSet<bip32::DerivationPath>>,
-) -> bool {
-    // If it comes from a signer used in this path
-    if let Some(der_paths) = path_origins.get(&pubkey.origin.0) {
-        // Get the derivation path of the parent used to derive this key. Note this is fine
-        // to only drop the last derivation step, because the keys in the policy are
-        // normalized (so the derivation path up to the wildcard is part of the origin).
-        if let Some((_, der_path_no_wildcard)) = pubkey.origin.1[..].split_last() {
-            // Now make sure it was actually derived from the xpub used in this derivation
-            // path, and not from the same signer used in another path.
-            return der_paths.contains(&(der_path_no_wildcard.into()));
-        }
-    }
-    false
-}
-
 impl DaemonControl {
     // Get the derived descriptor for this coin
     fn derived_desc(&self, coin: &Coin) -> descriptors::DerivedSinglePathLianaDesc {
@@ -374,23 +352,6 @@ impl DaemonControl {
         }
         let mut db_conn = self.db.connection();
 
-        // Some signing devices (such as the Bitbox02) would not sign for all keys present in a
-        // script. This can lead to disruptions. For instance it could not provide a signature for
-        // the same key across a PSBT inputs. (See
-        // https://github.com/wizardsardine/liana/pull/706#issuecomment-1744705808 for details.)
-        // Therefore, only include in the PSBT_IN_BIP32_DERIVATION field the keys for the spending
-        // path we are interested in. Here the primary path. This guides the signer to sign for
-        // what we expect, and as long as the same signer isn't reused within the same spending
-        // path the signatures it will return will be consistent across inputs.
-        let (_, prim_path_origins) = self
-            .config
-            .main_descriptor
-            .policy()
-            .primary_path()
-            .thresh_origins();
-        let is_prim_path_key =
-            |pubkey: &keys::DerivedPublicKey| is_key_for_spending_path(pubkey, &prim_path_origins);
-
         // Iterate through given outpoints to fetch the coins (hence checking their existence
         // at the same time). We checked there is at least one, therefore after this loop the
         // list of coins is not empty.
@@ -439,7 +400,7 @@ impl DaemonControl {
                 script_pubkey: coin_desc.script_pubkey(),
             });
             let non_witness_utxo = spent_txs.get(op).cloned();
-            let bip32_derivation = coin_desc.bip32_derivations(is_prim_path_key);
+            let bip32_derivation = coin_desc.bip32_derivations();
             psbt_ins.push(PsbtIn {
                 witness_script,
                 witness_utxo,
@@ -474,9 +435,7 @@ impl DaemonControl {
                     } else {
                         self.config.main_descriptor.receive_descriptor()
                     };
-                    // NOTE: include all derivations for change outputs, to make sure signers are
+                    desc.derive(index, &self.secp).bip32_derivations()
-                    // able to re-compute change addresses.
-                    desc.derive(index, &self.secp).bip32_derivations(|_| true)
                 } else {
                     Default::default()
                 };
@@ -554,10 +513,8 @@ impl DaemonControl {
                     // TODO: shuffle once we have Taproot
                     change_txo.value = change_amount.to_sat();
                     tx.output.push(change_txo);
-                    // NOTE: include all derivations for change outputs, to make sure signers are
-                    // able to re-compute change addresses.
                     psbt_outs.push(PsbtOut {
-                        bip32_derivation: change_desc.bip32_derivations(|_| true),
+                        bip32_derivation: change_desc.bip32_derivations(),
                         ..PsbtOut::default()
                     });
                 } else if is_self_send {
@@ -816,30 +773,6 @@ impl DaemonControl {
                     .unwrap_or(false)
             });
 
-        // Some signing devices (such as the Bitbox02) would not sign for all keys present in a
-        // script. This can lead to disruptions. For instance it could not provide a signature for
-        // the same key across a PSBT inputs. (See
-        // https://github.com/wizardsardine/liana/pull/706#issuecomment-1744705808 for details.)
-        // Therefore, only include in the PSBT_IN_BIP32_DERIVATION field the keys for the requested
-        // recovery path. This guides the signer toward signing only for this path, and as long as
-        // the same signer isn't reused within the same spending path the signatures it will return
-        // will be consistent across inputs.
-        // Note it's particularly unfortunate for a recovery PSBT. This is because absent this
-        // restriction a (set of) user(s) could satisfy any previously available paths (the primary
-        // path and any timelocked path which necessitates a lower sequence) to finalize a PSBT.
-        // Now they can only create and pass around a PSBT that can be used to sign for a single
-        // recovery path.
-        let (_, reco_path_origins) = self
-            .config
-            .main_descriptor
-            .policy()
-            .recovery_paths()
-            .get(&timelock)
-            .ok_or(CommandError::UnknownRecoveryTimelock(timelock))?
-            .thresh_origins();
-        let is_key_for_this_path =
-            |pubkey: &keys::DerivedPublicKey| is_key_for_spending_path(pubkey, &reco_path_origins);
-
         // Fill-in the transaction inputs and PSBT inputs information. Record the value
         // that is fed to the transaction while doing so, to compute the fees afterward.
         let mut in_value = bitcoin::Amount::from_sat(0);
@@ -872,7 +805,7 @@ impl DaemonControl {
                 script_pubkey: coin_desc.script_pubkey(),
             });
             let non_witness_utxo = spent_txs.get(&coin.outpoint).cloned();
-            let bip32_derivation = coin_desc.bip32_derivations(is_key_for_this_path);
+            let bip32_derivation = coin_desc.bip32_derivations();
             psbt.inputs.push(PsbtIn {
                 witness_script,
                 witness_utxo,

src/descriptors/mod.rs

@@ -378,11 +378,7 @@ impl DerivedSinglePathLianaDesc {
         self.0.explicit_script().expect("Not a Taproot descriptor")
     }
 
-    /// Get the BIP32 derivations for this derived descriptor, filtered by the given predicate.
+    pub fn bip32_derivations(&self) -> Bip32Deriv {
-    pub fn bip32_derivations<P>(&self, mut predicate: P) -> Bip32Deriv
-    where
-        P: FnMut(&DerivedPublicKey) -> bool,
-    {
         let ms = match self.0 {
             descriptor::Descriptor::Wsh(ref wsh) => match wsh.as_inner() {
                 descriptor::WshInner::Ms(ms) => ms,
@@ -395,13 +391,7 @@ impl DerivedSinglePathLianaDesc {
 
         // For DerivedPublicKey, Pk::Hash == Self.
         ms.iter_pk()
-            .filter_map(|k| {
+            .map(|k| (k.key.inner, (k.origin.0, k.origin.1)))
-                if predicate(&k) {
-                    Some((k.key.inner, (k.origin.0, k.origin.1)))
-                } else {
-                    None
-                }
-            })
             .collect()
     }
 }
@@ -688,7 +678,7 @@ mod tests {
         // Sanity check we can call the methods on the derived desc
         der_desc.script_pubkey();
         der_desc.witness_script();
-        assert!(!der_desc.bip32_derivations(|_| true).is_empty());
+        assert!(!der_desc.bip32_derivations().is_empty());
     }
 
     #[test]

src/jsonrpc/mod.rs

@@ -164,7 +164,6 @@ impl From<commands::CommandError> for Error {
             | commands::CommandError::SpendFinalization(..)
             | commands::CommandError::InsaneRescanTimestamp(..)
             | commands::CommandError::AlreadyRescanning
-            | commands::CommandError::UnknownRecoveryTimelock(..)
             | commands::CommandError::RecoveryNotAvailable => {
                 Error::new(ErrorCode::InvalidParams, e.to_string())
             }

src/signer.rs

@@ -455,7 +455,7 @@ mod tests {
             unknown: BTreeMap::new(),
             inputs: vec![PsbtIn {
                 witness_script: Some(spent_coin_desc.witness_script()),
-                bip32_derivation: spent_coin_desc.bip32_derivations(|_| true),
+                bip32_derivation: spent_coin_desc.bip32_derivations(),
                 witness_utxo: Some(bitcoin::TxOut {
                     value: 19_000,
                     script_pubkey: spent_coin_desc.script_pubkey(),
@@ -497,7 +497,7 @@ mod tests {
         let other_spent_coin_desc = desc.receive_descriptor().derive(84.into(), &secp);
         dummy_psbt.inputs.push(PsbtIn {
             witness_script: Some(other_spent_coin_desc.witness_script()),
-            bip32_derivation: other_spent_coin_desc.bip32_derivations(|_| true),
+            bip32_derivation: other_spent_coin_desc.bip32_derivations(),
             witness_utxo: Some(bitcoin::TxOut {
                 value: 19_000,
                 script_pubkey: other_spent_coin_desc.script_pubkey(),

tests/fixtures.py

@@ -156,41 +156,6 @@ def lianad(bitcoind, directory):
     lianad.cleanup()
 
 
-@pytest.fixture
-def lianad_same_signer(bitcoind, directory):
-    """A simple lianad but the same signer is used for primary and recovery."""
-    datadir = os.path.join(directory, "lianad")
-    os.makedirs(datadir, exist_ok=True)
-    bitcoind_cookie = os.path.join(bitcoind.bitcoin_dir, "regtest", ".cookie")
-
-    # We use the same xpub for both paths, but /0/<0;1>/* for the primary and /1/<0;1>/*
-    # for the recovery.
-    signer = SingleSigner()
-    fingerprint = xpub_fingerprint(signer.primary_hd)
-    xpub = signer.primary_hd.get_xpub()
-    csv_value = 10
-    main_desc = Descriptor.from_str(
-        f"wsh(or_d(pk([{fingerprint}]{xpub}/0/<0;1>/*),and_v(v:pkh([{fingerprint}]{xpub}/1/<0;1>/*),older({csv_value}))))"
-    )
-
-    lianad = Lianad(
-        datadir,
-        signer,
-        main_desc,
-        bitcoind.rpcport,
-        bitcoind_cookie,
-    )
-
-    try:
-        lianad.start()
-        yield lianad
-    except Exception:
-        lianad.cleanup()
-        raise
-
-    lianad.cleanup()
-
-
 def multi_expression(thresh, keys):
     exp = f"multi({thresh},"
     for i, key in enumerate(keys):

tests/test_framework/lianad.py

@@ -79,14 +79,14 @@ class Lianad(TailableProc):
                 int.from_bytes(raw_der_path[i : i + 4], byteorder="little", signed=True)
                 for i in range(0, len(raw_der_path), 4)
             ]
+            assert len(der_path) == 2
 
             # Create a copy of the descriptor to derive it at the index used in this input.
             # Then create a satisfaction for it using the signature we just created.
-            is_change = der_path[len(der_path) - 2] == 1
             desc = Descriptor.from_str(
-                str(self.change_desc if is_change else self.receive_desc)
+                str(self.receive_desc if der_path[0] == 0 else self.change_desc)
             )
-            desc.derive(der_path[len(der_path) - 1])
+            desc.derive(der_path[1])
             sat_material = SatisfactionMaterial(
                 signatures=psbt_in.map[PSBT_IN_PARTIAL_SIG],
             )

tests/test_rpc.py

@@ -6,7 +6,6 @@ import time
 from fixtures import *
 from test_framework.serializations import (
     PSBT,
-    PSBT_IN_BIP32_DERIVATION,
     PSBT_IN_PARTIAL_SIG,
     PSBT_IN_NON_WITNESS_UTXO,
 )
@@ -386,35 +385,6 @@ def test_create_spend(lianad, bitcoind):
     with pytest.raises(RpcError, match=".*is from an immature coinbase transaction."):
         lianad.rpc.createspend(destinations, [imma_coin["outpoint"]], 1)
 
-    # Receive a coin and make it immediately available for the recovery path.
-    txid = bitcoind.rpc.sendtoaddress(lianad.rpc.getnewaddress()["address"], 1)
-    bitcoind.generate_block(10, wait_for_mempool=txid)
-    wait_for(
-        lambda: lianad.rpc.getinfo()["block_height"] == bitcoind.rpc.getblockcount()
-    )
-
-    # Create both a spend transaction and recovery transaction spending this coin.
-    outpoints = [c["outpoint"] for c in lianad.rpc.listcoins(["confirmed"])["coins"]]
-    res_spend = lianad.rpc.createspend(destinations, outpoints, 1)
-    res_reco = lianad.rpc.createrecovery(bitcoind.rpc.getnewaddress(), 2)
-
-    # The two PSBTs don't share any BIP32 derivation paths in their inputs.
-    res_spend_psbt = PSBT.from_base64(res_spend["psbt"])
-    res_spend_keys = set()
-    for i in res_spend_psbt.i:
-        res_spend_keys = res_spend_keys | set(i.map[PSBT_IN_BIP32_DERIVATION])
-    res_reco_psbt = PSBT.from_base64(res_reco["psbt"])
-    res_reco_keys = set()
-    for i in res_reco_psbt.i:
-        res_reco_keys = res_reco_keys | set(i.map[PSBT_IN_BIP32_DERIVATION])
-    assert res_spend_keys.intersection(res_reco_keys) == set()
-
-
-def test_create_spend_duplicate_signer(lianad_same_signer, bitcoind):
-    """Test spend creation (esp. around what bip32 derivations are added to the PSBT) but
-    with a Liana setup where a single signer is repeated between paths."""
-    test_create_spend(lianad_same_signer, bitcoind)
-
 
 def test_list_spend(lianad, bitcoind):
     # Start by creating two conflicting Spend PSBTs. The first one will have a change
@@ -871,39 +841,6 @@ def test_create_recovery(lianad, bitcoind):
     sign_and_broadcast(lianad, bitcoind, reco_psbt, recovery=True)
 
 
-def test_create_recovery_specific_paths(lianad_multipath, bitcoind):
-    """Test creating recovery PSBTs for specific recovery paths."""
-    # We can't create a recovery for a specific recovery path without specifying the precise
-    # timelock value of this recovery path.
-    with pytest.raises(
-        RpcError,
-        match="Provided timelock does not correspond to any recovery path: '42424'",
-    ):
-        lianad_multipath.rpc.createrecovery(bitcoind.rpc.getnewaddress(), 2, 42424)
-
-    # Receive a coin and make it immediately available for both reco paths.
-    txid = bitcoind.rpc.sendtoaddress(
-        lianad_multipath.rpc.getnewaddress()["address"], 1
-    )
-    bitcoind.generate_block(20, wait_for_mempool=txid)
-    wait_for(
-        lambda: lianad_multipath.rpc.getinfo()["block_height"]
-        == bitcoind.rpc.getblockcount()
-    )
-
-    # But we can create one for both existing recovery paths.
-    res_10 = lianad_multipath.rpc.createrecovery(bitcoind.rpc.getnewaddress(), 2, 10)
-    res_20 = lianad_multipath.rpc.createrecovery(bitcoind.rpc.getnewaddress(), 2, 20)
-
-    # Both don't have the same BIP32 derivations set in their input, unfortunately. This
-    # is because we only set them for the keys from a specific spending path.
-    res_10_psbt = PSBT.from_base64(res_10["psbt"])
-    res_20_psbt = PSBT.from_base64(res_20["psbt"])
-    res_10_keys = set(res_10_psbt.i[0].map[PSBT_IN_BIP32_DERIVATION])
-    res_20_keys = set(res_20_psbt.i[0].map[PSBT_IN_BIP32_DERIVATION])
-    assert res_10_keys.intersection(res_20_keys) == set()
-
-
 def test_labels(lianad, bitcoind):
     """Test the creation and updating of labels."""
     # We can set a label for an address.