Feature: Add taproot psbt fields (#1837)

* Adds BIP-371 Taproot bip32 derivation fields

Co-authored-by: moneymanolis <[email protected]>
Co-authored-by: Stepan Snigirev <[email protected]>

requirements.in

@@ -15,7 +15,7 @@ requests==2.26.0
 pysocks==1.7.1
 six==1.16.0
 stem==1.8.0
-embit==0.5.0
+embit==0.6.1
 psutil==5.9.0
 pyopenssl==20.0.1
 flask_wtf==0.15.1

requirements.txt

@@ -163,8 +163,8 @@ ecdsa==0.18.0 \
     # via
     #   bitbox02
     #   hwi
-embit==0.5.0 \
-    --hash=sha256:5644ae6ed07bb71bf7fb15daf7f5af73d889180e623f5ff1f35a20ad01f0405e
+embit==0.6.1 \
+    --hash=sha256:16a84c6668dc9ffc907594457a46f7142cee379646bc009a5a9b77b0d2cb4e12
     # via -r requirements.in
 flask==2.1.1 \
     --hash=sha256:8a4cf32d904cf5621db9f0c9fbcd7efabf3003f22a04e4d0ce790c7137ec5264 \
@@ -437,9 +437,9 @@ pytimeparse==1.1.8 \
     --hash=sha256:04b7be6cc8bd9f5647a6325444926c3ac34ee6bc7e69da4367ba282f076036bd \
     --hash=sha256:e86136477be924d7e670646a98561957e8ca7308d44841e21f5ddea757556a0a
     # via -r requirements.in
-pytz==2022.4 \
-    --hash=sha256:2c0784747071402c6e99f0bafdb7da0fa22645f06554c7ae06bf6358897e9c91 \
-    --hash=sha256:48ce799d83b6f8aab2020e369b627446696619e79645419610b9facd909b3174
+pytz==2022.5 \
+    --hash=sha256:335ab46900b1465e714b4fda4963d87363264eb662aab5e65da039c25f1f5b22 \
+    --hash=sha256:c4d88f472f54d615e9cd582a5004d1e5f624854a6a27a6211591c251f22a6914
     # via
     #   apscheduler
     #   babel
@@ -485,9 +485,9 @@ typing-extensions==3.10.0.2 \
     # via
     #   bitbox02
     #   hwi
-tzdata==2022.4 \
-    --hash=sha256:74da81ecf2b3887c94e53fc1d466d4362aaf8b26fc87cda18f22004544694583 \
-    --hash=sha256:ada9133fbd561e6ec3d1674d3fba50251636e918aa97bd59d63735bef5a513bb
+tzdata==2022.5 \
+    --hash=sha256:323161b22b7802fdc78f20ca5f6073639c64f1a7227c40cd3e19fd1d0ce6650a \
+    --hash=sha256:e15b2b3005e2546108af42a0eb4ccab4d9e225e2dfbf4f77aad50c70a4b1f3ab
     # via pytz-deprecation-shim
 tzlocal==4.2 \
     --hash=sha256:89885494684c929d9191c57aa27502afc87a579be5cdd3225c77c463ea043745 \
@@ -507,9 +507,9 @@ wtforms==3.0.1 \
     --hash=sha256:6b351bbb12dd58af57ffef05bc78425d08d1914e0fd68ee14143b7ade023c5bc \
     --hash=sha256:837f2f0e0ca79481b92884962b914eba4e72b7a2daaf1f939c890ed0124b834b
     # via flask-wtf
-zipp==3.8.1 \
-    --hash=sha256:05b45f1ee8f807d0cc928485ca40a07cb491cf092ff587c0df9cb1fd154848d2 \
-    --hash=sha256:47c40d7fe183a6f21403a199b3e4192cca5774656965b0a4988ad2f8feb5f009
+zipp==3.10.0 \
+    --hash=sha256:4fcb6f278987a6605757302a6e40e896257570d11c51628968ccb2a47e80c6c1 \
+    --hash=sha256:7a7262fd930bd3e36c50b9a64897aec3fafff3dfdeec9623ae22b40e93f99bb8
     # via importlib-metadata
 
 # WARNING: The following packages were not pinned, but pip requires them to be

src/cryptoadvance/specter/device.py

@@ -159,7 +159,7 @@ def has_key_types(self, wallet_type, network="main"):
                     return True
         elif wallet_type == "simple":
             for key_type in self.key_types(network):
-                if key_type in ["", "sh-wpkh", "wpkh"]:
+                if key_type in ["", "sh-wpkh", "wpkh", "tr"]:
                     return True
         return "" in self.key_types(network)
 

src/cryptoadvance/specter/devices/specter.py

@@ -103,9 +103,7 @@ def create_psbts(self, base64_psbt, wallet):
         base64_psbt = psbt.to_string()
         psbts = super().create_psbts(base64_psbt, wallet)
         # remove non-witness utxo if they are there to reduce QR code size
-        updated_psbt = wallet.fill_psbt(
-            base64_psbt, non_witness=False, xpubs=False, taproot_derivations=True
-        )
+        updated_psbt = wallet.fill_psbt(base64_psbt, non_witness=False, xpubs=False)
         try:
             qr_psbt = PSBT.from_string(updated_psbt)
         except:
@@ -141,12 +139,8 @@ def create_psbts(self, base64_psbt, wallet):
         psbts["qrcode"] = qr_psbt.to_string()
 
         # we can add xpubs to SD card, but non_witness can be too large for MCU
-        psbts["sdcard"] = wallet.fill_psbt(
-            base64_psbt, non_witness=False, xpubs=True, taproot_derivations=True
-        )
-        psbts["hwi"] = wallet.fill_psbt(
-            base64_psbt, non_witness=False, xpubs=True, taproot_derivations=True
-        )
+        psbts["sdcard"] = wallet.fill_psbt(base64_psbt, non_witness=False, xpubs=True)
+        psbts["hwi"] = wallet.fill_psbt(base64_psbt, non_witness=False, xpubs=True)
         return psbts
 
     def export_wallet(self, wallet):

src/cryptoadvance/specter/util/psbt.py

@@ -160,6 +160,18 @@ def to_dict(self) -> dict:
                 }
                 for pub, der in self.scope.bip32_derivations.items()
             ]
+        if self.scope.taproot_bip32_derivations:
+            obj["taproot_bip32_derivs"] = [
+                {
+                    "pubkey": pub.xonly().hex(),
+                    "master_fingerprint": der.fingerprint.hex(),
+                    "path": bip32.path_to_str(der.derivation),
+                    "leaf_hashes": [leaf.hex() for leaf in leafs],
+                }
+                for pub, (leafs, der) in self.scope.taproot_bip32_derivations.items()
+            ]
+        if self.scope.taproot_internal_key:
+            obj["taproot_internal_key"] = self.scope.taproot_internal_key.xonly().hex()
         return obj
 
 

src/cryptoadvance/specter/wallet.py

@@ -1642,8 +1642,9 @@ def createpsbt(
         readonly=False,  # fee estimation
         rbf=True,
         rbf_edit_mode=False,
-    ):
+    ) -> dict:
         """
+        Returns psbt as dictionary.
         fee_rate: in sat/B or BTC/kB. If set to 0 Bitcoin Core sets feeRate automatically.
         """
         if fee_rate != 0 and fee_rate < self.MIN_FEE_RATE:
@@ -1717,7 +1718,10 @@ def createpsbt(
             True,  # bip32-der
         )
 
-        b64psbt = r["psbt"]
+        # Always explicitly fill psbt with any missing fields
+        # TODO: Re-evaluate if this is necessary if user is running Bitcoin Core w/BIP-371 support
+        b64psbt = self.fill_psbt(r["psbt"])
+
         psbt = self.PSBTCls(
             b64psbt,
             self.descriptor,
@@ -1740,14 +1744,16 @@ def createpsbt(
                 True,  # bip32-der
             )
 
-            b64psbt = r["psbt"]
+            # Always explicitly fill psbt with any missing fields
+            # TODO: Re-evaluate if this is necessary if user is running Bitcoin Core w/BIP-371 support
+            b64psbt = self.fill_psbt(r["psbt"])
+
             psbt = self.PSBTCls(
                 b64psbt,
                 self.descriptor,
                 self.network,
                 devices=list(zip(self.keys, self._devices)),
             )
-
         if not readonly:
             self.save_pending_psbt(psbt)
         return psbt.to_dict()
@@ -1866,25 +1872,36 @@ def fill_psbt(
         b64psbt,
         non_witness: bool = True,
         xpubs: bool = True,
-        taproot_derivations: bool = False,
     ):
         psbt = self.PSBTCls.from_string(b64psbt)
 
         # Core doesn't fill derivations yet, so we do it ourselves
-        if taproot_derivations and self.is_taproot:
-
+        # Provide the BIP-371 `PSBT_IN_TAP_BIP32_DERIVATION` 0x16 field
+        if self.is_taproot:
             net = self.network
             for sc in psbt.inputs + psbt.outputs:
+                if sc.taproot_internal_key is not None:
+                    # psbt already has Taproot fields for this `InputScope`/`OutputScope`
+                    continue
                 addr = sc.script_pubkey.address(net)
                 info = self._addresses.get(addr)
                 if info and not info.is_external:
                     d = self.descriptor.derive(
                         info.index, branch_index=int(info.change)
                     )
                     for k in d.keys:
-                        sc.bip32_derivations[PublicKey.parse(k.sec())] = DerivationPath(
+                        # TODO: support keysigns from within the taptree (note: embit
+                        # must be updated first).
+                        leaf_hashes = []
+                        derivation = DerivationPath(
                             k.origin.fingerprint, k.origin.derivation
                         )
+                        pub = PublicKey.from_xonly(k.xonly())
+                        sc.taproot_bip32_derivations[pub] = (
+                            leaf_hashes,
+                            derivation,
+                        )
+                        sc.taproot_internal_key = pub
 
         if non_witness:
             for inp in psbt.inputs:

tests/test_util_psbt.py

@@ -0,0 +1,37 @@
+from cryptoadvance.specter.util.psbt import SpecterPSBT, Descriptor
+
+PSBT = "cHNidP8BAH0CAAAAAZ3WUGbo+qq+uhku8ZGlccpVnEUe7DpXc2WT8eFBOY5NAAAAAAD9////AoCWmAAAAAAAFgAUknv5/4QLPO9YhpvFXh8Yd1A2uiloP10FAAAAACJRIBTYMq2X8Wb48epOTevfvMFFs1Rywret7quv2PDmQ27QAAAAAAABASsA4fUFAAAAACJRIGGkhZUFZ4wSdPRYh8+NH8rT4OPZ96DG+4g40XzsrkswIRbrOsC4g4bRFEv7o2eV6PjsmXRITNcywDkjoKE3IXZEcBkAc8XaClYAAIABAACAAAAAgAAAAAAAAAAAARcg6zrAuIOG0RRL+6Nnlej47Jl0SEzXMsA5I6ChNyF2RHAAAAEFIPF/TUgvecmr7Omn2RaD3/WuEWxZkvyKAVX8FtRQMndaIQfxf01IL3nJq+zpp9kWg9/1rhFsWZL8igFV/BbUUDJ3WhkAc8XaClYAAIABAACAAAAAgAEAAAABAAAAAA=="
+DESC = "tr([73c5da0a/86h/1h/0h]tprv8h5RpVZ1VP6ZenvqJAuUYCenQqYgRjsAMjqnVY54FqQzk52jqP12mPHa77wXQm9WeJSRjDhT3N5RL2Ye93Z4kR6rWTNo25Tdq6UfopDczBZ/{0,1}/*)"
+
+
+def test_taproot_psbt_to_dict():
+    psbt = SpecterPSBT(PSBT, Descriptor.from_string(DESC), "regtest")
+    obj = psbt.to_dict()
+    assert obj["inputs"][0]["taproot_bip32_derivs"] == [
+        {
+            "pubkey": "eb3ac0b88386d1144bfba36795e8f8ec9974484cd732c03923a0a13721764470",
+            "master_fingerprint": "73c5da0a",
+            "path": "m/86h/1h/0h/0/0",
+            "leaf_hashes": [],
+        }
+    ]
+    assert (
+        obj["inputs"][0]["taproot_internal_key"]
+        == "eb3ac0b88386d1144bfba36795e8f8ec9974484cd732c03923a0a13721764470"
+    )
+
+    assert obj["outputs"][1]["taproot_bip32_derivs"] == [
+        {
+            "pubkey": "f17f4d482f79c9abece9a7d91683dff5ae116c5992fc8a0155fc16d45032775a",
+            "master_fingerprint": "73c5da0a",
+            "path": "m/86h/1h/0h/1/1",
+            "leaf_hashes": [],
+        }
+    ]
+    assert (
+        obj["outputs"][1]["taproot_internal_key"]
+        == "f17f4d482f79c9abece9a7d91683dff5ae116c5992fc8a0155fc16d45032775a"
+    )
+    assert obj["outputs"][1]["change"] == True
+    assert obj["outputs"][1]["is_mine"] == True
+    assert obj["inputs"][0]["is_mine"] == True

tests/test_wallet.py

@@ -37,46 +37,78 @@ def test_createpsbt(
         )
     unspents = wallet.rpc.listunspent()
     selected_coin = [{"txid": unspents[0]["txid"], "vout": unspents[0]["vout"]}]
-    # Spending it all
     psbt = wallet.createpsbt(
         [random_address],
-        [20],
-        True,
+        [19],
+        False,  # Important because otherwise there is no change output!
         0,
         1,
         selected_coins=selected_coin,  # Selecting only one UTXO since input ordering seems to also be random in Core.
     )
     assert len(psbt["tx"]["vin"]) == 1
     assert len(psbt["inputs"]) == 1
 
-    # Check the PSBT fields - inputs
-    # The first input seems to be last selected coin from selected_coins
+    # Input fields
     assert (
         psbt["inputs"][0]["bip32_derivs"][0]["pubkey"]
         == "0330955ab511845fb48fc5739da551875ed54fa1f2fdd4cf77f3473ce2cffb4c75"
     )
     assert psbt["inputs"][0]["bip32_derivs"][0]["path"] == "m/84h/1h/0h/0/1"
     assert psbt["inputs"][0]["bip32_derivs"][0]["master_fingerprint"] == "8c24a510"
 
-    # Check the PSBT fields - outputs
-    logger.info(f"the whole {psbt}")
-
-    logger.info(f"the outputs of the {psbt['outputs']}")
-    assert (
-        psbt["outputs"][0]["address"] == "bcrt1q7mlxxdna2e2ufzgalgp5zhtnndl7qddlxjy5eg"
-    )
-    assert psbt["outputs"][0]["change"] == False
-    # assert psbt["outputs"][0]["bip32_derivs"][0]["master_fingerprint"] == "1e9cf8a7"
+    # Output fields
+    for output in psbt["outputs"]:  # The ordering of the outputs is random
+        if output["change"] == False:
+            assert output["address"] == "bcrt1q7mlxxdna2e2ufzgalgp5zhtnndl7qddlxjy5eg"
+        else:
+            assert output["is_mine"] == True
+            assert (
+                output["bip32_derivs"][0]["pubkey"]
+                == "02251fe2ee4bc43729b0903ffadbcf846d9e6acbb3aa593b09d60085645cbe3653"
+            )
+            assert output["bip32_derivs"][0]["path"] == "m/84h/1h/0h/1/0"
 
-    # Check the fields of a PSBT created by a taproot wallet
-    # Could be moved to a dedicated test of taproot functionalites in the future
+    # Taproot fields (could be moved to a dedicated test of taproot functionalites in the future)
     taproot_wallet = funded_taproot_wallet
     assert taproot_wallet.is_taproot == True
     address = taproot_wallet.getnewaddress()
-    # Taproot test addrs are bcrt1p
     assert address.startswith("bcrt1p")
     assert taproot_wallet.amount_total == 20
-    # TODO: Test psbts with taproot wallet, especially the new taproot fields.
+    # Let's keep the random address so we have a "mixed" set of outputs: segwit and taproot
+    psbt = taproot_wallet.createpsbt(
+        [random_address],
+        [3],
+        False,
+        0,
+        1,
+    )
+    # Input fields
+    assert psbt["inputs"][0]["taproot_bip32_derivs"][0]["path"] == "m/86h/1h/0h/0/1"
+    assert (
+        psbt["inputs"][0]["taproot_bip32_derivs"][0]["master_fingerprint"] == "8c24a510"
+    )
+    assert psbt["inputs"][0]["taproot_bip32_derivs"][0]["leaf_hashes"] == []
+    complete_pubkey = (
+        "0274fea50d7f2a69489c2d2a146e317e02f47ad032e81b35fe6059e066670a100e"
+    )
+    assert (
+        psbt["inputs"][0]["taproot_bip32_derivs"][0]["pubkey"] == complete_pubkey[2:]
+    )  # The pubkey is "xonly", for details: https://embit.rocks/#/api/ec/public_key?id=xonly
+    # Output fields
+    for output in psbt["outputs"]:
+        if output["change"] == False:
+            assert output["address"] == "bcrt1q7mlxxdna2e2ufzgalgp5zhtnndl7qddlxjy5eg"
+        else:
+            assert output["taproot_bip32_derivs"][0]["path"] == "m/86h/1h/0h/1/0"
+            assert (
+                output["taproot_bip32_derivs"][0]["pubkey"]
+                == "85b747f5ffc1a1ff951790771c86b24725e283afb2d7e5b8392858bc04f5d05c"
+            )
+            assert (
+                output["taproot_bip32_derivs"][0]["pubkey"]
+                == output["taproot_internal_key"]
+            )
+            assert output["taproot_bip32_derivs"][0]["leaf_hashes"] == []
 
 
 @pytest.mark.slow