Simplify how Eclair manages Bitcoin Core's private keys

We don't even need to use Bitcoin Core's external signer interface, instead we can simply create an empty
wallet and import descriptors generated by Eclair.
This is functionnaly equivalent to what we had (we disabled tx singing from Bitcoin Core) and much simpler.
This is actually a small change and the signing worflow remains the same.

eclair-core/src/main/resources/reference.conf

@@ -40,7 +40,7 @@ eclair {
  // - ignore: eclair will leave these utxos locked and start
  startup-locked-utxos-behavior = "stop"
  final-pubkey-refresh-delay = 3 seconds
- use-external-signer = false
+ use-eclair-signer = false
  }
 
  node-alias = "eclair"

eclair-core/src/main/scala/fr/acinq/eclair/Setup.scala

@@ -108,7 +108,7 @@ class Setup(val datadir: File,
  val nodeKeyManager = new LocalNodeKeyManager(nodeSeed, NodeParams.hashFromChain(chain))
  val channelKeyManager = new LocalChannelKeyManager(channelSeed, NodeParams.hashFromChain(chain))
  val onchainKeyManager = {
- val passphrase = if (config.hasPath("bitcoind.external-signer-passphrase")) config.getString("bitcoind.external-signer-passphrase") else ""
+ val passphrase = if (config.hasPath("bitcoind.eclair-signer-passphrase")) config.getString("bitcoind.eclair-signer-passphrase") else ""
  new LocalOnchainKeyManager(onchainSeed, NodeParams.hashFromChain(chain), passphrase)
  }
  val instanceId = UUID.randomUUID()
@@ -270,8 +270,8 @@ class Setup(val datadir: File,
 
  finalPubkey = new AtomicReference[PublicKey](null)
  pubkeyRefreshDelay = FiniteDuration(config.getDuration("bitcoind.final-pubkey-refresh-delay").getSeconds, TimeUnit.SECONDS)
- useExternalSigner = if (config.hasPath("bitcoind.use-external-signer")) config.getBoolean("bitcoind.use-external-signer") else false
- bitcoinClient = new BitcoinCoreClient(bitcoin, if (useExternalSigner) Some(onchainKeyManager) else None) with OnchainPubkeyCache {
+ useEclairSigner = if (config.hasPath("bitcoind.use-eclair-signer")) config.getBoolean("bitcoind.use-eclair-signer") else false
+ bitcoinClient = new BitcoinCoreClient(bitcoin, if (useEclairSigner) Some(onchainKeyManager) else None) with OnchainPubkeyCache {
  val refresher: typed.ActorRef[OnchainPubkeyRefresher.Command] = system.spawn(Behaviors.supervise(OnchainPubkeyRefresher(this, finalPubkey, pubkeyRefreshDelay)).onFailure(typed.SupervisorStrategy.restart), name = "onchain-address-manager")
 
  override def getP2wpkhPubkey(renew: Boolean): PublicKey = {

eclair-core/src/main/scala/fr/acinq/eclair/blockchain/bitcoind/rpc/BitcoinCoreClient.scala

@@ -48,8 +48,7 @@ import scala.util.{Failure, Success, Try}
  *
  * @param rpcClient bitcoin core JSON rpc client
  * @param onchainKeyManager_opt optional onchain key manager. If provided, it will be used to sign transactions (it is assumed that bitcoin
- * core uses a wallet with "external signer" enabled, and that this external signer is eclair through its
- * RPC API and using the same onchain key manager)
+ * core uses a watch-only wallet with descriptors generated by Eclair with this onchain key manager)
  */
 class BitcoinCoreClient(val rpcClient: BitcoinJsonRPCClient, val onchainKeyManager_opt: Option[OnchainKeyManager] = None) extends OnChainWallet with Logging {
 
@@ -663,4 +662,5 @@ object BitcoinCoreClient {
 
  def toSatoshi(btcAmount: BigDecimal): Satoshi = Satoshi(btcAmount.bigDecimal.scaleByPowerOfTen(8).longValue)
 
+ case class Descriptor(desc: String, internal: Boolean = false, timestamp: String = "now", active: Boolean = true)
 }

eclair-core/src/main/scala/fr/acinq/eclair/crypto/keymanager/LocalOnchainKeyManager.scala

@@ -20,8 +20,8 @@ class LocalOnchainKeyManager(entropy: ByteVector, chainHash: ByteVector32, passp
 
  // master key. we will use it to generate a BIP84 wallet that can be used:
  // - to generate a watch-only wallet with any BIP84-compatible bitcoin wallet
- // - to generate descriptors that can be used by Bitcoin Core through HWI to create a wallet with the `external signer`
- // option set, the external signer being this onchain key manager accessed through Eclair's API
+ // - to generate descriptors that can be import into Bitcoin Core to create a watch-only wallet which can be used
+ // by Eclair to fund transactions (only Eclair will be able to sign wallet inputs).
 
  // m / purpose' / coin_type' / account' / change / address_index
  private val mnemonics = MnemonicCode.toMnemonics(entropy)

eclair-core/src/test/scala/fr/acinq/eclair/blockchain/bitcoind/BitcoinCoreClientSpec.scala

@@ -61,7 +61,7 @@ class BitcoinCoreClientSpec extends TestKitBaseClass with BitcoindService with A
  }
 
  test("encrypt wallet") {
- assume(!useExternalSigner)
+ assume(!useEclairSigner)
 
  val sender = TestProbe()
  val bitcoinClient = makeBitcoinCoreClient
@@ -1137,8 +1137,8 @@ class BitcoinCoreClientSpec extends TestKitBaseClass with BitcoindService with A
 
 }
 
-class BitcoinCoreClientWithExternalSignerSpec extends BitcoinCoreClientSpec {
- override val useExternalSigner = true
+class BitcoinCoreClientWithEclairSignerSpec extends BitcoinCoreClientSpec {
+ override val useEclairSigner = true
 
  test("wallets managed by eclair implement BIP84") {
  val sender = TestProbe()
@@ -1149,20 +1149,18 @@ class BitcoinCoreClientWithExternalSignerSpec extends BitcoinCoreClientSpec {
  val master = DeterministicWallet.generate(seed)
 
  val onchainKeyManager = new LocalOnchainKeyManager(entropy, Block.RegtestGenesisBlock.hash, passphrase = "")
- setExternalSignerScript(onchainKeyManager)
- bitcoinrpcclient.invoke("createwallet", s"eclair_$hex", true, false, "", false, true, true, true).pipeTo(sender.ref)
+ bitcoinrpcclient.invoke("createwallet", s"eclair_$hex", true, false, "", false, true, true, false).pipeTo(sender.ref)
  sender.expectMsgType[JValue]
+ val jsonRpcClient = new BasicBitcoinJsonRPCClient(rpcAuthMethod = bitcoinrpcauthmethod, host = "localhost", port = bitcoindRpcPort, wallet = Some(s"eclair_$hex"))
+ importEclairDescriptors(jsonRpcClient, onchainKeyManager)
 
  // this account xpub can be used to create a watch-only wallet
  val accountXPub = DeterministicWallet.encode(
  DeterministicWallet.publicKey(DeterministicWallet.derivePrivateKey(master, DeterministicWallet.KeyPath("m/84'/1'/0'"))),
  DeterministicWallet.vpub)
  assert(onchainKeyManager.getOnchainMasterPubKey(0) == accountXPub)
 
- val wallet1 = new BitcoinCoreClient(
- new BasicBitcoinJsonRPCClient(rpcAuthMethod = bitcoinrpcauthmethod, host = "localhost", port = bitcoindRpcPort, wallet = Some(s"eclair_$hex")),
- Some(onchainKeyManager)
- )
+ val wallet1 = new BitcoinCoreClient(jsonRpcClient, Some(onchainKeyManager))
 
  def getBip32Path(address: String): DeterministicWallet.KeyPath = {
  wallet1.rpcClient.invoke("getaddressinfo", address).pipeTo(sender.ref)
@@ -1192,14 +1190,12 @@ class BitcoinCoreClientWithExternalSignerSpec extends BitcoinCoreClientSpec {
  val entropy = randomBytes32()
  val hex = entropy.toString()
  val onchainKeyManager = new LocalOnchainKeyManager(entropy, Block.RegtestGenesisBlock.hash, passphrase = "")
- setExternalSignerScript(onchainKeyManager)
- bitcoinrpcclient.invoke("createwallet", s"eclair_$hex", true, false, "", false, true, true, true).pipeTo(sender.ref)
+ bitcoinrpcclient.invoke("createwallet", s"eclair_$hex", true, false, "", false, true, true, false).pipeTo(sender.ref)
  sender.expectMsgType[JValue]
 
- val wallet1 = new BitcoinCoreClient(
- new BasicBitcoinJsonRPCClient(rpcAuthMethod = bitcoinrpcauthmethod, host = "localhost", port = bitcoindRpcPort, wallet = Some(s"eclair_$hex")),
- Some(onchainKeyManager)
- )
+ val jsonRpcClient = new BasicBitcoinJsonRPCClient(rpcAuthMethod = bitcoinrpcauthmethod, host = "localhost", port = bitcoindRpcPort, wallet = Some(s"eclair_$hex"))
+ importEclairDescriptors(jsonRpcClient, onchainKeyManager)
+ val wallet1 = new BitcoinCoreClient(jsonRpcClient, Some(onchainKeyManager))
  wallet1.getReceiveAddress().pipeTo(sender.ref)
  val address = sender.expectMsgType[String]
 

eclair-core/src/test/scala/fr/acinq/eclair/blockchain/bitcoind/BitcoindService.scala

@@ -22,7 +22,7 @@ import akka.testkit.{TestKitBase, TestProbe}
 import fr.acinq.bitcoin.psbt.Psbt
 import fr.acinq.bitcoin.scalacompat.Crypto.PrivateKey
 import fr.acinq.bitcoin.scalacompat.{Block, Btc, BtcAmount, MilliBtc, Satoshi, Transaction, TxOut, computeP2WpkhAddress}
-import fr.acinq.eclair.blockchain.bitcoind.rpc.BitcoinCoreClient.PreviousTx
+import fr.acinq.eclair.blockchain.bitcoind.rpc.BitcoinCoreClient.{Descriptor, PreviousTx}
 import fr.acinq.eclair.blockchain.bitcoind.rpc.BitcoinJsonRPCAuthMethod.{SafeCookie, UserPassword}
 import fr.acinq.eclair.blockchain.bitcoind.rpc.{BasicBitcoinJsonRPCClient, BitcoinCoreClient, BitcoinJsonRPCAuthMethod, BitcoinJsonRPCClient}
 import fr.acinq.eclair.blockchain.fee.{FeeratePerByte, FeeratePerKB, FeeratePerKw}
@@ -35,7 +35,6 @@ import scodec.bits.ByteVector
 import sttp.client3.okhttp.OkHttpFutureBackend
 
 import java.io.File
-import java.nio.charset.StandardCharsets
 import java.nio.file.Files
 import scala.concurrent.ExecutionContext.Implicits.global
 import scala.concurrent.Future
@@ -45,7 +44,7 @@ import scala.io.Source
 trait BitcoindService extends Logging {
  self: TestKitBase =>
 
- def useExternalSigner: Boolean = false
+ def useEclairSigner: Boolean = false
 
  import BitcoindService._
 
@@ -54,7 +53,7 @@ trait BitcoindService extends Logging {
  implicit val system: ActorSystem
  implicit val sttpBackend = OkHttpFutureBackend()
 
- val defaultWallet: String = if (useExternalSigner) "eclair" else "miner"
+ val defaultWallet: String = if (useEclairSigner) "eclair" else "miner"
  val bitcoindPort: Int = TestUtils.availablePort
  val bitcoindRpcPort: Int = TestUtils.availablePort
  val bitcoindZmqBlockPort: Int = TestUtils.availablePort
@@ -75,28 +74,6 @@ trait BitcoindService extends Logging {
  var bitcoinrpcauthmethod: BitcoinJsonRPCAuthMethod = _
  var bitcoincli: ActorRef = _
  val onchainKeyManager = new LocalOnchainKeyManager(ByteVector.fromValidHex("01" * 32), Block.RegtestGenesisBlock.hash)
-
- def setExternalSignerScript(keyManager: OnchainKeyManager): Unit = {
- val (main, change) = keyManager.getDescriptors(0, Some("regtest"), 0)
- val script =
- s"""|#!/bin/bash
- |
- | while [ -n "$$1" ]; do # while loop starts
- | case "$$1" in
- | enumerate) echo '[{"type":"eclair","model":"eclair","label":"","path":"","fingerprint":"${keyManager.getOnchainMasterMasterFingerprint}","needs_pin_sent":false,"needs_passphrase_sent":false}]'; exit ;;
- | getdescriptors) echo "{\\"receive\\":[\\"${main.head}\\"],\\"internal\\":[\\"${change.head}\\"]}"; exit ;;
- | --stdin)
- | read -r cmdline
- | ;;
- | *) shift ;;
- | esac
- | shift
- |done
- |""".stripMargin
- Files.write(PATH_BITCOIND_DATADIR.toPath.resolve("eclair-hwi.sh"), script.getBytes(StandardCharsets.UTF_8))
- PATH_BITCOIND_DATADIR.toPath.resolve("eclair-hwi.sh").toFile.setExecutable(true)
- }
-
  def startBitcoind(useCookie: Boolean = false,
  defaultAddressType_opt: Option[String] = None,
  mempoolSize_opt: Option[Int] = None, // mempool size in MB
@@ -115,7 +92,6 @@ trait BitcoindService extends Logging {
  .appendedAll(defaultAddressType_opt.map(addressType => s"changetype=$addressType\n").getOrElse(""))
  .appendedAll(mempoolSize_opt.map(mempoolSize => s"maxmempool=$mempoolSize\n").getOrElse(""))
  .appendedAll(mempoolMinFeerate_opt.map(mempoolMinFeerate => s"minrelaytxfee=${FeeratePerKB(mempoolMinFeerate).feerate.toBtc.toBigDecimal}\n").getOrElse(""))
- .appendedAll(s"signer=${PATH_BITCOIND_DATADIR.toPath.resolve("eclair-hwi.sh").toAbsolutePath}")
  if (useCookie) {
  defaultConf
  .replace("rpcuser=foo", "")
@@ -124,7 +100,6 @@ trait BitcoindService extends Logging {
  defaultConf
  }
  }
- setExternalSignerScript(onchainKeyManager)
  Files.writeString(new File(PATH_BITCOIND_DATADIR.toString, "bitcoin.conf").toPath, conf)
  }
 
@@ -144,7 +119,7 @@ trait BitcoindService extends Logging {
  }))
  }
 
- def makeBitcoinCoreClient: BitcoinCoreClient = new BitcoinCoreClient(bitcoinrpcclient, if (useExternalSigner) Some(onchainKeyManager) else None)
+ def makeBitcoinCoreClient: BitcoinCoreClient = new BitcoinCoreClient(bitcoinrpcclient, if (useEclairSigner) Some(onchainKeyManager) else None)
 
  def stopBitcoind(): Unit = {
  // gracefully stopping bitcoin will make it store its state cleanly to disk, which is good for later debugging
@@ -181,18 +156,30 @@ trait BitcoindService extends Logging {
  def waitForBitcoindReady(): Unit = {
  val sender = TestProbe()
  waitForBitcoindUp(sender)
- if (useExternalSigner) {
- bitcoinrpcclient.invoke("createwallet", defaultWallet, true, false, "", false, true, true, true).pipeTo(sender.ref)
+ if (useEclairSigner) {
+ // wallet_name, disable_private_keys, blank, passphrase, avoid_reuse, descriptors, load_on_startup, external_signer
+ bitcoinrpcclient.invoke("createwallet", defaultWallet, true, false, "", false, true, true, false).pipeTo(sender.ref)
+ sender.expectMsgType[JValue]
+
+ val jsonRpcClient = new BasicBitcoinJsonRPCClient(rpcAuthMethod = bitcoinrpcauthmethod, host = "localhost", port = bitcoindRpcPort, wallet = Some(defaultWallet))
+ importEclairDescriptors(jsonRpcClient, onchainKeyManager)
  } else {
  sender.send(bitcoincli, BitcoinReq("createwallet", defaultWallet))
+ sender.expectMsgType[JValue]
  }
- sender.expectMsgType[JValue]
  logger.info(s"generating initial blocks to wallet=$defaultWallet...")
  generateBlocks(150)
  awaitCond(currentBlockHeight(sender) >= BlockHeight(150), max = 3 minutes, interval = 2 second)
  }
 
 
+ def importEclairDescriptors(jsonRpcClient: BitcoinJsonRPCClient, keyManager: OnchainKeyManager, probe: TestProbe = TestProbe()): Unit = {
+ val (main, change) = keyManager.getDescriptors(keyManager.getOnchainMasterMasterFingerprint, Some("regtest"), 0)
+ val descriptors = main.map(d => Descriptor(d)) ++ change.map(d => Descriptor(d, internal = true))
+ jsonRpcClient.invoke("importdescriptors", descriptors).pipeTo(probe.ref)
+ probe.expectMsgType[JValue]
+ }
+
  def generateBlocks(blockCount: Int, address: Option[String] = None, timeout: FiniteDuration = 10 seconds)(implicit system: ActorSystem): Unit = {
  val sender = TestProbe()
  val addressToUse = address match {
@@ -247,7 +234,7 @@ trait BitcoindService extends Logging {
  val tx = Transaction(version = 2, Nil, TxOut(amountSat, addressToPublicKeyScript(address, Block.RegtestGenesisBlock.hash)) :: Nil, lockTime = 0)
  val client = makeBitcoinCoreClient
  val f = for {
- funded <- client.fundTransaction(tx, FeeratePerKw(Satoshi(1000)), true)
+ funded <- client.fundTransaction(tx, FeeratePerKw(FeeratePerByte(Satoshi(10))), true)
  signed <- client.signPsbt(new Psbt(funded.tx), funded.tx.txIn.indices, Nil)
  txid <- client.publishTransaction(signed.finalTx)
  tx <- client.getTransaction(txid)

eclair-core/src/test/scala/fr/acinq/eclair/blockchain/fee/BitcoinCoreFeeProviderSpec.scala

@@ -34,7 +34,7 @@ import scala.concurrent.{ExecutionContext, Future}
 
 class BitcoinCoreFeeProviderSpec extends TestKitBaseClass with BitcoindService with AnyFunSuiteLike with BeforeAndAfterAll with Logging {
 
- override val useExternalSigner = false
+ override val useEclairSigner = false
 
  override def beforeAll(): Unit = {
  startBitcoind()

eclair-core/src/test/scala/fr/acinq/eclair/channel/InteractiveTxBuilderSpec.scala

@@ -36,8 +36,7 @@ import fr.acinq.eclair.io.OpenChannelInterceptor.makeChannelParams
 import fr.acinq.eclair.transactions.Scripts
 import fr.acinq.eclair.transactions.Transactions.InputInfo
 import fr.acinq.eclair.wire.protocol._
-import fr.acinq.eclair.{Feature, FeatureSupport, Features, InitFeature, MilliSatoshiLong, NodeParams, TestConstants, TestKitBaseClass, ToMilliSatoshiConversion, UInt64, randomBytes32, randomKey}
-import fr.acinq.eclair.{Feature, FeatureSupport, Features, InitFeature, MilliSatoshiLong, NodeParams, TestConstants, TestKitBaseClass, UInt64, addressToPublicKeyScript, randomBytes32, randomKey}
+import fr.acinq.eclair.{Feature, FeatureSupport, Features, InitFeature, MilliSatoshiLong, NodeParams, TestConstants, TestKitBaseClass, ToMilliSatoshiConversion, UInt64, addressToPublicKeyScript, randomBytes32, randomKey}
 import org.scalatest.BeforeAndAfterAll
 import org.scalatest.funsuite.AnyFunSuiteLike
 import scodec.bits.{ByteVector, HexStringSyntax}
@@ -2670,6 +2669,6 @@ class InteractiveTxBuilderSpec extends TestKitBaseClass with AnyFunSuiteLike wit
 
 }
 
-class InteractiveTxBuilderWithExternalSignerSpec extends InteractiveTxBuilderSpec {
- override val useExternalSigner = true
+class InteractiveTxBuilderWithEclairSignerSpec extends InteractiveTxBuilderSpec {
+ override val useEclairSigner = true
 }

eclair-core/src/test/scala/fr/acinq/eclair/channel/publish/ReplaceableTxPublisherSpec.scala

@@ -42,8 +42,8 @@ import fr.acinq.eclair.transactions.Transactions._
 import fr.acinq.eclair.wire.protocol.{CommitSig, RevokeAndAck}
 import fr.acinq.eclair.{BlockHeight, MilliSatoshiLong, NodeParams, NotificationsLogger, TestConstants, TestFeeEstimator, TestKitBaseClass, randomKey}
 import org.json4s.JValue
+import org.scalatest.BeforeAndAfterAll
 import org.scalatest.funsuite.AnyFunSuiteLike
-import org.scalatest.{BeforeAndAfterAll, DoNotDiscover, Sequential}
 import scodec.bits.ByteVector
 
 import java.util.UUID
@@ -1589,16 +1589,16 @@ class ReplaceableTxPublisherSpec extends TestKitBaseClass with AnyFunSuiteLike w
 
 }
 
-class ReplaceableTxPublisherWithExternalSignerSpec extends ReplaceableTxPublisherSpec {
+class ReplaceableTxPublisherWithEclairSignerSpec extends ReplaceableTxPublisherSpec {
  override def createTestWallet(walletName: String) = {
  val probe = TestProbe()
  // we use the wallet name as a passphrase to make sure we get a new empty wallet
  val keyManager = new LocalOnchainKeyManager(ByteVector.fromValidHex("01" * 32), Block.RegtestGenesisBlock.hash, walletName)
- setExternalSignerScript(keyManager)
- bitcoinrpcclient.invoke("createwallet", walletName, true, false, "", false, true, true, true).pipeTo(probe.ref)
+ bitcoinrpcclient.invoke("createwallet", walletName, true, false, "", false, true, true, false).pipeTo(probe.ref)
  probe.expectMsgType[JValue]
 
  val walletRpcClient = new BasicBitcoinJsonRPCClient(rpcAuthMethod = bitcoinrpcauthmethod, host = "localhost", port = bitcoindRpcPort, wallet = Some(walletName))
+ importEclairDescriptors(walletRpcClient, keyManager)
  val walletClient = new BitcoinCoreClient(walletRpcClient, Some(keyManager)) with OnchainPubkeyCache {
  val pubkey = {
  getP2wpkhPubkey().pipeTo(probe.ref)