From 99848eea0bc6068ce8bca7ea542be9cfb2332331 Mon Sep 17 00:00:00 2001
From: Sjors Provoost <sjors@sprovoost.nl>
Date: Wed, 18 Dec 2024 10:42:35 +0700
Subject: [PATCH 01/11] Add waitNext() to BlockTemplate interface

---
 src/interfaces/mining.h    | 12 +++++-
 src/ipc/capnp/mining.capnp |  6 +++
 src/node/interfaces.cpp    | 84 +++++++++++++++++++++++++++++++++++++-
 src/node/types.h           | 18 ++++++++
 src/test/miner_tests.cpp   | 24 ++++++++---
 5 files changed, 135 insertions(+), 9 deletions(-)

diff --git a/src/interfaces/mining.h b/src/interfaces/mining.h
index bc5955ded6628..49aa53feb1dcf 100644
--- a/src/interfaces/mining.h
+++ b/src/interfaces/mining.h
@@ -7,7 +7,7 @@
 
 #include <consensus/amount.h>       // for CAmount
 #include <interfaces/types.h>       // for BlockRef
-#include <node/types.h>             // for BlockCreateOptions
+#include <node/types.h>             // for BlockCreateOptions, BlockWaitOptions
 #include <primitives/block.h>       // for CBlock, CBlockHeader
 #include <primitives/transaction.h> // for CTransactionRef
 #include <stdint.h>                 // for int64_t
@@ -56,6 +56,16 @@ class BlockTemplate
      * @returns if the block was processed, independent of block validity
      */
     virtual bool submitSolution(uint32_t version, uint32_t timestamp, uint32_t nonce, CTransactionRef coinbase) = 0;
+
+    /**
+     * Waits for fees in the next block to rise, a new tip or the timeout.
+     *
+     * @param[in] options   Control the timeout (default forever) and by how much total fees
+     *                      for the next block should rise (default infinite).
+     *
+     * @returns a new BlockTemplate or nullptr if the timeout occurs.
+     */
+    virtual std::unique_ptr<BlockTemplate> waitNext(const node::BlockWaitOptions options = {}) = 0;
 };
 
 //! Interface giving clients (RPC, Stratum v2 Template Provider in the future)
diff --git a/src/ipc/capnp/mining.capnp b/src/ipc/capnp/mining.capnp
index 50b07bda708b7..bc1005ffa82cd 100644
--- a/src/ipc/capnp/mining.capnp
+++ b/src/ipc/capnp/mining.capnp
@@ -31,6 +31,7 @@ interface BlockTemplate $Proxy.wrap("interfaces::BlockTemplate") {
     getWitnessCommitmentIndex @7 (context: Proxy.Context) -> (result: Int32);
     getCoinbaseMerklePath @8 (context: Proxy.Context) -> (result: List(Data));
     submitSolution @9 (context: Proxy.Context, version: UInt32, timestamp: UInt32, nonce: UInt32, coinbase :Data) -> (result: Bool);
+    waitNext @10 (context: Proxy.Context, options: BlockWaitOptions) -> (result: BlockTemplate);
 }
 
 struct BlockCreateOptions $Proxy.wrap("node::BlockCreateOptions") {
@@ -39,6 +40,11 @@ struct BlockCreateOptions $Proxy.wrap("node::BlockCreateOptions") {
     coinbaseOutputMaxAdditionalSigops @2 :UInt64 $Proxy.name("coinbase_output_max_additional_sigops");
 }
 
+struct BlockWaitOptions $Proxy.wrap("node::BlockWaitOptions") {
+    timeout @0 : Float64 $Proxy.name("timeout");
+    feeThreshold @1 : Int64 $Proxy.name("fee_threshold");
+}
+
 # Note: serialization of the BlockValidationState C++ type is somewhat fragile
 # and using the struct can be awkward. It would be good if testBlockValidity
 # method were changed to return validity information in a simpler format.
diff --git a/src/node/interfaces.cpp b/src/node/interfaces.cpp
index fe88a6dad1108..372f84aa4cdf2 100644
--- a/src/node/interfaces.cpp
+++ b/src/node/interfaces.cpp
@@ -81,6 +81,7 @@ using interfaces::Mining;
 using interfaces::Node;
 using interfaces::WalletLoader;
 using node::BlockAssembler;
+using node::BlockWaitOptions;
 using util::Join;
 
 namespace node {
@@ -871,7 +872,7 @@ class ChainImpl : public Chain
 class BlockTemplateImpl : public BlockTemplate
 {
 public:
-    explicit BlockTemplateImpl(std::unique_ptr<CBlockTemplate> block_template, NodeContext& node) : m_block_template(std::move(block_template)), m_node(node)
+    explicit BlockTemplateImpl(BlockAssembler::Options assemble_options, std::unique_ptr<CBlockTemplate> block_template, NodeContext& node) : m_assemble_options(std::move(assemble_options)), m_block_template(std::move(block_template)), m_node(node)
     {
         assert(m_block_template);
     }
@@ -936,9 +937,88 @@ class BlockTemplateImpl : public BlockTemplate
         return chainman().ProcessNewBlock(block_ptr, /*force_processing=*/true, /*min_pow_checked=*/true, /*new_block=*/nullptr);
     }
 
+    std::unique_ptr<BlockTemplate> waitNext(BlockWaitOptions options) override
+    {
+        // Delay calculating the current template fees, just in case a new block
+        // comes in before the next tick.
+        CAmount current_fees = -1;
+
+        // Alternate waiting for a new tip and checking if fees have risen.
+        // The latter check is expensive so we only run it once per second.
+        auto now{NodeClock::now()};
+        const auto deadline = now + options.timeout;
+        const MillisecondsDouble tick{1000};
+
+        while (now <= deadline) {
+            bool tip_changed{false};
+            {
+                WAIT_LOCK(notifications().m_tip_block_mutex, lock);
+                notifications().m_tip_block_cv.wait_until(lock, std::min(now + tick, deadline), [&]() EXCLUSIVE_LOCKS_REQUIRED(notifications().m_tip_block_mutex) {
+                    Assert(notifications().TipBlock());
+                    tip_changed = notifications().TipBlock() != m_block_template->block.hashPrevBlock;
+                    return tip_changed || chainman().m_interrupt;
+                });
+            }
+
+            if (chainman().m_interrupt) return nullptr;
+
+            // Must release m_tip_block_mutex before locking cs_main, to avoid deadlocks.
+            LOCK(::cs_main);
+            /**
+             * The only way to determine if fees increased compared to the previous template,
+             * is to generate a fresh template. Cluster Mempool may allow for a more efficient
+             * way to determine how much (approximate) fees for the next block increased.
+             *
+             * We'll also create a new template if the tip changed during the last tick.
+             */
+            if (options.fee_threshold < MAX_MONEY || tip_changed) {
+                auto block_template{std::make_unique<BlockTemplateImpl>(m_assemble_options, BlockAssembler{chainman().ActiveChainstate(), context()->mempool.get(), m_assemble_options}.CreateNewBlock(), m_node)};
+
+                // If the tip changed, return the new template regardless of its fees.
+                if (tip_changed) {
+                    return block_template;
+                }
+
+                // Calculate the original template total fees if we haven't already
+                if (current_fees == -1) {
+                    current_fees = 0;
+                    for (CAmount fee : m_block_template->vTxFees) {
+                        // Skip coinbase
+                        if (fee < 0) continue;
+                        current_fees += fee;
+                    }
+                }
+
+                CAmount new_fees = 0;
+                for (CAmount fee : block_template->m_block_template->vTxFees) {
+                    // Skip coinbase
+                    if (fee < 0) continue;
+                    new_fees += fee;
+                    if (new_fees >= current_fees + options.fee_threshold) return block_template;
+                }
+            }
+
+            /**
+             * Break out of while when using mock time. While functional tests
+             * can call setmocktime to move the clock and exit this loop, a
+             * unit tests would need to spawn a thread to achieve this.
+             * Making the while loop use now < deadline won't work either, because
+             * then there's no wait to test its internals.
+             */
+            if (now == deadline) break;
+            now = NodeClock::now();
+        }
+
+        return nullptr;
+    }
+
+    const BlockAssembler::Options m_assemble_options;
+
     const std::unique_ptr<CBlockTemplate> m_block_template;
 
+    NodeContext* context() { return &m_node; }
     ChainstateManager& chainman() { return *Assert(m_node.chainman); }
+    KernelNotifications& notifications() { return *Assert(m_node.notifications); }
     NodeContext& m_node;
 };
 
@@ -985,7 +1065,7 @@ class MinerImpl : public Mining
     {
         BlockAssembler::Options assemble_options{options};
         ApplyArgsManOptions(*Assert(m_node.args), assemble_options);
-        return std::make_unique<BlockTemplateImpl>(BlockAssembler{chainman().ActiveChainstate(), context()->mempool.get(), assemble_options}.CreateNewBlock(), m_node);
+        return std::make_unique<BlockTemplateImpl>(assemble_options, BlockAssembler{chainman().ActiveChainstate(), context()->mempool.get(), assemble_options}.CreateNewBlock(), m_node);
     }
 
     NodeContext* context() override { return &m_node; }
diff --git a/src/node/types.h b/src/node/types.h
index 4b0de084ab3d0..6d935f2ecb07f 100644
--- a/src/node/types.h
+++ b/src/node/types.h
@@ -14,7 +14,9 @@
 #define BITCOIN_NODE_TYPES_H
 
 #include <cstddef>
+#include <consensus/amount.h>
 #include <script/script.h>
+#include <util/time.h>
 
 namespace node {
 enum class TransactionError {
@@ -61,6 +63,22 @@ struct BlockCreateOptions {
      */
     CScript coinbase_output_script{CScript() << OP_TRUE};
 };
+
+struct BlockWaitOptions {
+    /**
+     * How long to wait before returning nullptr instead of a new template.
+     * Default is to wait forever.
+     */
+    MillisecondsDouble timeout{MillisecondsDouble::max()};
+
+    /**
+     * Wait until total fees in the new template exceed fees in the origal
+     * template by at least this amount (in sats). The default is to ignore
+     * fee increases and only wait for a tip change.
+     */
+    CAmount fee_threshold{MAX_MONEY};
+};
+
 } // namespace node
 
 #endif // BITCOIN_NODE_TYPES_H
diff --git a/src/test/miner_tests.cpp b/src/test/miner_tests.cpp
index 5bfcde7718327..28e4319cc81b3 100644
--- a/src/test/miner_tests.cpp
+++ b/src/test/miner_tests.cpp
@@ -180,8 +180,11 @@ void MinerTestingSetup::TestPackageSelection(const CScript& scriptPubKey, const
     tx.vout[0].nValue = 5000000000LL - 1000 - 50000 - feeToUse;
     Txid hashLowFeeTx = tx.GetHash();
     AddToMempool(tx_mempool, entry.Fee(feeToUse).FromTx(tx));
-    block_template = mining->createNewBlock(options);
-    BOOST_REQUIRE(block_template);
+
+    // waitNext() should return nullptr because there is no better template
+    auto should_be_nullptr = block_template->waitNext({.timeout = MillisecondsDouble{0}, .fee_threshold = 1});
+    BOOST_REQUIRE(should_be_nullptr == nullptr);
+
     block = block_template->getBlock();
     // Verify that the free tx and the low fee tx didn't get selected
     for (size_t i=0; i<block.vtx.size(); ++i) {
@@ -196,7 +199,9 @@ void MinerTestingSetup::TestPackageSelection(const CScript& scriptPubKey, const
     tx.vout[0].nValue -= 2; // Now we should be just over the min relay fee
     hashLowFeeTx = tx.GetHash();
     AddToMempool(tx_mempool, entry.Fee(feeToUse + 2).FromTx(tx));
-    block_template = mining->createNewBlock(options);
+
+    // waitNext() should return if fees for the new template are at least 1 sat up
+    block_template = block_template->waitNext({.fee_threshold = 1});
     BOOST_REQUIRE(block_template);
     block = block_template->getBlock();
     BOOST_REQUIRE_EQUAL(block.vtx.size(), 6U);
@@ -672,8 +677,10 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity)
         const int current_height{mining->getTip()->height};
 
         // Simple block creation, nothing special yet:
-        block_template = mining->createNewBlock(options);
-        BOOST_REQUIRE(block_template);
+        if (current_height % 2 == 0) {
+            block_template = mining->createNewBlock(options);
+            BOOST_REQUIRE(block_template);
+        } // for odd heights the next template is created at the end of this loop
 
         CBlock block{block_template->getBlock()};
         CMutableTransaction txCoinbase(*block.vtx[0]);
@@ -710,7 +717,12 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity)
             BOOST_REQUIRE_EQUAL(maybe_new_tip->GetBlockHash(), block.GetHash());
         }
         // This just adds coverage
-        mining->waitTipChanged(block.hashPrevBlock);
+        if (current_height % 2 == 0) {
+            block_template = block_template->waitNext();
+            BOOST_REQUIRE(block_template);
+        } else {
+            mining->waitTipChanged(block.hashPrevBlock);
+        }
     }
 
     LOCK(cs_main);

From affb0c119573cab15b6d7fb617acb1c597e7004d Mon Sep 17 00:00:00 2001
From: Sjors Provoost <sjors@sprovoost.nl>
Date: Fri, 24 Jan 2025 11:41:58 +0100
Subject: [PATCH 02/11] On testnet create min diff template after 20 mins

---
 src/interfaces/mining.h           |  3 ++
 src/node/interfaces.cpp           | 13 +++++-
 src/test/CMakeLists.txt           |  1 +
 src/test/testnet4_miner_tests.cpp | 73 +++++++++++++++++++++++++++++++
 src/test/util/setup_common.h      |  6 +++
 5 files changed, 95 insertions(+), 1 deletion(-)
 create mode 100644 src/test/testnet4_miner_tests.cpp

diff --git a/src/interfaces/mining.h b/src/interfaces/mining.h
index 49aa53feb1dcf..d6038fe718a05 100644
--- a/src/interfaces/mining.h
+++ b/src/interfaces/mining.h
@@ -64,6 +64,9 @@ class BlockTemplate
      *                      for the next block should rise (default infinite).
      *
      * @returns a new BlockTemplate or nullptr if the timeout occurs.
+     *
+     * On testnet this will additionally return a template with difficulty 1 if
+     * the tip is more than 20 minutes old.
      */
     virtual std::unique_ptr<BlockTemplate> waitNext(const node::BlockWaitOptions options = {}) = 0;
 };
diff --git a/src/node/interfaces.cpp b/src/node/interfaces.cpp
index 372f84aa4cdf2..38773f6f4ff44 100644
--- a/src/node/interfaces.cpp
+++ b/src/node/interfaces.cpp
@@ -948,6 +948,7 @@ class BlockTemplateImpl : public BlockTemplate
         auto now{NodeClock::now()};
         const auto deadline = now + options.timeout;
         const MillisecondsDouble tick{1000};
+        const bool allow_min_difficulty{chainman().GetParams().GetConsensus().fPowAllowMinDifficultyBlocks};
 
         while (now <= deadline) {
             bool tip_changed{false};
@@ -964,6 +965,15 @@ class BlockTemplateImpl : public BlockTemplate
 
             // Must release m_tip_block_mutex before locking cs_main, to avoid deadlocks.
             LOCK(::cs_main);
+
+            // On test networks return a minimum difficulty block after 20 minutes
+            if (!tip_changed && allow_min_difficulty) {
+                const NodeClock::time_point tip_time{std::chrono::seconds{chainman().ActiveChain().Tip()->GetBlockTime()}};
+                if (now > tip_time + std::chrono::seconds(20 * 60)) {
+                    tip_changed = true;
+                }
+            }
+
             /**
              * The only way to determine if fees increased compared to the previous template,
              * is to generate a fresh template. Cluster Mempool may allow for a more efficient
@@ -1003,7 +1013,8 @@ class BlockTemplateImpl : public BlockTemplate
              * can call setmocktime to move the clock and exit this loop, a
              * unit tests would need to spawn a thread to achieve this.
              * Making the while loop use now < deadline won't work either, because
-             * then there's no wait to test its internals.
+             * then there's no wait to test its internals, e.g. the  20 minute
+             * testnet exception.
              */
             if (now == deadline) break;
             now = NodeClock::now();
diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt
index a4c847c2e9865..ff042558d6151 100644
--- a/src/test/CMakeLists.txt
+++ b/src/test/CMakeLists.txt
@@ -120,6 +120,7 @@ add_executable(test_bitcoin
   streams_tests.cpp
   sync_tests.cpp
   system_tests.cpp
+  testnet4_miner_tests.cpp
   timeoffsets_tests.cpp
   torcontrol_tests.cpp
   transaction_tests.cpp
diff --git a/src/test/testnet4_miner_tests.cpp b/src/test/testnet4_miner_tests.cpp
new file mode 100644
index 0000000000000..434605882c202
--- /dev/null
+++ b/src/test/testnet4_miner_tests.cpp
@@ -0,0 +1,73 @@
+// Copyright (c) 2025 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <common/system.h>
+#include <interfaces/mining.h>
+#include <node/miner.h>
+#include <util/time.h>
+#include <validation.h>
+
+#include <test/util/setup_common.h>
+
+#include <boost/test/unit_test.hpp>
+
+using interfaces::BlockTemplate;
+using interfaces::Mining;
+using node::BlockAssembler;
+using node::BlockWaitOptions;
+
+namespace testnet4_miner_tests {
+
+struct Testnet4MinerTestingSetup : public Testnet4Setup {
+    std::unique_ptr<Mining> MakeMining()
+    {
+        return interfaces::MakeMining(m_node);
+    }
+};
+} // namespace testnet4_miner_tests
+
+BOOST_FIXTURE_TEST_SUITE(testnet4_miner_tests, Testnet4MinerTestingSetup)
+
+BOOST_AUTO_TEST_CASE(MiningInterface)
+{
+    auto mining{MakeMining()};
+    BOOST_REQUIRE(mining);
+
+    BlockAssembler::Options options;
+    std::unique_ptr<BlockTemplate> block_template;
+
+    // Set node time a few minutes past the testnet4 genesis block
+    const int64_t genesis_time{WITH_LOCK(cs_main, return m_node.chainman->ActiveChain().Tip()->GetBlockTime())};
+    SetMockTime(genesis_time + 3 * 60);
+
+    block_template = mining->createNewBlock(options);
+    BOOST_REQUIRE(block_template);
+
+    // The template should use the mocked system time
+    BOOST_REQUIRE_EQUAL(block_template->getBlockHeader().nTime, genesis_time + 3 * 60);
+
+    const BlockWaitOptions wait_options{.timeout = MillisecondsDouble{0}, .fee_threshold = 1};
+
+    // waitNext() should return nullptr because there is no better template
+    auto should_be_nullptr = block_template->waitNext(wait_options);
+    BOOST_REQUIRE(should_be_nullptr == nullptr);
+
+    // This remains the case when exactly 20 minutes have gone by
+    {
+        LOCK(cs_main);
+        SetMockTime(m_node.chainman->ActiveChain().Tip()->GetBlockTime() + 20 * 60);
+    }
+    should_be_nullptr = block_template->waitNext(wait_options);
+    BOOST_REQUIRE(should_be_nullptr == nullptr);
+
+    // One second later the difficulty drops (not tested here) and it returns a new template
+    {
+        LOCK(cs_main);
+        SetMockTime(m_node.chainman->ActiveChain().Tip()->GetBlockTime() + 20 * 60 + 1);
+    }
+    block_template = block_template->waitNext(wait_options);
+    BOOST_REQUIRE(block_template);
+}
+
+BOOST_AUTO_TEST_SUITE_END()
diff --git a/src/test/util/setup_common.h b/src/test/util/setup_common.h
index 33ad258457353..57bea9086b99a 100644
--- a/src/test/util/setup_common.h
+++ b/src/test/util/setup_common.h
@@ -130,6 +130,12 @@ struct RegTestingSetup : public TestingSetup {
         : TestingSetup{ChainType::REGTEST} {}
 };
 
+/** Identical to TestingSetup, but chain set to testnet4 */
+struct Testnet4Setup : public TestingSetup {
+    Testnet4Setup()
+        : TestingSetup{ChainType::TESTNET4} {}
+};
+
 class CBlock;
 struct CMutableTransaction;
 class CScript;

From a7642240a46b224b926e2cf8537921c8271799db Mon Sep 17 00:00:00 2001
From: Sjors Provoost <sjors@sprovoost.nl>
Date: Fri, 15 Nov 2024 15:42:41 +0100
Subject: [PATCH 03/11] Stratum v2 template provider scaffold

The template provider will listen for a Job Declarator client.
It can establish a connection and detect various protocol errors.

Co-Authored-By: Christopher Coverdale <chris.coverdale24@gmail.com>
Co-Authored-By: Fi3
---
 src/node/context.h                       |   1 +
 src/sv2/CMakeLists.txt                   |   1 +
 src/sv2/template_provider.cpp            |  84 +++++++++++++
 src/sv2/template_provider.h              | 106 ++++++++++++++++
 src/test/CMakeLists.txt                  |   1 +
 src/test/sv2_template_provider_tests.cpp | 153 +++++++++++++++++++++++
 6 files changed, 346 insertions(+)
 create mode 100644 src/sv2/template_provider.cpp
 create mode 100644 src/sv2/template_provider.h
 create mode 100644 src/test/sv2_template_provider_tests.cpp

diff --git a/src/node/context.h b/src/node/context.h
index debc12212064c..8334a84d5b8ac 100644
--- a/src/node/context.h
+++ b/src/node/context.h
@@ -25,6 +25,7 @@ class ChainstateManager;
 class ECC_Context;
 class NetGroupManager;
 class PeerManager;
+class Sv2TemplateProvider;
 namespace interfaces {
 class Chain;
 class ChainClient;
diff --git a/src/sv2/CMakeLists.txt b/src/sv2/CMakeLists.txt
index a628204612fcd..7b632032c233f 100644
--- a/src/sv2/CMakeLists.txt
+++ b/src/sv2/CMakeLists.txt
@@ -6,6 +6,7 @@ add_library(bitcoin_sv2 STATIC EXCLUDE_FROM_ALL
   noise.cpp
   transport.cpp
   connman.cpp
+  template_provider.cpp
 )
 
 target_link_libraries(bitcoin_sv2
diff --git a/src/sv2/template_provider.cpp b/src/sv2/template_provider.cpp
new file mode 100644
index 0000000000000..eb22e8d9f8a63
--- /dev/null
+++ b/src/sv2/template_provider.cpp
@@ -0,0 +1,84 @@
+#include <sv2/template_provider.h>
+
+#include <base58.h>
+#include <crypto/hex_base.h>
+#include <common/args.h>
+#include <logging.h>
+#include <sv2/noise.h>
+#include <util/strencodings.h>
+#include <util/thread.h>
+
+Sv2TemplateProvider::Sv2TemplateProvider(interfaces::Mining& mining) : m_mining{mining}
+{
+    // TODO: persist static key
+    CKey static_key;
+    static_key.MakeNewKey(true);
+
+    auto authority_key{GenerateRandomKey()};
+
+    // SRI uses base58 encoded x-only pubkeys in its configuration files
+    std::array<unsigned char, 34> version_pubkey_bytes;
+    version_pubkey_bytes[0] = 1;
+    version_pubkey_bytes[1] = 0;
+    m_authority_pubkey = XOnlyPubKey(authority_key.GetPubKey());
+    std::copy(m_authority_pubkey.begin(), m_authority_pubkey.end(), version_pubkey_bytes.begin() + 2);
+    LogInfo("Template Provider authority key: %s\n", EncodeBase58Check(version_pubkey_bytes));
+    LogTrace(BCLog::SV2, "Authority key: %s\n", HexStr(m_authority_pubkey));
+
+    // Generate and sign certificate
+    auto now{GetTime<std::chrono::seconds>()};
+    uint16_t version = 0;
+    // Start validity a little bit in the past to account for clock difference
+    uint32_t valid_from = static_cast<uint32_t>(std::chrono::duration_cast<std::chrono::seconds>(now).count()) - 3600;
+    uint32_t valid_to =  std::numeric_limits<unsigned int>::max(); // 2106
+    Sv2SignatureNoiseMessage certificate = Sv2SignatureNoiseMessage(version, valid_from, valid_to, XOnlyPubKey(static_key.GetPubKey()), authority_key);
+
+    // TODO: persist certificate
+
+    m_connman = std::make_unique<Sv2Connman>(TP_SUBPROTOCOL, static_key, m_authority_pubkey, certificate);
+
+    // Suppress unused variable warning, result is unused.
+    m_mining.getTip();
+}
+
+bool Sv2TemplateProvider::Start(const Sv2TemplateProviderOptions& options)
+{
+    m_options = options;
+
+    if (!m_connman->Start(this, m_options.host, m_options.port)) {
+        return false;
+    }
+
+    m_thread_sv2_handler = std::thread(&util::TraceThread, "sv2", [this] { ThreadSv2Handler(); });
+    return true;
+}
+
+Sv2TemplateProvider::~Sv2TemplateProvider()
+{
+    AssertLockNotHeld(m_tp_mutex);
+
+    m_connman->Interrupt();
+    m_connman->StopThreads();
+
+    Interrupt();
+    StopThreads();
+}
+
+void Sv2TemplateProvider::Interrupt()
+{
+    m_flag_interrupt_sv2 = true;
+}
+
+void Sv2TemplateProvider::StopThreads()
+{
+    if (m_thread_sv2_handler.joinable()) {
+        m_thread_sv2_handler.join();
+    }
+}
+
+void Sv2TemplateProvider::ThreadSv2Handler()
+{
+    while (!m_flag_interrupt_sv2) {
+        // TODO: handle messages
+    }
+}
diff --git a/src/sv2/template_provider.h b/src/sv2/template_provider.h
new file mode 100644
index 0000000000000..64612d65a6a6e
--- /dev/null
+++ b/src/sv2/template_provider.h
@@ -0,0 +1,106 @@
+#ifndef BITCOIN_SV2_TEMPLATE_PROVIDER_H
+#define BITCOIN_SV2_TEMPLATE_PROVIDER_H
+
+#include <interfaces/mining.h>
+#include <sv2/connman.h>
+#include <sv2/messages.h>
+#include <logging.h>
+#include <net.h>
+#include <util/sock.h>
+#include <util/time.h>
+#include <streams.h>
+
+struct Sv2TemplateProviderOptions
+{
+    /**
+     * Running inside a test
+     */
+    bool is_test{false};
+
+    /**
+     * Host for the server to bind to.
+     */
+    std::string host{"127.0.0.1"};
+
+    /**
+     * The listening port for the server.
+     */
+    uint16_t port{8336};
+};
+
+/**
+ * The main class that runs the template provider server.
+ */
+class Sv2TemplateProvider : public Sv2EventsInterface
+{
+
+private:
+    /**
+    * The Mining interface is used to build new valid blocks, get the best known
+    * block hash and to check whether the node is still in IBD.
+    */
+    interfaces::Mining& m_mining;
+
+    /*
+     * The template provider subprotocol used in setup connection messages. The stratum v2
+     * template provider only recognizes its own subprotocol.
+     */
+    static constexpr uint8_t TP_SUBPROTOCOL{0x02};
+
+    std::unique_ptr<Sv2Connman> m_connman;
+
+    /**
+    * Configuration
+    */
+    Sv2TemplateProviderOptions m_options;
+
+    /**
+     * The main thread for the template provider.
+     */
+    std::thread m_thread_sv2_handler;
+
+    /**
+     * Signal for handling interrupts and stopping the template provider event loop.
+     */
+    std::atomic<bool> m_flag_interrupt_sv2{false};
+    CThreadInterrupt m_interrupt_sv2;
+
+public:
+    explicit Sv2TemplateProvider(interfaces::Mining& mining);
+
+    ~Sv2TemplateProvider() EXCLUSIVE_LOCKS_REQUIRED(!m_tp_mutex);
+
+    Mutex m_tp_mutex;
+
+    /**
+     * Starts the template provider server and thread.
+     * returns false if port is unable to bind.
+     */
+    [[nodiscard]] bool Start(const Sv2TemplateProviderOptions& options = {});
+
+    /**
+     * The main thread for the template provider, contains an event loop handling
+     * all tasks for the template provider.
+     */
+    void ThreadSv2Handler();
+
+    /**
+     * Triggered on interrupt signals to stop the main event loop in ThreadSv2Handler().
+     */
+    void Interrupt();
+
+    /**
+     * Tear down of the template provider thread and any other necessary tear down.
+     */
+    void StopThreads();
+
+    /**
+     * Main handler for all received stratum v2 messages.
+     */
+    void ProcessSv2Message(const node::Sv2NetMsg& sv2_header, Sv2Client& client);
+
+    // Only used for tests
+    XOnlyPubKey m_authority_pubkey;
+};
+
+#endif // BITCOIN_SV2_TEMPLATE_PROVIDER_H
diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt
index ff042558d6151..3a4b2e28feabf 100644
--- a/src/test/CMakeLists.txt
+++ b/src/test/CMakeLists.txt
@@ -183,6 +183,7 @@ if(WITH_SV2)
     sv2_noise_tests.cpp
     sv2_transport_tests.cpp
     sv2_messages_tests.cpp
+    sv2_template_provider_tests.cpp
   )
   target_link_libraries(test_bitcoin bitcoin_sv2)
 endif()
diff --git a/src/test/sv2_template_provider_tests.cpp b/src/test/sv2_template_provider_tests.cpp
new file mode 100644
index 0000000000000..cae9162545172
--- /dev/null
+++ b/src/test/sv2_template_provider_tests.cpp
@@ -0,0 +1,153 @@
+#include <boost/test/unit_test.hpp>
+#include <interfaces/mining.h>
+#include <sv2/messages.h>
+#include <sv2/template_provider.h>
+#include <test/util/net.h>
+#include <test/util/setup_common.h>
+#include <util/sock.h>
+
+#include <memory>
+
+BOOST_FIXTURE_TEST_SUITE(sv2_template_provider_tests, TestChain100Setup)
+
+/**
+  * A class for testing the Template Provider. Each TPTester encapsulates a
+  * Sv2TemplateProvider (the one being tested) as well as a Sv2Cipher
+  * to act as the other side.
+  */
+class TPTester {
+private:
+    std::unique_ptr<Sv2Transport> m_peer_transport; //!< Transport for peer
+    // Sockets that will be returned by the TP listening socket Accept() method.
+    std::shared_ptr<DynSock::Queue> m_tp_accepted_sockets{std::make_shared<DynSock::Queue>()};
+    std::shared_ptr<DynSock::Pipes> m_current_client_pipes;
+
+public:
+    std::unique_ptr<Sv2TemplateProvider> m_tp; //!< Sv2TemplateProvider being tested
+    Sv2TemplateProviderOptions m_tp_options{.is_test = true}; //! Options passed to the TP
+
+    TPTester(interfaces::Mining& mining)
+    {
+        m_tp = std::make_unique<Sv2TemplateProvider>(mining);
+
+        CreateSock = [this](int, int, int) -> std::unique_ptr<Sock> {
+            // This will be the bind/listen socket from m_tp. It will
+            // create other sockets via its Accept() method.
+            return std::make_unique<DynSock>(std::make_shared<DynSock::Pipes>(), m_tp_accepted_sockets);
+        };
+
+        BOOST_REQUIRE(m_tp->Start(m_tp_options));
+    }
+
+    void SendPeerBytes()
+    {
+        const auto& [data, more, _m_message_type] = m_peer_transport->GetBytesToSend(/*have_next_message=*/false);
+        BOOST_REQUIRE(data.size() > 0);
+
+        // Schedule data to be returned by the next Recv() call from
+        // Sv2Connman on the socket it has accepted.
+        m_current_client_pipes->recv.PushBytes(data.data(), data.size());
+        m_peer_transport->MarkBytesSent(data.size());
+    }
+
+    // Have the peer receive and process bytes:
+    size_t PeerReceiveBytes()
+    {
+        uint8_t buf[0x10000];
+        // Get the data that has been written to the accepted socket with Send() by TP.
+        // Wait until the bytes appear in the "send" pipe.
+        ssize_t n;
+        for (;;) {
+            n = m_current_client_pipes->send.GetBytes(buf, sizeof(buf), 0);
+            if (n != -1 || errno != EAGAIN) {
+                break;
+            }
+            UninterruptibleSleep(50ms);
+        }
+
+        // Inform client's transport that some bytes have been received (sent by TP).
+        if (n > 0) {
+            Span<const uint8_t> s(buf, n);
+            BOOST_REQUIRE(m_peer_transport->ReceivedBytes(s));
+        }
+
+        return n;
+    }
+
+    /* Create a new client and perform handshake */
+    void handshake()
+    {
+        m_peer_transport.reset();
+
+        auto peer_static_key{GenerateRandomKey()};
+        m_peer_transport = std::make_unique<Sv2Transport>(std::move(peer_static_key), m_tp->m_authority_pubkey);
+
+        // Have Sv2Connman's listen socket's Accept() simulate a newly arrived connection.
+        m_current_client_pipes = std::make_shared<DynSock::Pipes>();
+        m_tp_accepted_sockets->Push(
+            std::make_unique<DynSock>(m_current_client_pipes, std::make_shared<DynSock::Queue>()));
+
+        // Flush transport for handshake part 1
+        SendPeerBytes();
+
+        // Read handshake part 2 from transport
+        BOOST_REQUIRE_EQUAL(PeerReceiveBytes(), Sv2HandshakeState::HANDSHAKE_STEP2_SIZE);
+    }
+
+    void receiveMessage(Sv2NetMsg& msg)
+    {
+        // Client encrypts message and puts it on the transport:
+        CSerializedNetMsg net_msg{std::move(msg)};
+        BOOST_REQUIRE(m_peer_transport->SetMessageToSend(net_msg));
+        SendPeerBytes();
+    }
+
+    Sv2NetMsg SetupConnectionMsg()
+    {
+        std::vector<uint8_t> bytes{
+            0x02,                                                 // protocol
+            0x02, 0x00,                                           // min_version
+            0x02, 0x00,                                           // max_version
+            0x01, 0x00, 0x00, 0x00,                               // flags
+            0x07, 0x30, 0x2e, 0x30, 0x2e, 0x30, 0x2e, 0x30,       // endpoint_host
+            0x61, 0x21,                                           // endpoint_port
+            0x07, 0x42, 0x69, 0x74, 0x6d, 0x61, 0x69, 0x6e,       // vendor
+            0x08, 0x53, 0x39, 0x69, 0x20, 0x31, 0x33, 0x2e, 0x35, // hardware_version
+            0x1c, 0x62, 0x72, 0x61, 0x69, 0x69, 0x6e, 0x73, 0x2d, 0x6f, 0x73, 0x2d, 0x32, 0x30,
+            0x31, 0x38, 0x2d, 0x30, 0x39, 0x2d, 0x32, 0x32, 0x2d, 0x31, 0x2d, 0x68, 0x61, 0x73,
+            0x68, // firmware
+            0x10, 0x73, 0x6f, 0x6d, 0x65, 0x2d, 0x64, 0x65, 0x76, 0x69, 0x63, 0x65, 0x2d, 0x75,
+            0x75, 0x69, 0x64, // device_id
+        };
+
+        return node::Sv2NetMsg{node::Sv2MsgType::SETUP_CONNECTION, std::move(bytes)};
+    }
+};
+
+BOOST_AUTO_TEST_CASE(client_tests)
+{
+    auto mining{interfaces::MakeMining(m_node)};
+    TPTester tester{*mining};
+
+    tester.handshake();
+
+    // After the handshake the client must send a SetupConnection message to the
+    // Template Provider.
+
+    tester.handshake();
+    BOOST_TEST_MESSAGE("Handshake done, send SetupConnectionMsg");
+
+    node::Sv2NetMsg setup{tester.SetupConnectionMsg()};
+    tester.receiveMessage(setup);
+    // SetupConnection.Success is 6 bytes
+    BOOST_REQUIRE_EQUAL(tester.PeerReceiveBytes(), SV2_HEADER_ENCRYPTED_SIZE + 6 + Poly1305::TAGLEN);
+
+    std::vector<uint8_t> coinbase_output_max_additional_size_bytes{
+        0x01, 0x00, 0x00, 0x00
+    };
+    node::Sv2NetMsg msg{node::Sv2MsgType::COINBASE_OUTPUT_DATA_SIZE, std::move(coinbase_output_max_additional_size_bytes)};
+    // No reply expected, not yet implemented
+    tester.receiveMessage(msg);
+}
+
+BOOST_AUTO_TEST_SUITE_END()

From f31e6a39aa4a5b5358d0b16dcbe5c9d81f140adc Mon Sep 17 00:00:00 2001
From: Sjors Provoost <sjors@sprovoost.nl>
Date: Wed, 29 Nov 2023 18:06:59 +0100
Subject: [PATCH 04/11] Chainparams: add default sv2 port

Co-authored-by: Christopher Coverdale <chris.coverdale24@gmail.com>
---
 src/chainparamsbase.cpp | 10 +++++-----
 src/chainparamsbase.h   |  6 ++++--
 2 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/src/chainparamsbase.cpp b/src/chainparamsbase.cpp
index 060d519d9207c..06793e1ebd18a 100644
--- a/src/chainparamsbase.cpp
+++ b/src/chainparamsbase.cpp
@@ -41,15 +41,15 @@ std::unique_ptr<CBaseChainParams> CreateBaseChainParams(const ChainType chain)
 {
     switch (chain) {
     case ChainType::MAIN:
-        return std::make_unique<CBaseChainParams>("", 8332);
+        return std::make_unique<CBaseChainParams>("", 8332, 8336);
     case ChainType::TESTNET:
-        return std::make_unique<CBaseChainParams>("testnet3", 18332);
+        return std::make_unique<CBaseChainParams>("testnet3", 18332, 18336);
     case ChainType::TESTNET4:
-        return std::make_unique<CBaseChainParams>("testnet4", 48332);
+        return std::make_unique<CBaseChainParams>("testnet4", 48332, 48336);
     case ChainType::SIGNET:
-        return std::make_unique<CBaseChainParams>("signet", 38332);
+        return std::make_unique<CBaseChainParams>("signet", 38332, 38336);
     case ChainType::REGTEST:
-        return std::make_unique<CBaseChainParams>("regtest", 18443);
+        return std::make_unique<CBaseChainParams>("regtest", 18443, 18447);
     }
     assert(false);
 }
diff --git a/src/chainparamsbase.h b/src/chainparamsbase.h
index adbd6a5174695..d7decc39ea18d 100644
--- a/src/chainparamsbase.h
+++ b/src/chainparamsbase.h
@@ -22,13 +22,15 @@ class CBaseChainParams
 public:
     const std::string& DataDir() const { return strDataDir; }
     uint16_t RPCPort() const { return m_rpc_port; }
+    uint16_t Sv2Port() const { return m_sv2_port; }
 
     CBaseChainParams() = delete;
-    CBaseChainParams(const std::string& data_dir, uint16_t rpc_port)
-        : m_rpc_port(rpc_port), strDataDir(data_dir) {}
+    CBaseChainParams(const std::string& data_dir, uint16_t rpc_port, uint16_t sv2_port)
+        : m_rpc_port(rpc_port), m_sv2_port(sv2_port), strDataDir(data_dir) {}
 
 private:
     const uint16_t m_rpc_port;
+    const uint16_t m_sv2_port;
     std::string strDataDir;
 };
 

From 00fe8d89b90309a8eff90bbe0e759d4e0445fd68 Mon Sep 17 00:00:00 2001
From: Sjors Provoost <sjors@sprovoost.nl>
Date: Mon, 24 Jun 2024 14:22:05 +0200
Subject: [PATCH 05/11] Add remaining sv2 messages for TemplateProvider

---
 src/sv2/CMakeLists.txt          |   1 +
 src/sv2/messages.cpp            |  41 ++++
 src/sv2/messages.h              | 352 ++++++++++++++++++++++++++++++++
 src/test/sv2_messages_tests.cpp | 135 ++++++++++++
 4 files changed, 529 insertions(+)
 create mode 100644 src/sv2/messages.cpp

diff --git a/src/sv2/CMakeLists.txt b/src/sv2/CMakeLists.txt
index 7b632032c233f..0504ea2a5c51f 100644
--- a/src/sv2/CMakeLists.txt
+++ b/src/sv2/CMakeLists.txt
@@ -6,6 +6,7 @@ add_library(bitcoin_sv2 STATIC EXCLUDE_FROM_ALL
   noise.cpp
   transport.cpp
   connman.cpp
+  messages.cpp
   template_provider.cpp
 )
 
diff --git a/src/sv2/messages.cpp b/src/sv2/messages.cpp
new file mode 100644
index 0000000000000..c5881ea8983fc
--- /dev/null
+++ b/src/sv2/messages.cpp
@@ -0,0 +1,41 @@
+#include <sv2/messages.h>
+
+#include <arith_uint256.h>
+#include <primitives/block.h>
+#include <primitives/transaction.h>
+
+node::Sv2NewTemplateMsg::Sv2NewTemplateMsg(const CBlockHeader& header, const CTransactionRef coinbase_tx, std::vector<uint256> coinbase_merkle_path, int witness_commitment_index, uint64_t template_id, bool future_template)
+    : m_template_id{template_id}, m_future_template{future_template}
+{
+    m_version = header.nVersion;
+
+    m_coinbase_tx_version = coinbase_tx->CURRENT_VERSION;
+    m_coinbase_prefix = coinbase_tx->vin[0].scriptSig;
+    m_coinbase_tx_input_sequence = coinbase_tx->vin[0].nSequence;
+
+    // The coinbase nValue already contains the nFee + the Block Subsidy when built using CreateBlock().
+    m_coinbase_tx_value_remaining = static_cast<uint64_t>(coinbase_tx->vout[0].nValue);
+
+    m_coinbase_tx_outputs_count = 0;
+    if (witness_commitment_index != NO_WITNESS_COMMITMENT) {
+        m_coinbase_tx_outputs_count = 1;
+
+        std::vector<CTxOut> coinbase_tx_outputs{coinbase_tx->vout[witness_commitment_index]};
+        m_coinbase_tx_outputs = coinbase_tx_outputs;
+    }
+
+    m_coinbase_tx_locktime = coinbase_tx->nLockTime;
+
+    for (const auto& hash : coinbase_merkle_path) {
+        m_merkle_path.push_back(hash);
+    }
+
+}
+
+node::Sv2SetNewPrevHashMsg::Sv2SetNewPrevHashMsg(const CBlockHeader& header, uint64_t template_id) : m_template_id{template_id}
+{
+    m_prev_hash = header.hashPrevBlock;
+    m_header_timestamp = header.nTime;
+    m_nBits = header.nBits;
+    m_target = ArithToUint256(arith_uint256().SetCompact(header.nBits));
+}
diff --git a/src/sv2/messages.h b/src/sv2/messages.h
index 29dd5cc2b437e..17a9d8ca023e2 100644
--- a/src/sv2/messages.h
+++ b/src/sv2/messages.h
@@ -35,6 +35,12 @@ enum class Sv2MsgType : uint8_t {
     SETUP_CONNECTION = 0x00,
     SETUP_CONNECTION_SUCCESS = 0x01,
     SETUP_CONNECTION_ERROR = 0x02,
+    NEW_TEMPLATE = 0x71,
+    SET_NEW_PREV_HASH = 0x72,
+    REQUEST_TRANSACTION_DATA = 0x73,
+    REQUEST_TRANSACTION_DATA_SUCCESS = 0x74,
+    REQUEST_TRANSACTION_DATA_ERROR = 0x75,
+    SUBMIT_SOLUTION = 0x76,
     COINBASE_OUTPUT_DATA_SIZE = 0x70,
 };
 
@@ -42,6 +48,12 @@ static const std::map<Sv2MsgType, std::string> SV2_MSG_NAMES{
     {Sv2MsgType::SETUP_CONNECTION, "SetupConnection"},
     {Sv2MsgType::SETUP_CONNECTION_SUCCESS, "SetupConnectionSuccess"},
     {Sv2MsgType::SETUP_CONNECTION_ERROR, "SetupConnectionError"},
+    {Sv2MsgType::NEW_TEMPLATE, "NewTemplate"},
+    {Sv2MsgType::SET_NEW_PREV_HASH, "SetNewPrevHash"},
+    {Sv2MsgType::REQUEST_TRANSACTION_DATA, "RequestTransactionData"},
+    {Sv2MsgType::REQUEST_TRANSACTION_DATA_SUCCESS, "RequestTransactionData.Success"},
+    {Sv2MsgType::REQUEST_TRANSACTION_DATA_ERROR, "RequestTransactionData.Error"},
+    {Sv2MsgType::SUBMIT_SOLUTION, "SubmitSolution"},
     {Sv2MsgType::COINBASE_OUTPUT_DATA_SIZE, "CoinbaseOutputDataSize"},
 };
 
@@ -213,6 +225,346 @@ struct Sv2SetupConnectionErrorMsg
     }
 };
 
+/**
+ * The work template for downstream devices. Can be used for future work or immediate work.
+ * The NewTemplate will be matched to a cached block using the template id.
+ */
+struct Sv2NewTemplateMsg
+{
+    /**
+     * The default message type value for this Stratum V2 message.
+     */
+    static constexpr auto m_msg_type = Sv2MsgType::NEW_TEMPLATE;
+
+    /**
+     * Server’s identification of the template. Strictly increasing, the current UNIX
+     * time may be used in place of an ID.
+     */
+    uint64_t m_template_id;
+
+    /**
+     * True if the template is intended for future SetNewPrevHash message sent on the channel.
+     * If False, the job relates to the last sent SetNewPrevHash message on the channel
+     * and the miner should start to work on the job immediately.
+     */
+    bool m_future_template;
+
+    /**
+     * Valid header version field that reflects the current network consensus.
+     * The general purpose bits (as specified in BIP320) can be freely manipulated
+     * by the downstream node. The downstream node MUST NOT rely on the upstream
+     * node to set the BIP320 bits to any particular value.
+     */
+    uint32_t m_version;
+
+    /**
+     * The coinbase transaction nVersion field.
+     */
+    uint32_t m_coinbase_tx_version;
+
+    /**
+     * Up to 8 bytes (not including the length byte) which are to be placed at
+     * the beginning of the coinbase field in the coinbase transaction.
+     */
+    CScript m_coinbase_prefix;
+
+    /**
+     * The coinbase transaction input’s nSequence field.
+     */
+    uint32_t m_coinbase_tx_input_sequence;
+
+    /**
+     * The value, in satoshis, available for spending in coinbase outputs added
+     * by the client. Includes both transaction fees and block subsidy.
+     */
+    uint64_t m_coinbase_tx_value_remaining;
+
+    /**
+     * The number of transaction outputs included in coinbase_tx_outputs.
+     */
+    uint32_t m_coinbase_tx_outputs_count;
+
+    /**
+     * Bitcoin transaction outputs to be included as the last outputs in the coinbase transaction.
+     */
+    std::vector<CTxOut> m_coinbase_tx_outputs;
+
+    /**
+     * The locktime field in the coinbase transaction.
+     */
+    uint32_t m_coinbase_tx_locktime;
+
+    /**
+     * Merkle path hashes ordered from deepest.
+     */
+    std::vector<uint256> m_merkle_path;
+
+    Sv2NewTemplateMsg() = default;
+    explicit Sv2NewTemplateMsg(const CBlockHeader& header, const CTransactionRef coinbase_tx, std::vector<uint256> coinbase_merkle_path, int witness_commitment_index, uint64_t template_id, bool future_template);
+
+    template <typename Stream>
+    void Serialize(Stream& s) const
+    {
+        s << m_template_id
+          << m_future_template
+          << m_version
+          << m_coinbase_tx_version
+          << m_coinbase_prefix
+          << m_coinbase_tx_input_sequence
+          << m_coinbase_tx_value_remaining
+          << m_coinbase_tx_outputs_count;
+
+        // If there are more than 0 coinbase tx outputs, then we need to serialize them
+        // as [B0_64K](https://github.com/stratum-mining/sv2-spec/blob/main/03-Protocol-Overview.md#31-data-types-mapping)
+        if (m_coinbase_tx_outputs_count > 0) {
+            std::vector<uint8_t> outputs_bytes;
+            VectorWriter{outputs_bytes, 0, m_coinbase_tx_outputs.at(0)};
+
+            s << static_cast<uint16_t>(outputs_bytes.size());
+            s.write(MakeByteSpan(outputs_bytes));
+        } else {
+            // We will still need to send 2 bytes indicating an empty coinbase-tx_outputs array as a B0_64K.
+            s << static_cast<uint16_t>(0);
+        }
+
+        s << m_coinbase_tx_locktime
+          << m_merkle_path;
+    }
+};
+
+/**
+ * When the template provider creates a new valid best block, the template provider
+ * MUST immediately send the SetNewPrevHash message. This message can also be used
+ * for a future template, indicating the client can begin work on a previously
+ * received and cached NewTemplate which contains the same template id.
+ */
+struct Sv2SetNewPrevHashMsg
+{
+    /**
+     * The default message type value for this Stratum V2 message.
+     */
+    static constexpr auto m_msg_type = Sv2MsgType::SET_NEW_PREV_HASH;
+
+    /**
+     * The id referenced in a previous NewTemplate message.
+     */
+    uint64_t m_template_id;
+
+    /**
+     * Previous block’s hash, as it must appear in the next block’s header.
+     */
+    uint256 m_prev_hash;
+
+    /**
+     * The nTime field in the block header at which the client should start (usually current time).
+     * This is NOT the minimum valid nTime value.
+     */
+    uint32_t m_header_timestamp;
+
+    /**
+     * Block header field.
+     */
+    uint32_t m_nBits;
+
+    /**
+     * The maximum double-SHA256 hash value which would represent a valid block.
+     * Note that this may be lower than the target implied by nBits in several cases,
+     * including weak-block based block propagation.
+     */
+    uint256 m_target;
+
+    Sv2SetNewPrevHashMsg() = default;
+    explicit Sv2SetNewPrevHashMsg(const CBlockHeader& header, uint64_t template_id);
+
+    template <typename Stream>
+    void Serialize(Stream& s) const
+    {
+        s << m_template_id
+          << m_prev_hash
+          << m_header_timestamp
+          << m_nBits
+          << m_target;
+    }
+};
+
+/**
+ * The client (usually a Job Negotiator) sends a RequestTransactionData message
+ * to the Template Provider asking for the full set of transaction data (excluding
+ * the coinbase) in the block and any additional data relevant for validation
+ * associated with the template_id.
+ */
+struct Sv2RequestTransactionDataMsg
+{
+    /**
+     * The default message type value for this Stratum V2 message.
+     */
+    static constexpr auto m_msg_type = Sv2MsgType::REQUEST_TRANSACTION_DATA;
+
+    /**
+     * The template_id corresponding to a NewTemplate message.
+     */
+    uint64_t m_template_id;
+
+    Sv2RequestTransactionDataMsg() = default;
+
+    template <typename Stream>
+    void Unserialize(Stream& s)
+    {
+        s >> m_template_id;
+    }
+};
+
+/**
+ * A message for a successful request for transaction data. It contains the full
+ * serialized transaction data from a NewTemplate according to the id.
+ */
+struct Sv2RequestTransactionDataSuccessMsg
+{
+    /**
+     * The default message type value for this Stratum V2 message.
+     */
+    static constexpr auto m_msg_type = Sv2MsgType::REQUEST_TRANSACTION_DATA_SUCCESS;
+
+    /**
+     * The template_id corresponding to a NewTemplate message.
+     */
+    uint64_t m_template_id;
+
+    /**
+     * Extra data which the Pool may require to validate the work
+     */
+    std::vector<uint8_t> m_excess_data;
+
+    /**
+     * List of full transactions requested by client found in the
+     * corresponding template.
+     */
+    std::vector<CTransactionRef> m_transactions_list;
+
+    Sv2RequestTransactionDataSuccessMsg() = default;
+
+    explicit Sv2RequestTransactionDataSuccessMsg(uint64_t template_id, std::vector<uint8_t>&& excess_data, std::vector<CTransactionRef>&& transactions_list) : m_template_id{template_id}, m_excess_data{excess_data}, m_transactions_list{transactions_list} {};
+
+    template <typename Stream>
+    void Serialize(Stream& s) const
+    {
+        s << m_template_id;
+
+        // excess data is expected to be serialized as a B0_64K type.
+        if (m_excess_data.empty()) {
+            s << static_cast<uint16_t>(0);
+        } else {
+            s << static_cast<uint16_t>(m_excess_data.size());
+            s.write(MakeByteSpan(m_excess_data));
+        }
+
+        // transactions list is expected to be serialized as a SEQ0_64K[B0_16M].
+        s << static_cast<uint16_t>(m_transactions_list.size());
+        for (const auto& tx : m_transactions_list) {
+            DataStream ss_tx{};
+            ss_tx << TX_WITH_WITNESS(*tx);
+            auto tx_size = static_cast<uint32_t>(ss_tx.size());
+
+            u24_t tx_byte_len;
+            tx_byte_len[2] = (tx_size >> 16) & 0xff;
+            tx_byte_len[1] = (tx_size >> 8) & 0xff;
+            tx_byte_len[0] = tx_size & 0xff;
+
+            s << tx_byte_len;
+            s.write(MakeByteSpan(ss_tx));
+        }
+    };
+};
+
+/**
+ * The error message for the client if the template provider is unable to send
+ * the full serialized transaction data.
+ */
+struct Sv2RequestTransactionDataErrorMsg
+{
+    /**
+     * The default message type value for this Stratum V2 message.
+     */
+    static constexpr auto m_msg_type = Sv2MsgType::REQUEST_TRANSACTION_DATA_ERROR;
+
+    /**
+     * The template_id corresponding to a NewTemplate/RequestTransactionData message.
+     */
+    uint64_t m_template_id;
+
+    /**
+     * Human-readable error code on why no transaction data has been provided.
+     */
+    std::string m_error_code;
+
+    explicit Sv2RequestTransactionDataErrorMsg(uint64_t template_id, std::string&& error_code) : m_template_id{template_id}, m_error_code{error_code} {};
+
+    template <typename Stream>
+    void Serialize(Stream& s) const
+    {
+        s << m_template_id
+          << m_error_code;
+    }
+};
+
+/**
+ * The client sends a SubmitSolution after finding a coinbase transaction/nonce
+ * pair which double-SHA256 hashes at or below SetNewPrevHash::target. The template provider
+ * finds the cached block according to the template id and reconstructs the block with the
+ * values from SubmitSolution. The template provider must then propagate the block to the
+ * Bitcoin Network.
+ */
+struct Sv2SubmitSolutionMsg
+{
+    /**
+     * The default message type value for this Stratum V2 message.
+     */
+    static constexpr auto m_msg_type = Sv2MsgType::SUBMIT_SOLUTION;
+
+    /**
+     * The id referenced in a NewTemplate.
+     */
+    uint64_t m_template_id;
+
+    /**
+     * The version field in the block header. Bits not defined by BIP320 as additional
+     * nonce MUST be the same as they appear in the NewWork message, other bits may
+     * be set to any value.
+     */
+    uint32_t m_version;
+
+    /**
+     * The nTime field in the block header. This MUST be greater than or equal to
+     * the header_timestamp field in the latest SetNewPrevHash message and lower
+     * than or equal to that value plus the number of seconds since the receipt
+     * of that message.
+     */
+    uint32_t m_header_timestamp;
+
+    /**
+     * The nonce field in the header.
+     */
+    uint32_t m_header_nonce;
+
+    /**
+     * The full serialized coinbase transaction, meeting all the requirements of the NewWork message, above.
+     */
+    CMutableTransaction m_coinbase_tx;
+
+    Sv2SubmitSolutionMsg() = default;
+
+    template <typename Stream>
+    void Unserialize(Stream& s)
+    {
+        s >> m_template_id >> m_version >> m_header_timestamp >> m_header_nonce;
+
+        // Ignore the 2 byte length as the rest of the stream is assumed to be
+        // the m_coinbase_tx.
+        s.ignore(2);
+        s >> TX_WITH_WITNESS(m_coinbase_tx);
+    }
+};
+
 /**
  * Header for all stratum v2 messages. Each header must contain the message type,
  * the length of the serialized message and a 2 byte extension field currently
diff --git a/src/test/sv2_messages_tests.cpp b/src/test/sv2_messages_tests.cpp
index 68d1a7009b500..70ef73ca6e85a 100644
--- a/src/test/sv2_messages_tests.cpp
+++ b/src/test/sv2_messages_tests.cpp
@@ -97,4 +97,139 @@ BOOST_AUTO_TEST_CASE(Sv2SetupConnectionError_test)
     BOOST_CHECK_EQUAL(HexStr(ss), expected);
 }
 
+
+BOOST_AUTO_TEST_CASE(Sv2NewTemplate_test)
+{
+    // 0100000000000000 - template_id
+    // 00 - future_template
+    // 00000030 - version
+    // 02000000 - coinbase tx version
+    // 04 - coinbase_prefix len
+    // 03012100 - coinbase prefix
+    // ffffffff - coinbase tx input sequence
+    // 0000000000000000 - coinbase tx value remaining
+    // 00000000 - coinbase tx outputs count
+    // 0000 - coinbase t outputs
+    // 01 - merkle path length
+    // 1a6240823de4c8d6aaf826851bdf2b0e8d5acf7c31e8578cff4c394b5a32bd4e - merkle path
+    std::string expected{"01000000000000000000000030020000000403012100ffffffff000000000000000000000000000000000000011a6240823de4c8d6aaf826851bdf2b0e8d5acf7c31e8578cff4c394b5a32bd4e"};
+
+    node::Sv2NewTemplateMsg new_template;
+    new_template.m_template_id = 1;
+    new_template.m_future_template = false;
+    new_template.m_version = 805306368;
+    new_template.m_coinbase_tx_version = 2;
+
+    std::vector<uint8_t> coinbase_prefix{0x03, 0x01, 0x21, 0x00};
+    CScript prefix(coinbase_prefix.begin(), coinbase_prefix.end());
+    new_template.m_coinbase_prefix = prefix;
+
+    new_template.m_coinbase_tx_input_sequence = 4294967295;
+    new_template.m_coinbase_tx_value_remaining = 0;
+    new_template.m_coinbase_tx_outputs_count = 0;
+
+    std::vector<CTxOut> coinbase_tx_ouputs;
+    new_template.m_coinbase_tx_outputs = coinbase_tx_ouputs;
+
+    new_template.m_coinbase_tx_locktime = 0;
+
+    std::vector<uint256> merkle_path;
+    CMutableTransaction mtx_tx;
+    CTransaction tx{mtx_tx};
+    merkle_path.push_back(tx.GetHash());
+    new_template.m_merkle_path = merkle_path;
+
+    DataStream ss{};
+    ss << new_template;
+
+    BOOST_CHECK_EQUAL(HexStr(ss), expected);
+}
+
+BOOST_AUTO_TEST_CASE(Sv2NetHeader_NewTemplate_test)
+{
+    // 0000 - extension type
+    // 71 - msg type (NewTemplate)
+    // 000000 - msg length
+    std::string expected{"000071000000"};
+    node::Sv2NetHeader sv2_header{node::Sv2MsgType::NEW_TEMPLATE, 0};
+
+    DataStream ss{};
+    ss << sv2_header;
+
+    BOOST_CHECK_EQUAL(HexStr(ss), expected);
+}
+
+
+BOOST_AUTO_TEST_CASE(Sv2SetNewPrevHash_test)
+{
+    // 0200000000000000 - template_id
+    // e59a2ef8d4826b89fe637f3b 5322c0f167fd2d3dc88ec568a7c2c8ffd8eb8873 - prev hash
+    // 973c0e63 - header timestamp
+    // ffff7f03 - nbits
+    // ffff7f0000000000000000000000000000000000000000000000000000000000 - target
+    std::string expected{"0200000000000000e59a2ef8d4826b89fe637f3b5322c0f167fd2d3dc88ec568a7c2c8ffd8eb8873973c0e63ffff7f03ffff7f0000000000000000000000000000000000000000000000000000000000"};
+
+    std::vector<uint8_t> prev_hash_input{
+        0xe5, 0x9a, 0x2e, 0xf8, 0xd4, 0x82, 0x6b, 0x89, 0xfe, 0x63, 0x7f, 0x3b,
+        0x53, 0x22, 0xc0, 0xf1, 0x67, 0xfd, 0x2d, 0x3d, 0xc8, 0x8e, 0xc5, 0x68,
+        0xa7, 0xc2, 0xc8, 0xff, 0xd8, 0xeb, 0x88, 0x73};
+    uint256 prev_hash = uint256(prev_hash_input);
+
+    CBlockHeader header;
+    header.hashPrevBlock = prev_hash;
+    header.nTime = 1661877399;
+    header.nBits = 58720255;
+
+    node::Sv2SetNewPrevHashMsg new_prev_hash{header, 2};
+
+    DataStream ss{};
+    ss << new_prev_hash;
+
+    BOOST_CHECK_EQUAL(HexStr(ss), expected);
+}
+
+BOOST_AUTO_TEST_CASE(Sv2NetHeader_SetNewPrevHash_test)
+{
+    // 0000 - extension type
+    // 72 - msg type (SetNewPrevHash)
+    // 000000 - msg length
+    std::string expected{"000072000000"};
+    node::Sv2NetHeader sv2_header{node::Sv2MsgType::SET_NEW_PREV_HASH, 0};
+
+    DataStream ss{};
+    ss << sv2_header;
+
+    BOOST_CHECK_EQUAL(HexStr(ss), expected);
+}
+
+BOOST_AUTO_TEST_CASE(Sv2SubmitSolution_test)
+{
+    uint8_t input[]{
+        0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,   // template_id
+        0x02, 0x00, 0x00, 0x00,                           // version
+        0x97, 0x3c, 0x0e, 0x63,                           // header_timestamp
+        0xff, 0xff, 0x7f, 0x03,                           // header_nonce
+        0x5d, 0x00,                                       // 2 byte length of coinbase_tx
+        0x2, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, // coinbase_tx
+        0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+        0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+        0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xff, 0xff,
+        0xff, 0x22, 0x1, 0x18, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0,
+        0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+        0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+        0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xff,
+        0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x16,
+        0x0, 0x14, 0x53, 0x12, 0x60, 0xaa, 0x2a, 0x19, 0x9e,
+        0x22, 0x8c, 0x53, 0x7d, 0xfa, 0x42, 0xc8, 0x2b, 0xea,
+        0x2c, 0x7c, 0x1f, 0x4d, 0x0, 0x0, 0x0, 0x0};
+    DataStream ss(input);
+
+    node::Sv2SubmitSolutionMsg submit_solution;
+    ss >> submit_solution;
+
+    // BOOST_CHECK_EQUAL(submit_solution.m_template_id, 2);
+    // BOOST_CHECK_EQUAL(submit_solution.m_version, 2);
+    // BOOST_CHECK_EQUAL(submit_solution.m_header_timestamp, 1661877399);
+    // BOOST_CHECK_EQUAL(submit_solution.m_header_nonce, 58720255);
+}
 BOOST_AUTO_TEST_SUITE_END()

From 7460cc2969f8f56c79714453657134f9aa28d4f5 Mon Sep 17 00:00:00 2001
From: Sjors Provoost <sjors@sprovoost.nl>
Date: Fri, 22 Nov 2024 14:41:56 +0100
Subject: [PATCH 06/11] Sv2: construct and submit block templates

Determine future_template only once
---
 src/sv2/connman.h                        |   6 +
 src/sv2/template_provider.cpp            | 218 ++++++++++++++++++++++-
 src/sv2/template_provider.h              |  51 +++++-
 src/test/sv2_template_provider_tests.cpp |  44 ++++-
 4 files changed, 312 insertions(+), 7 deletions(-)

diff --git a/src/sv2/connman.h b/src/sv2/connman.h
index 87468e4b01176..0d0161d3f2592 100644
--- a/src/sv2/connman.h
+++ b/src/sv2/connman.h
@@ -54,6 +54,12 @@ struct Sv2Client
      */
     unsigned int m_coinbase_tx_outputs_size;
 
+    /**
+     * Tracks the best template in the Template Provider m_block_template_cache map.
+     * Not guaranteed to still exist.
+    */
+    uint64_t m_best_template_id = 0;
+
     explicit Sv2Client(size_t id, std::unique_ptr<Sv2Transport> transport) :
                        m_id{id}, m_transport{std::move(transport)} {};
 
diff --git a/src/sv2/template_provider.cpp b/src/sv2/template_provider.cpp
index eb22e8d9f8a63..e9f297451f856 100644
--- a/src/sv2/template_provider.cpp
+++ b/src/sv2/template_provider.cpp
@@ -36,9 +36,6 @@ Sv2TemplateProvider::Sv2TemplateProvider(interfaces::Mining& mining) : m_mining{
     // TODO: persist certificate
 
     m_connman = std::make_unique<Sv2Connman>(TP_SUBPROTOCOL, static_key, m_authority_pubkey, certificate);
-
-    // Suppress unused variable warning, result is unused.
-    m_mining.getTip();
 }
 
 bool Sv2TemplateProvider::Start(const Sv2TemplateProviderOptions& options)
@@ -78,7 +75,220 @@ void Sv2TemplateProvider::StopThreads()
 
 void Sv2TemplateProvider::ThreadSv2Handler()
 {
+    // Wait for the node chainstate to be ready if needed.
+    auto tip{m_mining.waitTipChanged(uint256::ZERO)};
+    Assert(tip.hash != uint256::ZERO);
+
+    // Make sure it's initialized, doesn't need to be accurate.
+    {
+        LOCK(m_tp_mutex);
+        m_last_block_time = GetTime<std::chrono::seconds>();
+    }
+
+    // Wait to come out of IBD, except on signet, where we might be the only miner.
+    while (!m_flag_interrupt_sv2 && gArgs.GetChainType() != ChainType::SIGNET) {
+        // TODO: Wait until there's no headers-only branch with more work than our chaintip.
+        //       The current check can still cause us to broadcast a few dozen useless templates
+        //       at startup.
+        if (!m_mining.isInitialBlockDownload()) break;
+        LogPrintLevel(BCLog::SV2, BCLog::Level::Trace, "Waiting to come out of IBD\n");
+        std::this_thread::sleep_for(1000ms);
+    }
+
     while (!m_flag_interrupt_sv2) {
-        // TODO: handle messages
+        // We start with one template per client, which has an interface through
+        // which we monitor for better templates.
+
+        // TODO: give each client its own thread so they're treated equally
+        //       and so that newly connected clients don't have to wait.
+        std::optional<size_t> first_client_id{};
+        m_connman->ForEachClient([this, &first_client_id](Sv2Client& client) {
+            if (!client.m_coinbase_output_data_size_recv) return;
+
+            if (!first_client_id) {
+                first_client_id = client.m_id;
+            }
+
+            // Check if we already have a template interface for this client
+            if (client.m_best_template_id != 0) {
+                LOCK(m_tp_mutex);
+                auto cached_template = m_block_template_cache.find(client.m_best_template_id);
+                if (cached_template != m_block_template_cache.end()) return;
+            }
+
+            // Create block template and store interface reference
+            // TODO: reuse template_id for clients with the same m_default_coinbase_tx_additional_output_size
+            uint64_t template_id{WITH_LOCK(m_tp_mutex, return ++m_template_id;)};
+
+            // coinbase_max_additional_weight will be renamed to block_reserved_weight soon:
+            // https://github.com/bitcoin/bitcoin/pull/31384
+            // The node enforces a minimum of 2000, though not for IPC so we could go a bit
+            // lower, but let's not...
+            uint32_t block_reserved_weight{2000 + client.m_coinbase_tx_outputs_size * 4};
+
+            const auto time_start{SteadyClock::now()};
+            auto block_template = m_mining.createNewBlock({.use_mempool = true, .coinbase_max_additional_weight = block_reserved_weight});
+            LogPrintLevel(BCLog::SV2, BCLog::Level::Trace, "Assemble template: %.2fms\n",
+                Ticks<MillisecondsDouble>(SteadyClock::now() - time_start));
+
+            uint256 prev_hash{block_template->getBlockHeader().hashPrevBlock};
+            {
+                LOCK(m_tp_mutex);
+                if (prev_hash != m_best_prev_hash) {
+                    m_best_prev_hash = prev_hash;
+                    // Does not need to be accurate
+                    m_last_block_time = GetTime<std::chrono::seconds>();
+                }
+            }
+
+            if (!SendWork(client, template_id, *block_template, /*future_template=*/true)) {
+                LogPrintLevel(BCLog::SV2, BCLog::Level::Trace, "Disconnecting client id=%zu\n",
+                                client.m_id);
+                client.m_disconnect_flag = true;
+            }
+
+            LOCK(m_tp_mutex);
+            m_block_template_cache.insert({template_id, std::move(block_template)});
+            client.m_best_template_id = template_id;
+        });
+
+        bool new_template{false};
+
+        // Delay event loop is no client if fully connected
+        if (!first_client_id) std::this_thread::sleep_for(1000ms);
+
+        // The future template flag is set when there's a new prevhash,
+        // not when there's only a fee increase.
+        bool future_template{false};
+
+        // For the first connected client, wait for a new chaintip.
+        m_connman->ForEachClient([this, first_client_id, &future_template, &new_template](Sv2Client& client) {
+            if (!first_client_id || client.m_id != first_client_id) return;
+            Assert(client.m_coinbase_output_data_size_recv);
+
+            std::shared_ptr<BlockTemplate> block_template = WITH_LOCK(m_tp_mutex, return m_block_template_cache.find(client.m_best_template_id)->second;);
+
+            // We give waitNext() a timeout of 1 second to prevent it from generating
+            // new templates too quickly. During this wait we're not serving newly connected clients.
+            // This can be cleaned up by having every client run its own thread.
+            const MillisecondsDouble tick{m_options.is_test ? 0 : 1000};
+            block_template = block_template->waitNext({.timeout = tick, .fee_threshold = MAX_MONEY});
+            if (block_template) {
+                new_template = true;
+                uint256 prev_hash{block_template->getBlockHeader().hashPrevBlock};
+
+                {
+                    LOCK(m_tp_mutex);
+                    if (prev_hash != m_best_prev_hash) {
+                        future_template = true;
+                        m_best_prev_hash = prev_hash;
+                        // Does not need to be accurate
+                        m_last_block_time = GetTime<std::chrono::seconds>();
+                    }
+
+                    ++m_template_id;
+                }
+
+
+                // Send it the updated template
+                if (!SendWork(client, WITH_LOCK(m_tp_mutex, return m_template_id;), *block_template, future_template)) {
+                    LogPrintLevel(BCLog::SV2, BCLog::Level::Trace, "Disconnecting client id=%zu\n",
+                                    client.m_id);
+                    client.m_disconnect_flag = true;
+                }
+
+                LOCK(m_tp_mutex);
+                m_block_template_cache.insert({m_template_id, std::move(block_template)});
+                client.m_best_template_id = m_template_id;
+
+            }
+
+        });
+
+        if (new_template) {
+            // And generate new temlates for the other clients
+            m_connman->ForEachClient([this, first_client_id, &future_template](Sv2Client& client) {
+                if (!client.m_coinbase_output_data_size_recv) return;
+                if (client.m_id == first_client_id.value()) return;
+
+                std::shared_ptr<BlockTemplate> block_template = WITH_LOCK(m_tp_mutex, return m_block_template_cache.find(client.m_best_template_id)->second;);
+
+                const auto time_start{SteadyClock::now()};
+                // TODO: deduplicate
+                uint32_t block_reserved_weight{2000 + client.m_coinbase_tx_outputs_size * 4};
+
+                block_template = m_mining.createNewBlock({.use_mempool = true, .coinbase_max_additional_weight = block_reserved_weight});
+                LogPrintLevel(BCLog::SV2, BCLog::Level::Trace, "Assemble template: %.2fms\n",
+                              Ticks<MillisecondsDouble>(SteadyClock::now() - time_start));
+
+                if (Assert(block_template)) {
+                    {
+                        LOCK(m_tp_mutex);
+                        ++m_template_id;
+                    }
+
+                    // Send it the updated template
+                    if (!SendWork(client, WITH_LOCK(m_tp_mutex, return m_template_id;), *block_template, future_template)) {
+                        LogPrintLevel(BCLog::SV2, BCLog::Level::Trace, "Disconnecting client id=%zu\n",
+                                        client.m_id);
+                        client.m_disconnect_flag = true;
+                    }
+
+                    LOCK(m_tp_mutex);
+                    m_block_template_cache.insert({m_template_id, std::move(block_template)});
+                    client.m_best_template_id = m_template_id;
+                }
+
+            });
+        }
+
+        // Prune old templates and continue the loop.
+        LOCK(m_tp_mutex);
+        PruneBlockTemplateCache();
+
+        if (m_options.is_test) {
+            // Take a very short break
+            std::this_thread::sleep_for(1ms);
+        }
+    }
+}
+
+void Sv2TemplateProvider::PruneBlockTemplateCache()
+{
+    AssertLockHeld(m_tp_mutex);
+
+    // Allow a few seconds for clients to submit a block
+    auto recent = GetTime<std::chrono::seconds>() - std::chrono::seconds(10);
+    if (m_last_block_time > recent) return;
+    // If the blocks prevout is not the tip's prevout, delete it.
+    uint256 prev_hash = m_best_prev_hash;
+    std::erase_if(m_block_template_cache, [prev_hash] (const auto& kv) {
+        if (kv.second->getBlockHeader().hashPrevBlock != prev_hash) {
+            return true;
+        }
+        return false;
+    });
+}
+
+bool Sv2TemplateProvider::SendWork(Sv2Client& client, uint64_t template_id, BlockTemplate& block_template, bool future_template)
+{
+    CBlockHeader header{block_template.getBlockHeader()};
+
+    node::Sv2NewTemplateMsg new_template{header,
+                                        block_template.getCoinbaseTx(),
+                                        block_template.getCoinbaseMerklePath(),
+                                        block_template.getWitnessCommitmentIndex(),
+                                        template_id,
+                                        future_template};
+
+    LogPrintLevel(BCLog::SV2, BCLog::Level::Debug, "Send 0x71 NewTemplate id=%lu future=%d to client id=%zu\n", template_id, future_template, client.m_id);
+    client.m_send_messages.emplace_back(new_template);
+
+    if (future_template) {
+        node::Sv2SetNewPrevHashMsg new_prev_hash{header, template_id};
+        LogPrintLevel(BCLog::SV2, BCLog::Level::Debug, "Send 0x72 SetNewPrevHash to client id=%zu\n", client.m_id);
+        client.m_send_messages.emplace_back(new_prev_hash);
     }
+
+    return true;
 }
diff --git a/src/sv2/template_provider.h b/src/sv2/template_provider.h
index 64612d65a6a6e..8ba4c974327eb 100644
--- a/src/sv2/template_provider.h
+++ b/src/sv2/template_provider.h
@@ -1,6 +1,7 @@
 #ifndef BITCOIN_SV2_TEMPLATE_PROVIDER_H
 #define BITCOIN_SV2_TEMPLATE_PROVIDER_H
 
+#include <chrono>
 #include <interfaces/mining.h>
 #include <sv2/connman.h>
 #include <sv2/messages.h>
@@ -10,6 +11,8 @@
 #include <util/time.h>
 #include <streams.h>
 
+using interfaces::BlockTemplate;
+
 struct Sv2TemplateProviderOptions
 {
     /**
@@ -65,6 +68,28 @@ class Sv2TemplateProvider : public Sv2EventsInterface
     std::atomic<bool> m_flag_interrupt_sv2{false};
     CThreadInterrupt m_interrupt_sv2;
 
+    /**
+     * The most recent template id. This is incremented on creating new template,
+     * which happens for each connected client.
+     */
+    uint64_t m_template_id GUARDED_BY(m_tp_mutex){0};
+
+    /**
+     * The current best known block hash in the network.
+     */
+    uint256 m_best_prev_hash GUARDED_BY(m_tp_mutex){uint256(0)};
+
+    /** When we last saw a new block connection. Used to cache stale templates
+      * for some time after this.
+      */
+    std::chrono::nanoseconds m_last_block_time GUARDED_BY(m_tp_mutex);
+
+    /**
+     * A cache that maps ids used in NewTemplate messages and its associated block template.
+     */
+    using BlockTemplateCache = std::map<uint64_t, std::shared_ptr<BlockTemplate>>;
+    BlockTemplateCache m_block_template_cache GUARDED_BY(m_tp_mutex);
+
 public:
     explicit Sv2TemplateProvider(interfaces::Mining& mining);
 
@@ -82,7 +107,7 @@ class Sv2TemplateProvider : public Sv2EventsInterface
      * The main thread for the template provider, contains an event loop handling
      * all tasks for the template provider.
      */
-    void ThreadSv2Handler();
+    void ThreadSv2Handler() EXCLUSIVE_LOCKS_REQUIRED(!m_tp_mutex);
 
     /**
      * Triggered on interrupt signals to stop the main event loop in ThreadSv2Handler().
@@ -97,10 +122,32 @@ class Sv2TemplateProvider : public Sv2EventsInterface
     /**
      * Main handler for all received stratum v2 messages.
      */
-    void ProcessSv2Message(const node::Sv2NetMsg& sv2_header, Sv2Client& client);
+    void ProcessSv2Message(const node::Sv2NetMsg& sv2_header, Sv2Client& client) EXCLUSIVE_LOCKS_REQUIRED(!m_tp_mutex);
 
     // Only used for tests
     XOnlyPubKey m_authority_pubkey;
+
+    /* Block templates that connected clients may be working on, only used for tests */
+    BlockTemplateCache& GetBlockTemplates() EXCLUSIVE_LOCKS_REQUIRED(m_tp_mutex) { return m_block_template_cache; }
+
+private:
+
+    /* Forget templates from before the last block, but with a few seconds margin. */
+    void PruneBlockTemplateCache() EXCLUSIVE_LOCKS_REQUIRED(m_tp_mutex);
+
+    /**
+     * Sends the best NewTemplate and SetNewPrevHash to a client.
+     *
+     * The current implementation doesn't create templates for future empty
+     * or speculative blocks. Despite that, we first send NewTemplate with
+     * future_template set to true, followed by SetNewPrevHash. We do this
+     * both when first connecting and when a new block is found.
+     *
+     * When the template is update to take newer mempool transactions into
+     * account, we set future_template to false and don't send SetNewPrevHash.
+     */
+    [[nodiscard]] bool SendWork(Sv2Client& client, uint64_t template_id, BlockTemplate& block_template, bool future_template);
+
 };
 
 #endif // BITCOIN_SV2_TEMPLATE_PROVIDER_H
diff --git a/src/test/sv2_template_provider_tests.cpp b/src/test/sv2_template_provider_tests.cpp
index cae9162545172..96ebe08a02397 100644
--- a/src/test/sv2_template_provider_tests.cpp
+++ b/src/test/sv2_template_provider_tests.cpp
@@ -1,5 +1,6 @@
 #include <boost/test/unit_test.hpp>
 #include <interfaces/mining.h>
+#include <node/miner.h>
 #include <sv2/messages.h>
 #include <sv2/template_provider.h>
 #include <test/util/net.h>
@@ -122,6 +123,12 @@ class TPTester {
 
         return node::Sv2NetMsg{node::Sv2MsgType::SETUP_CONNECTION, std::move(bytes)};
     }
+
+    size_t GetBlockTemplateCount()
+    {
+        LOCK(m_tp->m_tp_mutex);
+        return m_tp->GetBlockTemplates().size();
+    }
 };
 
 BOOST_AUTO_TEST_CASE(client_tests)
@@ -142,12 +149,47 @@ BOOST_AUTO_TEST_CASE(client_tests)
     // SetupConnection.Success is 6 bytes
     BOOST_REQUIRE_EQUAL(tester.PeerReceiveBytes(), SV2_HEADER_ENCRYPTED_SIZE + 6 + Poly1305::TAGLEN);
 
+    // There should be no block templates before any client gave us their coinbase
+    // output data size:
+    BOOST_REQUIRE(tester.GetBlockTemplateCount() == 0);
+
     std::vector<uint8_t> coinbase_output_max_additional_size_bytes{
         0x01, 0x00, 0x00, 0x00
     };
     node::Sv2NetMsg msg{node::Sv2MsgType::COINBASE_OUTPUT_DATA_SIZE, std::move(coinbase_output_max_additional_size_bytes)};
-    // No reply expected, not yet implemented
     tester.receiveMessage(msg);
+    BOOST_TEST_MESSAGE("The reply should be NewTemplate and SetNewPrevHash");
+    BOOST_REQUIRE_EQUAL(tester.PeerReceiveBytes(), 2 * SV2_HEADER_ENCRYPTED_SIZE + 91 + 80 + 2 * Poly1305::TAGLEN);
+
+    // There should now be one template
+    BOOST_REQUIRE_EQUAL(tester.GetBlockTemplateCount(), 1);
+
+    // Create a new block
+    mineBlocks(1);
+
+    // We should send out another NewTemplate and SetNewPrevHash
+    // The new template contains the new prevhash.
+    BOOST_REQUIRE_EQUAL(tester.PeerReceiveBytes(), 2 * SV2_HEADER_ENCRYPTED_SIZE + 91 + 80 + 2 * Poly1305::TAGLEN);
+    // The SetNewPrevHash message is redundant
+    // TODO: don't send it?
+    // Background: in the future we want to send an empty or optimistic template
+    //             before a block is found, so ASIC's can preload it. We would
+    //             then immedidately send a SetNewPrevHash message when there's
+    //             a new block, and construct a better template _after_ that.
+
+    // Templates are briefly preserved
+    BOOST_REQUIRE_EQUAL(tester.GetBlockTemplateCount(), 2);
+
+    // Do not provide transactions for stale templates
+    // TODO
+
+    // But do allow SubmitSolution
+    // TODO
+
+    // Until after some time
+    SetMockTime(GetMockTime() + std::chrono::seconds{15});
+    UninterruptibleSleep(std::chrono::milliseconds{1100});
+    BOOST_REQUIRE_EQUAL(tester.GetBlockTemplateCount(), 1);
 }
 
 BOOST_AUTO_TEST_SUITE_END()

From 2bce1ff9d0c450ae91b98a5931085b068085cfbf Mon Sep 17 00:00:00 2001
From: Sjors Provoost <sjors@sprovoost.nl>
Date: Tue, 2 Jul 2024 15:13:49 +0200
Subject: [PATCH 07/11] Handle REQUEST_TRANSACTION_DATA

---
 src/sv2/connman.cpp                      | 16 ++++++++
 src/sv2/connman.h                        |  7 ++++
 src/sv2/template_provider.cpp            | 49 ++++++++++++++++++++++++
 src/sv2/template_provider.h              |  2 +
 src/test/sv2_connman_tests.cpp           |  4 ++
 src/test/sv2_template_provider_tests.cpp | 27 +++++++++++++
 6 files changed, 105 insertions(+)

diff --git a/src/sv2/connman.cpp b/src/sv2/connman.cpp
index c8b2069520073..5c904a9f7a652 100644
--- a/src/sv2/connman.cpp
+++ b/src/sv2/connman.cpp
@@ -344,6 +344,22 @@ void Sv2Connman::ProcessSv2Message(const Sv2NetMsg& sv2_net_msg, Sv2Client& clie
 
         break;
     }
+    case Sv2MsgType::REQUEST_TRANSACTION_DATA:
+    {
+        node::Sv2RequestTransactionDataMsg request_tx_data;
+
+        try {
+            ss >> request_tx_data;
+        } catch (const std::exception& e) {
+            LogPrintLevel(BCLog::SV2, BCLog::Level::Error, "Received invalid RequestTransactionData message from client id=%zu: %e\n",
+                          client.m_id, e.what());
+            return;
+        }
+
+        m_msgproc->RequestTransactionData(client, request_tx_data);
+
+        break;
+    }
     default: {
         uint8_t msg_type[1]{uint8_t(sv2_net_msg.m_msg_type)};
         LogPrintLevel(BCLog::SV2, BCLog::Level::Warning, "Received unknown message type 0x%s from client id=%zu\n",
diff --git a/src/sv2/connman.h b/src/sv2/connman.h
index 0d0161d3f2592..555b0363bed13 100644
--- a/src/sv2/connman.h
+++ b/src/sv2/connman.h
@@ -78,6 +78,13 @@ struct Sv2Client
 class Sv2EventsInterface
 {
 public:
+    /**
+     * We received and successfully parsed a RequestTransactionData message.
+     * Deal with it and respond with either RequestTransactionData.Success or
+     * RequestTransactionData.Error.
+     */
+    virtual void RequestTransactionData(Sv2Client& client, node::Sv2RequestTransactionDataMsg msg) = 0;
+
     virtual ~Sv2EventsInterface() = default;
 };
 
diff --git a/src/sv2/template_provider.cpp b/src/sv2/template_provider.cpp
index e9f297451f856..65bdf64a2f45c 100644
--- a/src/sv2/template_provider.cpp
+++ b/src/sv2/template_provider.cpp
@@ -253,6 +253,55 @@ void Sv2TemplateProvider::ThreadSv2Handler()
     }
 }
 
+void Sv2TemplateProvider::RequestTransactionData(Sv2Client& client, node::Sv2RequestTransactionDataMsg msg)
+{
+    CBlock block;
+    {
+        LOCK(m_tp_mutex);
+        auto cached_block = m_block_template_cache.find(msg.m_template_id);
+        if (cached_block == m_block_template_cache.end()) {
+            node::Sv2RequestTransactionDataErrorMsg request_tx_data_error{msg.m_template_id, "template-id-not-found"};
+
+            LogDebug(BCLog::SV2, "Send 0x75 RequestTransactionData.Error (template-id-not-found: %zu) to client id=%zu\n",
+                    msg.m_template_id, client.m_id);
+            client.m_send_messages.emplace_back(request_tx_data_error);
+
+            return;
+        }
+        block = (*cached_block->second).getBlock();
+    }
+
+    {
+        LOCK(m_tp_mutex);
+        if (block.hashPrevBlock != m_best_prev_hash) {
+            LogTrace(BCLog::SV2, "Template id=%lu prevhash=%s, tip=%s\n", msg.m_template_id, HexStr(block.hashPrevBlock), HexStr(m_best_prev_hash));
+            node::Sv2RequestTransactionDataErrorMsg request_tx_data_error{msg.m_template_id, "stale-template-id"};
+
+
+            LogDebug(BCLog::SV2, "Send 0x75 RequestTransactionData.Error (stale-template-id) to client id=%zu\n",
+                    client.m_id);
+            client.m_send_messages.emplace_back(request_tx_data_error);
+            return;
+        }
+    }
+
+    std::vector<uint8_t> witness_reserve_value;
+    auto scriptWitness = block.vtx[0]->vin[0].scriptWitness;
+    if (!scriptWitness.IsNull()) {
+        std::copy(scriptWitness.stack[0].begin(), scriptWitness.stack[0].end(), std::back_inserter(witness_reserve_value));
+    }
+    std::vector<CTransactionRef> txs;
+    if (block.vtx.size() > 0) {
+        std::copy(block.vtx.begin() + 1, block.vtx.end(), std::back_inserter(txs));
+    }
+
+    node::Sv2RequestTransactionDataSuccessMsg request_tx_data_success{msg.m_template_id, std::move(witness_reserve_value), std::move(txs)};
+
+    LogPrintLevel(BCLog::SV2, BCLog::Level::Debug, "Send 0x74 RequestTransactionData.Success to client id=%zu\n",
+                    client.m_id);
+    client.m_send_messages.emplace_back(request_tx_data_success);
+}
+
 void Sv2TemplateProvider::PruneBlockTemplateCache()
 {
     AssertLockHeld(m_tp_mutex);
diff --git a/src/sv2/template_provider.h b/src/sv2/template_provider.h
index 8ba4c974327eb..f8f9b232f2280 100644
--- a/src/sv2/template_provider.h
+++ b/src/sv2/template_provider.h
@@ -127,6 +127,8 @@ class Sv2TemplateProvider : public Sv2EventsInterface
     // Only used for tests
     XOnlyPubKey m_authority_pubkey;
 
+    void RequestTransactionData(Sv2Client& client, node::Sv2RequestTransactionDataMsg msg) EXCLUSIVE_LOCKS_REQUIRED(!m_tp_mutex) override;
+
     /* Block templates that connected clients may be working on, only used for tests */
     BlockTemplateCache& GetBlockTemplates() EXCLUSIVE_LOCKS_REQUIRED(m_tp_mutex) { return m_block_template_cache; }
 
diff --git a/src/test/sv2_connman_tests.cpp b/src/test/sv2_connman_tests.cpp
index ce214864b5f4c..5efd4955f7c12 100644
--- a/src/test/sv2_connman_tests.cpp
+++ b/src/test/sv2_connman_tests.cpp
@@ -155,6 +155,10 @@ class ConnTester : Sv2EventsInterface {
         return node::Sv2NetMsg{node::Sv2MsgType::SETUP_CONNECTION, std::move(bytes)};
     }
 
+    void RequestTransactionData(Sv2Client& client, node::Sv2RequestTransactionDataMsg msg) override {
+        BOOST_TEST_MESSAGE("Process RequestTransactionData");
+    }
+
 };
 
 BOOST_AUTO_TEST_CASE(client_tests)
diff --git a/src/test/sv2_template_provider_tests.cpp b/src/test/sv2_template_provider_tests.cpp
index 96ebe08a02397..7f60723c9607e 100644
--- a/src/test/sv2_template_provider_tests.cpp
+++ b/src/test/sv2_template_provider_tests.cpp
@@ -164,6 +164,33 @@ BOOST_AUTO_TEST_CASE(client_tests)
     // There should now be one template
     BOOST_REQUIRE_EQUAL(tester.GetBlockTemplateCount(), 1);
 
+    // Get the template id
+    uint64_t template_id = 0;
+    {
+        LOCK(tester.m_tp->m_tp_mutex);
+        for (auto& t : tester.m_tp->GetBlockTemplates()) {
+            if (t.first > template_id) {
+                template_id = t.first;
+            }
+        }
+    }
+
+    // Have the peer send us RequestTransactionData
+    // We should reply with RequestTransactionData.Success
+    node::Sv2NetHeader req_tx_data_header{node::Sv2MsgType::REQUEST_TRANSACTION_DATA, 8};
+    DataStream ss;
+    ss << template_id;
+    std::vector<unsigned char> template_id_bytes;
+    template_id_bytes.resize(8);
+    ss >> MakeWritableByteSpan(template_id_bytes);
+
+    msg = node::Sv2NetMsg{req_tx_data_header.m_msg_type, std::move(template_id_bytes)};
+    tester.receiveMessage(msg);
+    const size_t template_id_size = 8;
+    const size_t excess_data_size = 2 + 32;
+    size_t tx_list_size = 2; // no transactions, so transaction_list is 0x0100
+    BOOST_REQUIRE_EQUAL(tester.PeerReceiveBytes(), SV2_HEADER_ENCRYPTED_SIZE + template_id_size + excess_data_size + tx_list_size + Poly1305::TAGLEN);
+
     // Create a new block
     mineBlocks(1);
 

From 271fbf7e07652b8ccdfad4642eaa8d7a9d32b768 Mon Sep 17 00:00:00 2001
From: Sjors Provoost <sjors@sprovoost.nl>
Date: Tue, 2 Jul 2024 15:14:02 +0200
Subject: [PATCH 08/11] Handle SUBMIT_SOLUTION

---
 src/sv2/connman.cpp            | 19 ++++++++++++++
 src/sv2/connman.h              |  5 ++++
 src/sv2/template_provider.cpp  | 45 ++++++++++++++++++++++++++++++++++
 src/sv2/template_provider.h    |  2 ++
 src/test/sv2_connman_tests.cpp |  4 +++
 5 files changed, 75 insertions(+)

diff --git a/src/sv2/connman.cpp b/src/sv2/connman.cpp
index 5c904a9f7a652..4b65634803726 100644
--- a/src/sv2/connman.cpp
+++ b/src/sv2/connman.cpp
@@ -344,6 +344,25 @@ void Sv2Connman::ProcessSv2Message(const Sv2NetMsg& sv2_net_msg, Sv2Client& clie
 
         break;
     }
+    case Sv2MsgType::SUBMIT_SOLUTION: {
+        if (!client.m_setup_connection_confirmed && !client.m_coinbase_output_data_size_recv) {
+            client.m_disconnect_flag = true;
+            return;
+        }
+
+        node::Sv2SubmitSolutionMsg submit_solution;
+        try {
+            ss >> submit_solution;
+        } catch (const std::exception& e) {
+            LogPrintLevel(BCLog::SV2, BCLog::Level::Error, "Received invalid SubmitSolution message from client id=%zu: %e\n",
+                          client.m_id, e.what());
+            return;
+        }
+
+        m_msgproc->SubmitSolution(submit_solution);
+
+        break;
+    }
     case Sv2MsgType::REQUEST_TRANSACTION_DATA:
     {
         node::Sv2RequestTransactionDataMsg request_tx_data;
diff --git a/src/sv2/connman.h b/src/sv2/connman.h
index 555b0363bed13..287af427d2ba1 100644
--- a/src/sv2/connman.h
+++ b/src/sv2/connman.h
@@ -85,6 +85,11 @@ class Sv2EventsInterface
      */
     virtual void RequestTransactionData(Sv2Client& client, node::Sv2RequestTransactionDataMsg msg) = 0;
 
+    /**
+     * We received and successfully parsed a SubmitSolution message.
+     */
+    virtual void SubmitSolution(node::Sv2SubmitSolutionMsg solution) = 0;
+
     virtual ~Sv2EventsInterface() = default;
 };
 
diff --git a/src/sv2/template_provider.cpp b/src/sv2/template_provider.cpp
index 65bdf64a2f45c..f3adb55ff4ef4 100644
--- a/src/sv2/template_provider.cpp
+++ b/src/sv2/template_provider.cpp
@@ -1,6 +1,7 @@
 #include <sv2/template_provider.h>
 
 #include <base58.h>
+#include <consensus/merkle.h>
 #include <crypto/hex_base.h>
 #include <common/args.h>
 #include <logging.h>
@@ -302,6 +303,50 @@ void Sv2TemplateProvider::RequestTransactionData(Sv2Client& client, node::Sv2Req
     client.m_send_messages.emplace_back(request_tx_data_success);
 }
 
+void Sv2TemplateProvider::SubmitSolution(node::Sv2SubmitSolutionMsg solution)
+{
+        LogPrintLevel(BCLog::SV2, BCLog::Level::Debug, "id=%lu version=%d, timestamp=%d, nonce=%d\n",
+            solution.m_template_id,
+            solution.m_version,
+            solution.m_header_timestamp,
+            solution.m_header_nonce
+        );
+
+        std::shared_ptr<BlockTemplate> block_template;
+        {
+            // We can't hold this lock until submitSolution() because it's
+            // possible that the new block arrives via the p2p network at the
+            // same time. That leads to a deadlock in g_best_block_mutex.
+            LOCK(m_tp_mutex);
+            auto cached_block_template = m_block_template_cache.find(solution.m_template_id);
+            if (cached_block_template == m_block_template_cache.end()) {
+                LogPrintLevel(BCLog::SV2, BCLog::Level::Debug, "Template with id=%lu is no longer in cache\n",
+                solution.m_template_id);
+                return;
+            }
+            /**
+             * It's important to not delete this template from the cache in case
+             * another solution is submitted for the same template later.
+             *
+             * This is very unlikely on mainnet, but not impossible. Many mining
+             * devices may be working on the default pool template at the same
+             * time and they may not update the new tip right away.
+             *
+             * The node will never broadcast the second block. It's marked
+             * valid-headers in getchaintips. However a node or pool operator
+             * may wish to manually inspect the block or keep it as a souvenir.
+             * Additionally, because in Stratum v2 the block solution is sent
+             * to both the pool node and the template provider node, it's
+             * possibly they arrive out of order and two competing blocks propagate
+             * on the network. In case of a reorg the node will be able to switch
+             * faster because it already has (but not fully validated) the block.
+             */
+            block_template = cached_block_template->second;
+        }
+
+        block_template->submitSolution(solution.m_version, solution.m_header_timestamp, solution.m_header_nonce, MakeTransactionRef(solution.m_coinbase_tx));
+}
+
 void Sv2TemplateProvider::PruneBlockTemplateCache()
 {
     AssertLockHeld(m_tp_mutex);
diff --git a/src/sv2/template_provider.h b/src/sv2/template_provider.h
index f8f9b232f2280..881aa166df4f0 100644
--- a/src/sv2/template_provider.h
+++ b/src/sv2/template_provider.h
@@ -129,6 +129,8 @@ class Sv2TemplateProvider : public Sv2EventsInterface
 
     void RequestTransactionData(Sv2Client& client, node::Sv2RequestTransactionDataMsg msg) EXCLUSIVE_LOCKS_REQUIRED(!m_tp_mutex) override;
 
+    void SubmitSolution(node::Sv2SubmitSolutionMsg solution) EXCLUSIVE_LOCKS_REQUIRED(!m_tp_mutex) override;
+
     /* Block templates that connected clients may be working on, only used for tests */
     BlockTemplateCache& GetBlockTemplates() EXCLUSIVE_LOCKS_REQUIRED(m_tp_mutex) { return m_block_template_cache; }
 
diff --git a/src/test/sv2_connman_tests.cpp b/src/test/sv2_connman_tests.cpp
index 5efd4955f7c12..8ed51a26ba03c 100644
--- a/src/test/sv2_connman_tests.cpp
+++ b/src/test/sv2_connman_tests.cpp
@@ -159,6 +159,10 @@ class ConnTester : Sv2EventsInterface {
         BOOST_TEST_MESSAGE("Process RequestTransactionData");
     }
 
+    void SubmitSolution(node::Sv2SubmitSolutionMsg solution) override {
+        BOOST_TEST_MESSAGE("Process SubmitSolution");
+    }
+
 };
 
 BOOST_AUTO_TEST_CASE(client_tests)

From a11d7fb4cf93b86274da93f3fd50116b46df3c5e Mon Sep 17 00:00:00 2001
From: Sjors Provoost <sjors@sprovoost.nl>
Date: Mon, 30 Sep 2024 17:51:12 +0200
Subject: [PATCH 09/11] Incrementally update sv2 block template

---
 src/sv2/template_provider.cpp            | 37 ++++++++-
 src/sv2/template_provider.h              | 10 +++
 src/test/sv2_template_provider_tests.cpp | 95 ++++++++++++++++++++++--
 3 files changed, 134 insertions(+), 8 deletions(-)

diff --git a/src/sv2/template_provider.cpp b/src/sv2/template_provider.cpp
index f3adb55ff4ef4..03093306593c2 100644
--- a/src/sv2/template_provider.cpp
+++ b/src/sv2/template_provider.cpp
@@ -74,6 +74,31 @@ void Sv2TemplateProvider::StopThreads()
     }
 }
 
+class Timer {
+private:
+    std::chrono::seconds m_interval;
+    std::chrono::seconds m_last_triggered;
+
+public:
+    Timer(std::chrono::seconds interval) : m_interval(interval) {
+        reset();
+    }
+
+    bool trigger() {
+        auto now{GetTime<std::chrono::seconds>()};
+        if (now - m_last_triggered >= m_interval) {
+            m_last_triggered = now;
+            return true;
+        }
+        return false;
+    }
+
+    void reset() {
+        auto now{GetTime<std::chrono::seconds>()};
+        m_last_triggered = now;
+    }
+};
+
 void Sv2TemplateProvider::ThreadSv2Handler()
 {
     // Wait for the node chainstate to be ready if needed.
@@ -96,6 +121,8 @@ void Sv2TemplateProvider::ThreadSv2Handler()
         std::this_thread::sleep_for(1000ms);
     }
 
+    Timer timer(m_options.fee_check_interval);
+
     while (!m_flag_interrupt_sv2) {
         // We start with one template per client, which has an interface through
         // which we monitor for better templates.
@@ -153,6 +180,8 @@ void Sv2TemplateProvider::ThreadSv2Handler()
             client.m_best_template_id = template_id;
         });
 
+        // Do not send templates with improved fees more frequently than the fee check interval
+        const bool check_fees{timer.trigger()};
         bool new_template{false};
 
         // Delay event loop is no client if fully connected
@@ -162,18 +191,20 @@ void Sv2TemplateProvider::ThreadSv2Handler()
         // not when there's only a fee increase.
         bool future_template{false};
 
-        // For the first connected client, wait for a new chaintip.
-        m_connman->ForEachClient([this, first_client_id, &future_template, &new_template](Sv2Client& client) {
+        // For the first connected client, wait for fees to rise.
+        m_connman->ForEachClient([this, first_client_id, check_fees, &future_template, &new_template](Sv2Client& client) {
             if (!first_client_id || client.m_id != first_client_id) return;
             Assert(client.m_coinbase_output_data_size_recv);
 
             std::shared_ptr<BlockTemplate> block_template = WITH_LOCK(m_tp_mutex, return m_block_template_cache.find(client.m_best_template_id)->second;);
 
+            CAmount fee_delta{check_fees ? m_options.fee_delta : MAX_MONEY};
+
             // We give waitNext() a timeout of 1 second to prevent it from generating
             // new templates too quickly. During this wait we're not serving newly connected clients.
             // This can be cleaned up by having every client run its own thread.
             const MillisecondsDouble tick{m_options.is_test ? 0 : 1000};
-            block_template = block_template->waitNext({.timeout = tick, .fee_threshold = MAX_MONEY});
+            block_template = block_template->waitNext({.timeout = tick, .fee_threshold = fee_delta});
             if (block_template) {
                 new_template = true;
                 uint256 prev_hash{block_template->getBlockHeader().hashPrevBlock};
diff --git a/src/sv2/template_provider.h b/src/sv2/template_provider.h
index 881aa166df4f0..092290e34a4e9 100644
--- a/src/sv2/template_provider.h
+++ b/src/sv2/template_provider.h
@@ -29,6 +29,16 @@ struct Sv2TemplateProviderOptions
      * The listening port for the server.
      */
     uint16_t port{8336};
+
+    /**
+     * Minimum fee delta to send new template upstream
+     */
+    CAmount fee_delta{1000};
+
+    /**
+     * Block template update interval (to check for increased fees)
+     */
+    std::chrono::seconds fee_check_interval{30};
 };
 
 /**
diff --git a/src/test/sv2_template_provider_tests.cpp b/src/test/sv2_template_provider_tests.cpp
index 7f60723c9607e..d7badfeda2412 100644
--- a/src/test/sv2_template_provider_tests.cpp
+++ b/src/test/sv2_template_provider_tests.cpp
@@ -1,11 +1,15 @@
+#include <addresstype.h>
 #include <boost/test/unit_test.hpp>
 #include <interfaces/mining.h>
 #include <node/miner.h>
+#include <node/transaction.h>
 #include <sv2/messages.h>
 #include <sv2/template_provider.h>
 #include <test/util/net.h>
 #include <test/util/setup_common.h>
+#include <test/util/transaction_utils.h>
 #include <util/sock.h>
+#include <util/strencodings.h>
 
 #include <memory>
 
@@ -164,7 +168,47 @@ BOOST_AUTO_TEST_CASE(client_tests)
     // There should now be one template
     BOOST_REQUIRE_EQUAL(tester.GetBlockTemplateCount(), 1);
 
-    // Get the template id
+    // Move mock time
+    // If the mempool doesn't change, no new template is generated.
+    SetMockTime(GetMockTime() + std::chrono::seconds{10});
+    BOOST_REQUIRE_EQUAL(tester.GetBlockTemplateCount(), 1);
+
+    // Create a transaction with a large fee
+    size_t tx_size;
+    CKey key = GenerateRandomKey();
+    CScript locking_script = GetScriptForDestination(PKHash(key.GetPubKey()));
+    // Don't hold on to the transaction
+    {
+        LOCK(cs_main);
+        BOOST_REQUIRE_EQUAL(m_node.mempool->size(), 0);
+
+        auto mtx = CreateValidMempoolTransaction(/*input_transaction=*/m_coinbase_txns[0], /*input_vout=*/0,
+                                                        /*input_height=*/0, /*input_signing_key=*/coinbaseKey,
+                                                        /*output_destination=*/locking_script,
+                                                        /*output_amount=*/CAmount(49 * COIN), /*submit=*/true);
+        CTransactionRef tx = MakeTransactionRef(mtx);
+
+        // Get serialized transaction size
+        DataStream ss;
+        ss << TX_WITH_WITNESS(tx);
+        tx_size = ss.size();
+
+        BOOST_REQUIRE_EQUAL(m_node.mempool->size(), 1);
+    }
+
+    // Move mock time
+    SetMockTime(GetMockTime() + std::chrono::seconds{tester.m_tp_options.fee_check_interval});
+
+    // Briefly wait for block creation
+    UninterruptibleSleep(std::chrono::milliseconds{200});
+
+    // Expect our peer to receive a NewTemplate message
+    // This time it should contain the 32 byte prevhash (unchanged)
+    constexpr size_t expected_len = SV2_HEADER_ENCRYPTED_SIZE + 91 + 32 + Poly1305::TAGLEN;
+    BOOST_TEST_MESSAGE("Receive NewTemplate");
+    BOOST_REQUIRE_EQUAL(tester.PeerReceiveBytes(), expected_len);
+
+    // Get the latest template id
     uint64_t template_id = 0;
     {
         LOCK(tester.m_tp->m_tp_mutex);
@@ -175,6 +219,10 @@ BOOST_AUTO_TEST_CASE(client_tests)
         }
     }
 
+    BOOST_REQUIRE_EQUAL(template_id, 2);
+
+    UninterruptibleSleep(std::chrono::milliseconds{200});
+
     // Have the peer send us RequestTransactionData
     // We should reply with RequestTransactionData.Success
     node::Sv2NetHeader req_tx_data_header{node::Sv2MsgType::REQUEST_TRANSACTION_DATA, 8};
@@ -188,15 +236,49 @@ BOOST_AUTO_TEST_CASE(client_tests)
     tester.receiveMessage(msg);
     const size_t template_id_size = 8;
     const size_t excess_data_size = 2 + 32;
-    size_t tx_list_size = 2; // no transactions, so transaction_list is 0x0100
+    size_t tx_list_size = 2 + 3 + tx_size;
+    BOOST_TEST_MESSAGE("Receive RequestTransactionData.Success");
+    BOOST_REQUIRE_EQUAL(tester.PeerReceiveBytes(), SV2_HEADER_ENCRYPTED_SIZE + template_id_size + excess_data_size + tx_list_size + Poly1305::TAGLEN);
+    {
+        LOCK(cs_main);
+
+        // RBF the transaction with with > DEFAULT_SV2_FEE_DELTA
+        CreateValidMempoolTransaction(/*input_transaction=*/m_coinbase_txns[0], /*input_vout=*/0,
+                                                    /*input_height=*/0, /*input_signing_key=*/coinbaseKey,
+                                                    /*output_destination=*/locking_script,
+                                                    /*output_amount=*/CAmount(48 * COIN), /*submit=*/true);
+
+        BOOST_REQUIRE_EQUAL(m_node.mempool->size(), 1);
+    }
+
+    // Move mock time
+    SetMockTime(GetMockTime() + std::chrono::seconds{tester.m_tp_options.fee_check_interval});
+
+    // Briefly wait for the timer in ThreadSv2Handler and block creation
+    UninterruptibleSleep(std::chrono::milliseconds{200});
+
+    // Wait a bit more for macOS native CI
+    UninterruptibleSleep(std::chrono::milliseconds{1000});
+
+    // Expect our peer to receive a NewTemplate message
+    BOOST_REQUIRE_EQUAL(tester.PeerReceiveBytes(), SV2_HEADER_ENCRYPTED_SIZE + 91 + 32 + Poly1305::TAGLEN);
+
+    // Check that there's a new template
+    BOOST_REQUIRE_EQUAL(tester.GetBlockTemplateCount(), 3);
+
+    // Have the peer send us RequestTransactionData for the old template
+    // We should reply with RequestTransactionData.Success, and the original
+    // (replaced) transaction
+    tester.receiveMessage(msg);
+    tx_list_size = 2 + 3 + tx_size;
     BOOST_REQUIRE_EQUAL(tester.PeerReceiveBytes(), SV2_HEADER_ENCRYPTED_SIZE + template_id_size + excess_data_size + tx_list_size + Poly1305::TAGLEN);
 
-    // Create a new block
+    BOOST_TEST_MESSAGE("Create a new block");
     mineBlocks(1);
 
     // We should send out another NewTemplate and SetNewPrevHash
     // The new template contains the new prevhash.
-    BOOST_REQUIRE_EQUAL(tester.PeerReceiveBytes(), 2 * SV2_HEADER_ENCRYPTED_SIZE + 91 + 80 + 2 * Poly1305::TAGLEN);
+    BOOST_REQUIRE_EQUAL(tester.PeerReceiveBytes(), 2 * SV2_HEADER_ENCRYPTED_SIZE + 91 + 32 + 80 + 2 * Poly1305::TAGLEN);
     // The SetNewPrevHash message is redundant
     // TODO: don't send it?
     // Background: in the future we want to send an empty or optimistic template
@@ -205,7 +287,7 @@ BOOST_AUTO_TEST_CASE(client_tests)
     //             a new block, and construct a better template _after_ that.
 
     // Templates are briefly preserved
-    BOOST_REQUIRE_EQUAL(tester.GetBlockTemplateCount(), 2);
+    BOOST_REQUIRE_EQUAL(tester.GetBlockTemplateCount(), 4);
 
     // Do not provide transactions for stale templates
     // TODO
@@ -217,6 +299,9 @@ BOOST_AUTO_TEST_CASE(client_tests)
     SetMockTime(GetMockTime() + std::chrono::seconds{15});
     UninterruptibleSleep(std::chrono::milliseconds{1100});
     BOOST_REQUIRE_EQUAL(tester.GetBlockTemplateCount(), 1);
+
+    // Mine a block in order to interrupt waitNext()
+    mineBlocks(1);
 }
 
 BOOST_AUTO_TEST_SUITE_END()

From 9f9706c9c45c081e5dd92f5e0ce5c66822ec9bed Mon Sep 17 00:00:00 2001
From: Sjors Provoost <sjors@sprovoost.nl>
Date: Thu, 1 Feb 2024 14:18:49 +0100
Subject: [PATCH 10/11] CKey: add Serialize and Unserialize

Co-authored-by: Vasil Dimov <vd@FreeBSD.org>
---
 src/key.h              | 26 ++++++++++++++++++++++++++
 src/test/key_tests.cpp | 23 +++++++++++++++++++++++
 2 files changed, 49 insertions(+)

diff --git a/src/key.h b/src/key.h
index 6ed6fd0fc3da7..d2ac120a0b062 100644
--- a/src/key.h
+++ b/src/key.h
@@ -204,6 +204,7 @@ class CKey
     ECDHSecret ComputeBIP324ECDHSecret(const EllSwiftPubKey& their_ellswift,
                                        const EllSwiftPubKey& our_ellswift,
                                        bool initiating) const;
+
     /** Compute a KeyPair
      *
      *  Wraps a `secp256k1_keypair` type.
@@ -220,6 +221,31 @@ class CKey
      *                               Merkle root of the script tree).
      */
     KeyPair ComputeKeyPair(const uint256* merkle_root) const;
+
+    /** Straight-forward serialization of key bytes (and compressed flag).
+     *  Use GetPrivKey() for OpenSSL compatible DER encoding.
+     */
+    template <typename Stream>
+    void Serialize(Stream& s) const
+    {
+        if (!IsValid()) {
+            throw std::ios_base::failure("invalid key");
+        }
+        s << fCompressed;
+        ::Serialize(s, Span{*this});
+    }
+
+    template <typename Stream>
+    void Unserialize(Stream& s)
+    {
+        s >> fCompressed;
+        MakeKeyData();
+        s >> Span{*keydata};
+        if (!Check(keydata->data())) {
+            ClearKeyData();
+            throw std::ios_base::failure("invalid key");
+        }
+    }
 };
 
 CKey GenerateRandomKey(bool compressed = true) noexcept;
diff --git a/src/test/key_tests.cpp b/src/test/key_tests.cpp
index 90e04bed8751d..b6670218383f0 100644
--- a/src/test/key_tests.cpp
+++ b/src/test/key_tests.cpp
@@ -390,4 +390,27 @@ BOOST_AUTO_TEST_CASE(key_schnorr_tweak_smoke_test)
     secp256k1_context_destroy(secp256k1_context_sign);
 }
 
+BOOST_AUTO_TEST_CASE(key_serialization)
+{
+    {
+        DataStream s{};
+        CKey key;
+        BOOST_CHECK_EXCEPTION(s << key, std::ios_base::failure,
+                              HasReason{"invalid key"});
+
+        s << MakeByteSpan(std::vector<std::byte>(33, std::byte(0)));
+        BOOST_CHECK_EXCEPTION(s >> key, std::ios_base::failure,
+                              HasReason{"invalid key"});
+    }
+
+    for (bool compressed : {true, false}) {
+        CKey key{GenerateRandomKey(/*compressed=*/compressed)};
+        DataStream s{};
+        s << key;
+        CKey key_copy;
+        s >> key_copy;
+        BOOST_CHECK(key == key_copy);
+    }
+}
+
 BOOST_AUTO_TEST_SUITE_END()

From 4c075418fb024b612227cc0b2567448fd7ce4711 Mon Sep 17 00:00:00 2001
From: Sjors Provoost <sjors@sprovoost.nl>
Date: Thu, 11 Jan 2024 15:58:48 +0100
Subject: [PATCH 11/11] Persist static key for Template Provider

---
 src/sv2/template_provider.cpp | 57 +++++++++++++++++++++++++++++++----
 src/sv2/template_provider.h   |  6 ++++
 2 files changed, 57 insertions(+), 6 deletions(-)

diff --git a/src/sv2/template_provider.cpp b/src/sv2/template_provider.cpp
index 03093306593c2..8d8737720d065 100644
--- a/src/sv2/template_provider.cpp
+++ b/src/sv2/template_provider.cpp
@@ -6,6 +6,7 @@
 #include <common/args.h>
 #include <logging.h>
 #include <sv2/noise.h>
+#include <util/readwritefile.h>
 #include <util/strencodings.h>
 #include <util/thread.h>
 
@@ -13,10 +14,46 @@ Sv2TemplateProvider::Sv2TemplateProvider(interfaces::Mining& mining) : m_mining{
 {
     // TODO: persist static key
     CKey static_key;
-    static_key.MakeNewKey(true);
-
-    auto authority_key{GenerateRandomKey()};
-
+    try {
+        AutoFile{fsbridge::fopen(GetStaticKeyFile(), "rb")} >> static_key;
+        LogPrintLevel(BCLog::SV2, BCLog::Level::Debug, "Reading cached static key from %s\n", fs::PathToString(GetStaticKeyFile()));
+    } catch (const std::ios_base::failure&) {
+        // File is not expected to exist the first time.
+        // In the unlikely event that loading an existing key fails, create a new one.
+    }
+    if (!static_key.IsValid()) {
+        static_key = GenerateRandomKey();
+        try {
+            AutoFile{fsbridge::fopen(GetStaticKeyFile(), "wb")} << static_key;
+        } catch (const std::ios_base::failure&) {
+            LogPrintLevel(BCLog::SV2, BCLog::Level::Error, "Error writing static key to %s\n", fs::PathToString(GetStaticKeyFile()));
+            // Continue, because this is not a critical failure.
+        }
+        LogPrintLevel(BCLog::SV2, BCLog::Level::Debug, "Generated static key, saved to %s\n", fs::PathToString(GetStaticKeyFile()));
+    }
+    LogPrintLevel(BCLog::SV2, BCLog::Level::Info, "Static key: %s\n", HexStr(static_key.GetPubKey()));
+
+   // Generate self signed certificate using (cached) authority key
+    // TODO: skip loading authoritity key if -sv2cert is used
+
+    // Load authority key if cached
+    CKey authority_key;
+    try {
+        AutoFile{fsbridge::fopen(GetAuthorityKeyFile(), "rb")} >> authority_key;
+    } catch (const std::ios_base::failure&) {
+        // File is not expected to exist the first time.
+        // In the unlikely event that loading an existing key fails, create a new one.
+    }
+    if (!authority_key.IsValid()) {
+        authority_key = GenerateRandomKey();
+        try {
+            AutoFile{fsbridge::fopen(GetAuthorityKeyFile(), "wb")} << authority_key;
+        } catch (const std::ios_base::failure&) {
+            LogPrintLevel(BCLog::SV2, BCLog::Level::Error, "Error writing authority key to %s\n", fs::PathToString(GetAuthorityKeyFile()));
+            // Continue, because this is not a critical failure.
+        }
+        LogPrintLevel(BCLog::SV2, BCLog::Level::Debug, "Generated authority key, saved to %s\n", fs::PathToString(GetAuthorityKeyFile()));
+    }
     // SRI uses base58 encoded x-only pubkeys in its configuration files
     std::array<unsigned char, 34> version_pubkey_bytes;
     version_pubkey_bytes[0] = 1;
@@ -34,11 +71,19 @@ Sv2TemplateProvider::Sv2TemplateProvider(interfaces::Mining& mining) : m_mining{
     uint32_t valid_to =  std::numeric_limits<unsigned int>::max(); // 2106
     Sv2SignatureNoiseMessage certificate = Sv2SignatureNoiseMessage(version, valid_from, valid_to, XOnlyPubKey(static_key.GetPubKey()), authority_key);
 
-    // TODO: persist certificate
-
     m_connman = std::make_unique<Sv2Connman>(TP_SUBPROTOCOL, static_key, m_authority_pubkey, certificate);
 }
 
+fs::path Sv2TemplateProvider::GetStaticKeyFile()
+{
+    return gArgs.GetDataDirNet() / "sv2_static_key";
+}
+
+fs::path Sv2TemplateProvider::GetAuthorityKeyFile()
+{
+    return gArgs.GetDataDirNet() / "sv2_authority_key";
+}
+
 bool Sv2TemplateProvider::Start(const Sv2TemplateProviderOptions& options)
 {
     m_options = options;
diff --git a/src/sv2/template_provider.h b/src/sv2/template_provider.h
index 092290e34a4e9..0f271e85ec97d 100644
--- a/src/sv2/template_provider.h
+++ b/src/sv2/template_provider.h
@@ -62,6 +62,12 @@ class Sv2TemplateProvider : public Sv2EventsInterface
 
     std::unique_ptr<Sv2Connman> m_connman;
 
+    /** Get name of file to store static key */
+    fs::path GetStaticKeyFile();
+
+    /** Get name of file to store authority key */
+    fs::path GetAuthorityKeyFile();
+
     /**
     * Configuration
     */