Merge branch 'auxpow'

.gitignore

@@ -130,6 +130,7 @@ win32-build
 test/config.ini
 test/cache/*
 test/.mypy_cache/
+test/lint/test_runner/target/
 
 !src/leveldb*/Makefile
 

Makefile.am

@@ -335,7 +335,7 @@ clean-docs:
 clean-local: clean-docs
  rm -rf coverage_percent.txt test_bitcoin.coverage/ total.coverage/ fuzz.coverage/ test/tmp/ cache/ $(OSX_APP)
  rm -rf test/functional/__pycache__ test/functional/test_framework/__pycache__ test/cache share/rpcauth/__pycache__
- rm -rf osx_volname dist/
+ rm -rf osx_volname dist/ test/lint/test_runner/target/ test/lint/__pycache__
 
 test-security-check:
 if TARGET_DARWIN

ci/test/00_setup_env_native_asan.sh

@@ -20,7 +20,7 @@ export CONTAINER_NAME=ci_native_asan
 export PACKAGES="systemtap-sdt-dev clang-17 llvm-17 libclang-rt-17-dev python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev libboost-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libqrencode-dev libsqlite3-dev ${BPFCC_PACKAGE}"
 export NO_DEPENDS=1
 export GOAL="install"
-export BITCOIN_CONFIG="--enable-c++20 --enable-usdt --enable-zmq --with-incompatible-bdb --with-gui=qt5 \
+export BITCOIN_CONFIG="--enable-usdt --enable-zmq --with-incompatible-bdb --with-gui=qt5 \
 CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' \
 --with-sanitizers=address,float-divide-by-zero,integer,undefined \
 CC='clang-17 -ftrivial-auto-var-init=pattern' CXX='clang++-17 -ftrivial-auto-var-init=pattern'"

ci/test/00_setup_env_native_fuzz_with_msan.sh

@@ -6,7 +6,7 @@
 
 export LC_ALL=C.UTF-8
 
-export CI_IMAGE_NAME_TAG="docker.io/ubuntu:22.04"
+export CI_IMAGE_NAME_TAG="docker.io/ubuntu:24.04"
 LIBCXX_DIR="/msan/cxx_build/"
 export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls"
 LIBCXX_FLAGS="-nostdinc++ -nostdlib++ -isystem ${LIBCXX_DIR}include/c++/v1 -L${LIBCXX_DIR}lib -Wl,-rpath,${LIBCXX_DIR}lib -lc++ -lc++abi -lpthread -Wno-unused-command-line-argument"

ci/test/00_setup_env_native_msan.sh

@@ -6,7 +6,7 @@
 
 export LC_ALL=C.UTF-8
 
-export CI_IMAGE_NAME_TAG="docker.io/ubuntu:22.04"
+export CI_IMAGE_NAME_TAG="docker.io/ubuntu:24.04"
 LIBCXX_DIR="/msan/cxx_build/"
 export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls"
 LIBCXX_FLAGS="-nostdinc++ -nostdlib++ -isystem ${LIBCXX_DIR}include/c++/v1 -L${LIBCXX_DIR}lib -Wl,-rpath,${LIBCXX_DIR}lib -lc++ -lc++abi -lpthread -Wno-unused-command-line-argument"

ci/test/00_setup_env_native_previous_releases.sh

@@ -7,10 +7,10 @@
 export LC_ALL=C.UTF-8
 
 export CONTAINER_NAME=ci_native_previous_releases
-export CI_IMAGE_NAME_TAG="docker.io/debian:bullseye"
-# Use minimum supported python3.9 and gcc-10, see doc/dependencies.md
-export PACKAGES="gcc-10 g++-10 python3-zmq"
-export DEP_OPTS="NO_UPNP=1 NO_NATPMP=1 DEBUG=1 CC=gcc-10 CXX=g++-10"
+export CI_IMAGE_NAME_TAG="docker.io/ubuntu:22.04"
+# Use minimum supported python3.9 (or best effort 3.10) and gcc-11, see doc/dependencies.md
+export PACKAGES="gcc-11 g++-11 python3-zmq"
+export DEP_OPTS="NO_UPNP=1 NO_NATPMP=1 DEBUG=1 CC=gcc-11 CXX=g++-11"
 export TEST_RUNNER_EXTRA="--previous-releases --coverage --extended --exclude feature_dbcrash" # Run extended tests so that coverage does not fail, but exclude the very slow dbcrash
 export RUN_UNIT_TESTS_SEQUENTIAL="true"
 export RUN_UNIT_TESTS="false"

ci/test/00_setup_env_native_tidy.sh

@@ -8,7 +8,7 @@ export LC_ALL=C.UTF-8
 
 export CI_IMAGE_NAME_TAG="docker.io/ubuntu:24.04"
 export CONTAINER_NAME=ci_native_tidy
-export TIDY_LLVM_V="17"
+export TIDY_LLVM_V="18"
 export PACKAGES="clang-${TIDY_LLVM_V} libclang-${TIDY_LLVM_V}-dev llvm-${TIDY_LLVM_V}-dev libomp-${TIDY_LLVM_V}-dev clang-tidy-${TIDY_LLVM_V} jq bear libevent-dev libboost-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev systemtap-sdt-dev libqt5gui5 libqt5core5a libqt5dbus5 qttools5-dev qttools5-dev-tools libqrencode-dev libsqlite3-dev libdb++-dev"
 export NO_DEPENDS=1
 export RUN_UNIT_TESTS=false

ci/test/01_base_install.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 #
-# Copyright (c) 2018-2022 The Bitcoin Core developers
+# Copyright (c) 2018-present The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
@@ -36,7 +36,7 @@ if [ -n "$PIP_PACKAGES" ]; then
 fi
 
 if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then
- ${CI_RETRY_EXE} git clone --depth=1 https://github.com/llvm/llvm-project -b llvmorg-17.0.6 /msan/llvm-project
+ ${CI_RETRY_EXE} git clone --depth=1 https://github.com/llvm/llvm-project -b "llvmorg-18.1.1" /msan/llvm-project
 
  cmake -G Ninja -B /msan/clang_build/ \
  -DLLVM_ENABLE_PROJECTS="clang" \
@@ -53,13 +53,14 @@ if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then
  update-alternatives --install /usr/bin/llvm-symbolizer llvm-symbolizer /msan/clang_build/bin/llvm-symbolizer 100
 
  cmake -G Ninja -B /msan/cxx_build/ \
- -DLLVM_ENABLE_RUNTIMES='libcxx;libcxxabi' \
+ -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind" \
  -DCMAKE_BUILD_TYPE=Release \
  -DLLVM_USE_SANITIZER=MemoryWithOrigins \
  -DCMAKE_C_COMPILER=clang \
  -DCMAKE_CXX_COMPILER=clang++ \
  -DLLVM_TARGETS_TO_BUILD=Native \
  -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF \
+ -DLIBCXXABI_USE_LLVM_UNWINDER=OFF \
  -DLIBCXX_HARDENING_MODE=debug \
  -S /msan/llvm-project/runtimes
 

ci/test/03_test_script.sh

@@ -182,7 +182,11 @@ if [ "${RUN_TIDY}" = "true" ]; then
 
  set -eo pipefail
  cd "${BASE_BUILD_DIR}/bitcoin-$HOST/src/"
- ( run-clang-tidy-"${TIDY_LLVM_V}" -quiet -load="/tidy-build/libbitcoin-tidy.so" "${MAKEJOBS}" ) | grep -C5 "error"
+ if ! ( run-clang-tidy-"${TIDY_LLVM_V}" -quiet -load="/tidy-build/libbitcoin-tidy.so" "${MAKEJOBS}" | tee tmp.tidy-out.txt ); then
+ grep -C5 "error: " tmp.tidy-out.txt
+ echo "^^^ ⚠️ Failure generated from clang-tidy"
+ false
+ fi
  # Filter out files by regex here, because regex may not be
  # accepted in src/.bear-tidy-config
  # Filter out:

contrib/guix/libexec/prelude.bash

@@ -51,7 +51,7 @@ fi
 time-machine() {
  # shellcheck disable=SC2086
  guix time-machine --url=https://git.savannah.gnu.org/git/guix.git \
- --commit=d5ca4d4fd713a9f7e17e074a1e37dda99bbb09fc \
+ --commit=dc4842797bfdc5f9f3f5f725bf189c2b68bd6b5a \
  --cores="$JOBS" \
  --keep-failed \
  --fallback \

doc/dependencies.md

@@ -9,7 +9,7 @@ You can find installation instructions in the `build-*.md` file for your platfor
 | [Autoconf](https://www.gnu.org/software/autoconf/) | [2.69](https://github.com/bitcoin/bitcoin/pull/17769) |
 | [Automake](https://www.gnu.org/software/automake/) | [1.13](https://github.com/bitcoin/bitcoin/pull/18290) |
 | [Clang](https://clang.llvm.org) | [14.0](https://github.com/bitcoin/bitcoin/pull/29208) |
-| [GCC](https://gcc.gnu.org) | [10.1](https://github.com/bitcoin/bitcoin/pull/28348) |
+| [GCC](https://gcc.gnu.org) | [11.1](https://github.com/bitcoin/bitcoin/pull/29091) |
 | [Python](https://www.python.org) (scripts, tests) | [3.9](https://github.com/bitcoin/bitcoin/pull/28211) |
 | [systemtap](https://sourceware.org/systemtap/) ([tracing](tracing.md))| N/A |
 

doc/design/assumeutxo.md

@@ -51,18 +51,12 @@ The utility script
 
 ## Design notes
 
-- A new block index `nStatus` flag is introduced, `BLOCK_ASSUMED_VALID`, to mark block
- index entries that are required to be assumed-valid by a chainstate created
- from a UTXO snapshot. This flag is used as a way to modify certain
- CheckBlockIndex() logic to account for index entries that are pending validation by a
- chainstate running asynchronously in the background.
-
 - The concept of UTXO snapshots is treated as an implementation detail that lives
  behind the ChainstateManager interface. The external presentation of the changes
  required to facilitate the use of UTXO snapshots is the understanding that there are
- now certain regions of the chain that can be temporarily assumed to be valid (using
- the nStatus flag mentioned above). In certain cases, e.g. wallet rescanning, this is
- very similar to dealing with a pruned chain.
+ now certain regions of the chain that can be temporarily assumed to be valid.
+ In certain cases, e.g. wallet rescanning, this is very similar to dealing with
+ a pruned chain.
 
  Logic outside ChainstateManager should try not to know about snapshots, instead
  preferring to work in terms of more general states like assumed-valid.

src/.clang-tidy

@@ -12,6 +12,7 @@ modernize-use-noexcept,
 modernize-use-nullptr,
 performance-*,
 -performance-avoid-endl,
+-performance-enum-size,
 -performance-inefficient-string-concatenation,
 -performance-no-int-to-ptr,
 -performance-noexcept-move-constructor,

src/Makefile.bench.include

@@ -34,6 +34,7 @@ bench_bench_namecoin_SOURCES = \
  bench/examples.cpp \
  bench/gcs_filter.cpp \
  bench/hashpadding.cpp \
+ bench/index_blockfilter.cpp \
  bench/load_external.cpp \
  bench/lockedpool.cpp \
  bench/logging.cpp \

src/bench/index_blockfilter.cpp

@@ -0,0 +1,43 @@
+// Copyright (c) 2023-present The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or https://www.opensource.org/licenses/mit-license.php.
+
+#include <bench/bench.h>
+
+#include <addresstype.h>
+#include <index/blockfilterindex.h>
+#include <node/chainstate.h>
+#include <node/context.h>
+#include <test/util/setup_common.h>
+#include <util/strencodings.h>
+
+// Very simple block filter index sync benchmark, only using coinbase outputs.
+static void BlockFilterIndexSync(benchmark::Bench& bench)
+{
+ const auto test_setup = MakeNoLogFileContext<TestChain100Setup>();
+
+ // Create more blocks
+ int CHAIN_SIZE = 600;
+ CPubKey pubkey{ParseHex("02ed26169896db86ced4cbb7b3ecef9859b5952825adbeab998fb5b307e54949c9")};
+ CScript script = GetScriptForDestination(WitnessV0KeyHash(pubkey));
+ std::vector<CMutableTransaction> noTxns;
+ for (int i = 0; i < CHAIN_SIZE - 100; i++) {
+ test_setup->CreateAndProcessBlock(noTxns, script);
+ SetMockTime(GetTime() + 1);
+ }
+ assert(WITH_LOCK(::cs_main, return test_setup->m_node.chainman->ActiveHeight() == CHAIN_SIZE));
+
+ bench.minEpochIterations(5).run([&] {
+ BlockFilterIndex filter_index(interfaces::MakeChain(test_setup->m_node), BlockFilterType::BASIC,
+ /*n_cache_size=*/0, /*f_memory=*/false, /*f_wipe=*/true);
+ assert(filter_index.Init());
+ assert(!filter_index.BlockUntilSyncedToCurrentChain());
+ filter_index.Sync();
+
+ IndexSummary summary = filter_index.GetSummary();
+ assert(summary.synced);
+ assert(summary.best_block_hash == WITH_LOCK(::cs_main, return test_setup->m_node.chainman->ActiveTip()->GetBlockHash()));
+ });
+}
+
+BENCHMARK(BlockFilterIndexSync, benchmark::PriorityLevel::HIGH);

src/bitcoin-chainstate.cpp

@@ -89,14 +89,13 @@ int main(int argc, char* argv[])
  {
  std::cout << "Warning: " << warning.original << std::endl;
  }
- void flushError(const std::string& debug_message) override
+ void flushError(const bilingual_str& message) override
  {
- std::cerr << "Error flushing block data to disk: " << debug_message << std::endl;
+ std::cerr << "Error flushing block data to disk: " << message.original << std::endl;
  }
- void fatalError(const std::string& debug_message, const bilingual_str& user_message) override
+ void fatalError(const bilingual_str& message) override
  {
- std::cerr << "Error: " << debug_message << std::endl;
- std::cerr << (user_message.empty() ? "A fatal internal error occurred." : user_message.original) << std::endl;
+ std::cerr << "Error: " << message.original << std::endl;
  }
  };
  auto notifications = std::make_unique<KernelNotifications>();

src/chain.h

@@ -104,16 +104,20 @@ enum BlockStatus : uint32_t {
 
  /**
  * Only first tx is coinbase, 2 <= coinbase input script length <= 100, transactions valid, no duplicate txids,
- * sigops, size, merkle root. Implies all parents are at least TREE but not necessarily TRANSACTIONS. When all
- * parent blocks also have TRANSACTIONS, CBlockIndex::nChainTx will be set.
+ * sigops, size, merkle root. Implies all parents are at least TREE but not necessarily TRANSACTIONS.
+ *
+ * If a block's validity is at least VALID_TRANSACTIONS, CBlockIndex::nTx will be set. If a block and all previous
+ * blocks back to the genesis block or an assumeutxo snapshot block are at least VALID_TRANSACTIONS,
+ * CBlockIndex::nChainTx will be set.
  */
  BLOCK_VALID_TRANSACTIONS = 3,
 
  //! Outputs do not overspend inputs, no double spends, coinbase output ok, no immature coinbase spends, BIP30.
- //! Implies all parents are either at least VALID_CHAIN, or are ASSUMED_VALID
+ //! Implies all previous blocks back to the genesis block or an assumeutxo snapshot block are at least VALID_CHAIN.
  BLOCK_VALID_CHAIN = 4,
 
- //! Scripts & signatures ok. Implies all parents are either at least VALID_SCRIPTS, or are ASSUMED_VALID.
+ //! Scripts & signatures ok. Implies all previous blocks back to the genesis block or an assumeutxo snapshot block
+ //! are at least VALID_SCRIPTS.
  BLOCK_VALID_SCRIPTS = 5,
 
  //! All validity bits.
@@ -130,21 +134,8 @@ enum BlockStatus : uint32_t {
 
  BLOCK_OPT_WITNESS = 128, //!< block data in blk*.dat was received with a witness-enforcing client
 
- /**
- * If ASSUMED_VALID is set, it means that this block has not been validated
- * and has validity status less than VALID_SCRIPTS. Also that it may have
- * descendant blocks with VALID_SCRIPTS set, because they can be validated
- * based on an assumeutxo snapshot.
- *
- * When an assumeutxo snapshot is loaded, the ASSUMED_VALID flag is added to
- * unvalidated blocks at the snapshot height and below. Then, as the background
- * validation progresses, and these blocks are validated, the ASSUMED_VALID
- * flags are removed. See `doc/design/assumeutxo.md` for details.
- *
- * This flag is only used to implement checks in CheckBlockIndex() and
- * should not be used elsewhere.
- */
- BLOCK_ASSUMED_VALID = 256,
+ BLOCK_STATUS_RESERVED = 256, //!< Unused flag that was previously set on assumeutxo snapshot blocks and their
+ //!< ancestors before they were validated, and unset when they were validated.
 };
 
 /** The block chain is a tree shaped structure starting with the
@@ -179,21 +170,16 @@ class CBlockIndex
  //! (memory only) Total amount of work (expected number of hashes) in the chain up to and including this block
  arith_uint256 nChainWork{};
 
- //! Number of transactions in this block.
+ //! Number of transactions in this block. This will be nonzero if the block
+ //! reached the VALID_TRANSACTIONS level, and zero otherwise.
  //! Note: in a potential headers-first mode, this number cannot be relied upon
- //! Note: this value is faked during UTXO snapshot load to ensure that
- //! LoadBlockIndex() will load index entries for blocks that we lack data for.
- //! @sa ActivateSnapshot
  unsigned int nTx{0};
 
  //! (memory only) Number of transactions in the chain up to and including this block.
- //! This value will be non-zero only if and only if transactions for this block and all its parents are available.
+ //! This value will be non-zero if this block and all previous blocks back
+ //! to the genesis block or an assumeutxo snapshot block have reached the
+ //! VALID_TRANSACTIONS level.
  //! Change to 64-bit type before 2024 (assuming worst case of 60 byte transactions).
- //!
- //! Note: this value is faked during use of a UTXO snapshot because we don't
- //! have the underlying block data available during snapshot load.
- //! @sa AssumeutxoData
- //! @sa ActivateSnapshot
  unsigned int nChainTx{0};
 
  //! Verification status of this block. See enum BlockStatus
@@ -270,15 +256,14 @@ class CBlockIndex
  }
 
  /**
- * Check whether this block's and all previous blocks' transactions have been
- * downloaded (and stored to disk) at some point.
+ * Check whether this block and all previous blocks back to the genesis block or an assumeutxo snapshot block have
+ * reached VALID_TRANSACTIONS and had transactions downloaded (and stored to disk) at some point.
  *
  * Does not imply the transactions are consensus-valid (ConnectTip might fail)
  * Does not imply the transactions are still stored on disk. (IsBlockPruned might return true)
  *
- * Note that this will be true for the snapshot base block, if one is loaded (and
- * all subsequent assumed-valid blocks) since its nChainTx value will have been set
- * manually based on the related AssumeutxoData entry.
+ * Note that this will be true for the snapshot base block, if one is loaded, since its nChainTx value will have
+ * been set manually based on the related AssumeutxoData entry.
  */
  bool HaveNumChainTxs() const { return nChainTx != 0; }
 
@@ -326,14 +311,6 @@ class CBlockIndex
  return ((nStatus & BLOCK_VALID_MASK) >= nUpTo);
  }
 
- //! @returns true if the block is assumed-valid; this means it is queued to be
- //! validated by a background chainstate.
- bool IsAssumedValid() const EXCLUSIVE_LOCKS_REQUIRED(::cs_main)
- {
- AssertLockHeld(::cs_main);
- return nStatus & BLOCK_ASSUMED_VALID;
- }
-
  //! Raise the validity level of this block index entry.
  //! Returns true if the validity was changed.
  bool RaiseValidity(enum BlockStatus nUpTo) EXCLUSIVE_LOCKS_REQUIRED(::cs_main)
@@ -343,12 +320,6 @@ class CBlockIndex
  if (nStatus & BLOCK_FAILED_MASK) return false;
 
  if ((nStatus & BLOCK_VALID_MASK) < nUpTo) {
- // If this block had been marked assumed-valid and we're raising
- // its validity to a certain point, there is no longer an assumption.
- if (nStatus & BLOCK_ASSUMED_VALID && nUpTo >= BLOCK_VALID_SCRIPTS) {
- nStatus &= ~BLOCK_ASSUMED_VALID;
- }
-
  nStatus = (nStatus & ~BLOCK_VALID_MASK) | nUpTo;
  return true;
  }