From d6a9de468c3a96eb2118c53e9b47128a3c592cab Mon Sep 17 00:00:00 2001 From: Navia Bheeman Date: Thu, 14 Mar 2024 13:12:15 +0530 Subject: [PATCH] split validation.cpp into multiple files: validation.cpp - only transaction and block validation code verifydb.cpp - veridydb class chainstate.cpp - chainstate class and other helper code used in it cs_main.cpp - common critical sections file_io.cpp - all disk read and write operations blockprune.cpp - blockprune class --- src/Makefile.am | 14 +- src/blockprune.cpp | 167 ++ src/blockprune.h | 48 + src/chainstate.cpp | 1859 +++++++++++++++++ src/chainstate.h | 187 ++ src/connecttrace.h | 79 + src/cs_main.cpp | 10 + src/cs_main.h | 24 + src/file_io.cpp | 619 ++++++ src/file_io.h | 50 + src/index/base.cpp | 1 + src/init.cpp | 3 + src/net_processing.cpp | 1 + src/policy/packages.cpp | 2 +- src/rest.cpp | 1 + src/rpc/blockchain.cpp | 4 + src/rpc/rawtransaction.cpp | 2 + src/scriptcheck.h | 38 + src/validation.cpp | 3266 ++---------------------------- src/validation.h | 152 +- src/verifydb.cpp | 113 ++ src/verifydb.h | 18 + src/wallet/rpcwallet.cpp | 1 + src/wallet/test/wallet_tests.cpp | 1 + src/wallet/wallet.cpp | 1 + src/zmq/zmqpublishnotifier.cpp | 1 + test/functional/rpc_help.py | 2 +- 27 files changed, 3426 insertions(+), 3238 deletions(-) create mode 100644 src/blockprune.cpp create mode 100644 src/blockprune.h create mode 100644 src/chainstate.cpp create mode 100644 src/chainstate.h create mode 100644 src/connecttrace.h create mode 100644 src/cs_main.cpp create mode 100644 src/cs_main.h create mode 100644 src/file_io.cpp create mode 100644 src/file_io.h create mode 100644 src/scriptcheck.h create mode 100644 src/verifydb.cpp create mode 100644 src/verifydb.h diff --git a/src/Makefile.am b/src/Makefile.am index 0195027d0b..a9030d63a3 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -93,9 +93,11 @@ BITCOIN_CORE_H = \ bech32.h \ bloom.h \ blockencodings.h \ + blockprune.h \ chain.h \ chainparams.h \ chainparamsseeds.h \ + chainstate.h \ checkpoints.h \ checkqueue.h \ clientversion.h \ @@ -106,12 +108,15 @@ BITCOIN_CORE_H = \ compat/endian.h \ compat/sanity.h \ compressor.h \ + connecttrace.h \ consensus/consensus.h \ consensus/tx_verify.h \ core_io.h \ core_memusage.h \ + cs_main.h \ cuckoocache.h \ federationparams.h \ + file_io.h \ fs.h \ httprpc.h \ httpserver.h \ @@ -157,6 +162,7 @@ BITCOIN_CORE_H = \ rpc/register.h \ rpc/util.h \ scheduler.h \ + scriptcheck.h \ script/descriptor.h \ script/ismine.h \ script/sigcache.h \ @@ -175,6 +181,7 @@ BITCOIN_CORE_H = \ threadinterrupt.h \ timedata.h \ torcontrol.h \ + trace.h \ txdb.h \ txmempool.h \ ui_interface.h \ @@ -197,8 +204,8 @@ BITCOIN_CORE_H = \ wallet/walletutil.h \ wallet/coinselection.h \ warnings.h \ + verifydb.h \ xfieldhistory.h \ - trace.h \ zmq/zmqabstractnotifier.h \ zmq/zmqconfig.h\ zmq/zmqnotificationinterface.h \ @@ -220,9 +227,13 @@ libtapyrus_server_a_SOURCES = \ addrman.cpp \ bloom.cpp \ blockencodings.cpp \ + blockprune.cpp \ chain.cpp \ + chainstate.cpp \ checkpoints.cpp \ consensus/tx_verify.cpp \ + cs_main.cpp \ + file_io.cpp \ httprpc.cpp \ httpserver.cpp \ index/base.cpp \ @@ -255,6 +266,7 @@ libtapyrus_server_a_SOURCES = \ txmempool.cpp \ ui_interface.cpp \ validation.cpp \ + verifydb.cpp \ validationinterface.cpp \ $(BITCOIN_CORE_H) diff --git a/src/blockprune.cpp b/src/blockprune.cpp new file mode 100644 index 0000000000..475f7422e5 --- /dev/null +++ b/src/blockprune.cpp @@ -0,0 +1,167 @@ +// Copyright (c) 2024 Chaintope Inc. +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include +#include +#include + + +/** + * BLOCK PRUNING CODE + */ + +/* Calculate the amount of disk space the block & undo files currently use */ +uint64_t CalculateCurrentUsage() +{ + LOCK(cs_LastBlockFile); + + uint64_t retval = 0; + for (const CBlockFileInfo &file : vinfoBlockFile) { + retval += file.nSize + file.nUndoSize; + } + return retval; +} + +/* Prune a block file (modify associated database entries)*/ +void PruneOneBlockFile(const int fileNumber) +{ + LOCK(cs_LastBlockFile); + + for (const auto& entry : mapBlockIndex) { + CBlockIndex* pindex = entry.second; + if (pindex->nFile == fileNumber) { + pindex->nStatus &= ~BLOCK_HAVE_DATA; + pindex->nStatus &= ~BLOCK_HAVE_UNDO; + pindex->nFile = 0; + pindex->nDataPos = 0; + pindex->nUndoPos = 0; + setDirtyBlockIndex.insert(pindex); + + // Prune from mapBlocksUnlinked -- any block we prune would have + // to be downloaded again in order to consider its chain, at which + // point it would be considered as a candidate for + // mapBlocksUnlinked or setBlockIndexCandidates. + std::pair::iterator, std::multimap::iterator> range = mapBlocksUnlinked.equal_range(pindex->pprev); + while (range.first != range.second) { + std::multimap::iterator _it = range.first; + range.first++; + if (_it->second == pindex) { + mapBlocksUnlinked.erase(_it); + } + } + } + } + + vinfoBlockFile[fileNumber].SetNull(); + setDirtyFileInfo.insert(fileNumber); +} + + +void UnlinkPrunedFiles(const std::set& setFilesToPrune) +{ + for (std::set::iterator it = setFilesToPrune.begin(); it != setFilesToPrune.end(); ++it) { + CDiskBlockPos pos(*it, 0); + fs::remove(GetBlockPosFilename(pos, "blk")); + fs::remove(GetBlockPosFilename(pos, "rev")); + LogPrintf("Prune: %s deleted blk/rev (%05u)\n", __func__, *it); + } +} + +void FindFilesToPruneManual(std::set& setFilesToPrune, int nManualPruneHeight) +{ + assert(fPruneMode && nManualPruneHeight > 0); + + LOCK2(cs_main, cs_LastBlockFile); + if (chainActive.Tip() == nullptr) + return; + + // last block to prune is the lesser of (user-specified height, MIN_BLOCKS_TO_KEEP from the tip) + unsigned int nLastBlockWeCanPrune = std::min((unsigned)nManualPruneHeight, chainActive.Tip()->nHeight - MIN_BLOCKS_TO_KEEP); + int count=0; + for (int fileNumber = 0; fileNumber < nLastBlockFile; fileNumber++) { + if (vinfoBlockFile[fileNumber].nSize == 0 || vinfoBlockFile[fileNumber].nHeightLast > nLastBlockWeCanPrune) + continue; + PruneOneBlockFile(fileNumber); + setFilesToPrune.insert(fileNumber); + count++; + } + LogPrintf("Prune (Manual): prune_height=%d removed %d blk/rev pairs\n", nLastBlockWeCanPrune, count); +} + +/* This function is called from the RPC code for pruneblockchain */ +void PruneBlockFilesManual(int nManualPruneHeight) +{ + CValidationState state; + if (!FlushStateToDisk(state, FlushStateMode::NONE, nManualPruneHeight)) { + LogPrintf("%s: failed to flush state (%s)\n", __func__, FormatStateMessage(state)); + } +} + +void FindFilesToPrune(std::set& setFilesToPrune, uint32_t nPruneAfterHeight) +{ + LOCK2(cs_main, cs_LastBlockFile); + if (chainActive.Tip() == nullptr || nPruneTarget == 0) { + return; + } + if (chainActive.Tip()->nHeight <= nPruneAfterHeight) { + return; + } + + unsigned int nLastBlockWeCanPrune = chainActive.Tip()->nHeight - MIN_BLOCKS_TO_KEEP; + uint64_t nCurrentUsage = CalculateCurrentUsage(); + // We don't check to prune until after we've allocated new space for files + // So we should leave a buffer under our target to account for another allocation + // before the next pruning. + uint64_t nBuffer = BLOCKFILE_CHUNK_SIZE + UNDOFILE_CHUNK_SIZE; + uint64_t nBytesToPrune; + int count=0; + + if (nCurrentUsage + nBuffer >= nPruneTarget) { + // On a prune event, the chainstate DB is flushed. + // To avoid excessive prune events negating the benefit of high dbcache + // values, we should not prune too rapidly. + // So when pruning in IBD, increase the buffer a bit to avoid a re-prune too soon. + if (IsInitialBlockDownload()) { + // Since this is only relevant during IBD, we use a fixed 10% + nBuffer += nPruneTarget / 10; + } + + for (int fileNumber = 0; fileNumber < nLastBlockFile; fileNumber++) { + nBytesToPrune = vinfoBlockFile[fileNumber].nSize + vinfoBlockFile[fileNumber].nUndoSize; + + if (vinfoBlockFile[fileNumber].nSize == 0) + continue; + + if (nCurrentUsage + nBuffer < nPruneTarget) // are we below our target? + break; + + // don't prune files that could have a block within MIN_BLOCKS_TO_KEEP of the prod chain's tip but keep scanning + if (vinfoBlockFile[fileNumber].nHeightLast > nLastBlockWeCanPrune) + continue; + + PruneOneBlockFile(fileNumber); + // Queue up the files for removal + setFilesToPrune.insert(fileNumber); + nCurrentUsage -= nBytesToPrune; + count++; + } + } + + LogPrint(BCLog::PRUNE, "Prune: target=%dMiB actual=%dMiB diff=%dMiB max_prune_height=%d removed %d blk/rev pairs\n", + nPruneTarget/1024/1024, nCurrentUsage/1024/1024, + ((int64_t)nPruneTarget - (int64_t)nCurrentUsage)/1024/1024, + nLastBlockWeCanPrune, count); +} + +bool CheckDiskSpace(uint64_t nAdditionalBytes, bool blocks_dir) +{ + uint64_t nFreeBytesAvailable = fs::space(blocks_dir ? GetBlocksDir() : GetDataDir()).available; + + // Check for nMinDiskSpace bytes (currently 50MB) + if (nFreeBytesAvailable < nMinDiskSpace + nAdditionalBytes) + return AbortNode("Disk space is low!", _("Error: Disk space is low!")); + + return true; +} diff --git a/src/blockprune.h b/src/blockprune.h new file mode 100644 index 0000000000..016f9a7fb7 --- /dev/null +++ b/src/blockprune.h @@ -0,0 +1,48 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Copyright (c) 2024 Chaintope Inc. +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_BLOCK_PRUNE_H +#define BITCOIN_BLOCK_PRUNE_H + +/** Minimum disk space required - used in CheckDiskSpace() */ +static const uint64_t nMinDiskSpace = 52428800; + +/** Check whether enough disk space is available for an incoming block */ +bool CheckDiskSpace(uint64_t nAdditionalBytes = 0, bool blocks_dir = false); + +/** Calculate the amount of disk space the block & undo files currently use */ +uint64_t CalculateCurrentUsage(); + +/** + * Mark one block file as pruned. + */ +void PruneOneBlockFile(const int fileNumber); + +/** + * Actually unlink the specified files + */ +void UnlinkPrunedFiles(const std::set& setFilesToPrune); + +/* Calculate the block/rev files to delete based on height specified by user with RPC command pruneblockchain */ +void FindFilesToPruneManual(std::set& setFilesToPrune, int nManualPruneHeight); + +/** + * Prune block and undo files (blk???.dat and undo???.dat) so that the disk space used is less than a user-defined target. + * The user sets the target (in MB) on the command line or in config file. This will be run on startup and whenever new + * space is allocated in a block or undo file, staying below the target. Changing back to unpruned requires a reindex + * (which in this case means the blockchain must be re-downloaded.) + * + * Pruning functions are called from FlushStateToDisk when the global fCheckForPruning flag has been set. + * Block and undo files are deleted in lock-step (when blk00003.dat is deleted, so is rev00003.dat.) + * Pruning cannot take place until the longest chain is at least a certain length (100000 on mainnet, 1000 on testnet, 1000 on dev). + * Pruning will never delete a block within a defined distance (currently 288) from the active chain's tip. + * The block index is updated by unsetting HAVE_DATA and HAVE_UNDO for any blocks that were stored in the deleted files. + * A db flag records the fact that at least some block files have been pruned. + * + * @param[out] setFilesToPrune The set of file indices that can be unlinked will be returned + */ +void FindFilesToPrune(std::set& setFilesToPrune, uint32_t nPruneAfterHeight); + +#endif // BITCOIN_BLOCK_PRUNE_H diff --git a/src/chainstate.cpp b/src/chainstate.cpp new file mode 100644 index 0000000000..dcfb840ff0 --- /dev/null +++ b/src/chainstate.cpp @@ -0,0 +1,1859 @@ +// Copyright (c) 2024 Chaintope Inc. +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +static int64_t nTimeCheck = 0; +static int64_t nTimeForks = 0; +static int64_t nTimeVerify = 0; +static int64_t nTimeConnect = 0; +static int64_t nTimeIndex = 0; +static int64_t nTimeCallbacks = 0; +static int64_t nTimeTotal = 0; +static int64_t nBlocksTotal = 0; + +static int64_t nTimeReadFromDisk = 0; +static int64_t nTimeConnectTotal = 0; +static int64_t nTimeFlush = 0; +static int64_t nTimeChainState = 0; +static int64_t nTimePostConnect = 0; + +CChainState g_chainstate; + +static void AlertNotify(const std::string& strMessage) +{ + uiInterface.NotifyAlertChanged(); + std::string strCmd = gArgs.GetArg("-alertnotify", ""); + if (strCmd.empty()) return; + + // Alert text should be plain ascii coming from a trusted source, but to + // be safe we first strip anything not in safeChars, then add single quotes around + // the whole string before passing it to the shell: + std::string singleQuote("'"); + std::string safeStatus = SanitizeString(strMessage); + safeStatus = singleQuote+safeStatus+singleQuote; + boost::replace_all(strCmd, "%s", safeStatus); + + std::thread t(runCommand, strCmd); + t.detach(); // thread runs free +} + +CBlockIndex *pindexBestForkTip = nullptr, *pindexBestForkBase = nullptr; + +static void CheckForkWarningConditions() +{ + AssertLockHeld(cs_main); + // Before we get past initial download, we cannot reliably alert about forks + // (we assume we don't get stuck on a fork before finishing our initial sync) + if (IsInitialBlockDownload()) + return; + + // If our best fork is no longer within 72 blocks (+/- 12 hours if no one mines it) + // of our head, drop it + if (pindexBestForkTip && chainActive.Height() - pindexBestForkTip->nHeight >= 72) + pindexBestForkTip = nullptr; + + if (pindexBestForkTip || (pindexBestInvalid && pindexBestInvalid->nHeight > chainActive.Tip()->nHeight + 6)) + { + if (!GetfLargeWorkForkFound() && pindexBestForkBase) + { + std::string warning = std::string("'Warning: Large-work fork detected, forking after block ") + + pindexBestForkBase->phashBlock->ToString() + std::string("'"); + AlertNotify(warning); + } + if (pindexBestForkTip && pindexBestForkBase) + { + LogPrintf("%s: Warning: Large valid fork found\n forking the chain at height %d (%s)\n lasting to height %d (%s).\nChain state database corruption likely.\n", __func__, + pindexBestForkBase->nHeight, pindexBestForkBase->phashBlock->ToString(), + pindexBestForkTip->nHeight, pindexBestForkTip->phashBlock->ToString()); + SetfLargeWorkForkFound(true); + } + else + { + LogPrintf("%s: Warning: Found invalid chain at least ~6 blocks longer than our best chain.\nChain state database corruption likely.\n", __func__); + SetfLargeWorkInvalidChainFound(true); + } + } + else + { + SetfLargeWorkForkFound(false); + SetfLargeWorkInvalidChainFound(false); + } +} + +void static InvalidChainFound(CBlockIndex* pindexNew) +{ + if (!pindexBestInvalid || pindexNew->nHeight > pindexBestInvalid->nHeight) + pindexBestInvalid = pindexNew; + + LogPrintf("%s: invalid block=%s height=%d date=%s\n", __func__, + pindexNew->GetBlockHash().ToString(), pindexNew->nHeight, FormatISO8601DateTime(pindexNew->GetBlockTime())); + CBlockIndex *tip = chainActive.Tip(); + assert (tip); + LogPrintf("%s: current best=%s height=%d date=%s\n", __func__, + tip->GetBlockHash().ToString(), chainActive.Height(), FormatISO8601DateTime(tip->GetBlockTime())); + CheckForkWarningConditions(); +} + +void NotifyHeaderTip() LOCKS_EXCLUDED(cs_main) { + bool fNotify = false; + bool fInitialBlockDownload = false; + static CBlockIndex* pindexHeaderOld = nullptr; + CBlockIndex* pindexHeader = nullptr; + { + LOCK(cs_main); + pindexHeader = pindexBestHeader; + + if (pindexHeader != pindexHeaderOld) { + fNotify = true; + fInitialBlockDownload = IsInitialBlockDownload(); + pindexHeaderOld = pindexHeader; + } + } + // Send block tip changed notifications without cs_main + if (fNotify) { + uiInterface.NotifyHeaderTip(fInitialBlockDownload, pindexHeader); + } +} + +bool CChainState::PreciousBlock(CValidationState& state, CBlockIndex *pindex) +{ + { + LOCK(cs_main); + if (pindex->nHeight < chainActive.Tip()->nHeight) { + // Nothing to do, this block is not at the tip. + return true; + } + if (chainActive.Tip()->nHeight > nLastPreciousHeight) { + // The chain has been extended since the last call, reset the counter. + nBlockReverseSequenceId = -1; + } + nLastPreciousHeight = chainActive.Tip()->nHeight; + setBlockIndexCandidates.erase(pindex); + pindex->nSequenceId = nBlockReverseSequenceId; + if (nBlockReverseSequenceId > std::numeric_limits::min()) { + // We can't keep reducing the counter if somebody really wants to + // call preciousblock 2**31-1 times on the same set of tips... + nBlockReverseSequenceId--; + } + if (pindex->IsValid(BLOCK_VALID_TRANSACTIONS) && pindex->nChainTx) { + setBlockIndexCandidates.insert(pindex); + PruneBlockIndexCandidates(); + } + } + + return ActivateBestChain(state, std::shared_ptr()); +} + +void CChainState::InvalidBlockFound(CBlockIndex *pindex, const CValidationState &state) { + if (!state.CorruptionPossible()) { + pindex->nStatus |= BLOCK_FAILED_VALID; + m_failed_blocks.insert(pindex); + setDirtyBlockIndex.insert(pindex); + setBlockIndexCandidates.erase(pindex); + InvalidChainFound(pindex); + } +} + +bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex *pindex) +{ + CDiskBlockPos pos = pindex->GetUndoPos(); + if (pos.IsNull()) { + return error("%s: no undo data available", __func__); + } + + // Open history file to read + CAutoFile filein(OpenUndoFile(pos, true), SER_DISK, CLIENT_VERSION); + if (filein.IsNull()) + return error("%s: OpenUndoFile failed", __func__); + + // Read block + uint256 hashChecksum; + CHashVerifier verifier(&filein); // We need a CHashVerifier as reserializing may lose data + try { + verifier << pindex->pprev->GetBlockHash(); + verifier >> blockundo; + filein >> hashChecksum; + } + catch (const std::exception& e) { + return error("%s: Deserialize or I/O error - %s", __func__, e.what()); + } + + // Verify checksum + if (hashChecksum != verifier.GetHash()) + return error("%s: Checksum mismatch", __func__); + + return true; +} + +/** + * Restore the UTXO in a Coin at a given COutPoint + * @param undo The Coin to be restored. + * @param view The coins view to which to apply the changes. + * @param out The out point that corresponds to the tx input. + * @return A DisconnectResult as an int + */ +int ApplyTxInUndo(Coin&& undo, CCoinsViewCache& view, const COutPoint& out) +{ + bool fClean = true; + + if (view.HaveCoin(out)) fClean = false; // overwriting transaction output + + if (undo.nHeight == 0) { + // Missing undo metadata (height and coinbase). Older versions included this + // information only in undo records for the last spend of a transactions' + // outputs. This implies that it must be present for some other output of the same tx. + const Coin& alternate = AccessByTxid(view, out.hashMalFix); + if (!alternate.IsSpent()) { + undo.nHeight = alternate.nHeight; + undo.fCoinBase = alternate.fCoinBase; + } else { + return DISCONNECT_FAILED; // adding output for transaction without known metadata + } + } + // The potential_overwrite parameter to AddCoin is only allowed to be false if we know for + // sure that the coin did not already exist in the cache. As we have queried for that above + // using HaveCoin, we don't need to guess. When fClean is false, a coin already existed and + // it is an overwrite. + view.AddCoin(out, std::move(undo), !fClean); + + return fClean ? DISCONNECT_OK : DISCONNECT_UNCLEAN; +} + +/** Undo the effects of this block (with given index) on the UTXO set represented by coins. + * When FAILED is returned, view is left in an indeterminate state. */ +DisconnectResult CChainState::DisconnectBlock(const CBlock& block, const CBlockIndex* pindex, CCoinsViewCache& view) +{ + bool fClean = true; + + CBlockUndo blockUndo; + if (!UndoReadFromDisk(blockUndo, pindex)) { + error("DisconnectBlock(): failure reading undo data"); + return DISCONNECT_FAILED; + } + + if (blockUndo.vtxundo.size() + 1 != block.vtx.size()) { + error("DisconnectBlock(): block and undo data inconsistent"); + return DISCONNECT_FAILED; + } + + // undo transactions in reverse order + for (int i = block.vtx.size() - 1; i >= 0; i--) { + const CTransaction &tx = *(block.vtx[i]); + uint256 hash = tx.GetHashMalFix(); + bool is_coinbase = tx.IsCoinBase(); + + // Check that all outputs are available and match the outputs in the block itself + // exactly. + for (size_t o = 0; o < tx.vout.size(); o++) { + if (!tx.vout[o].scriptPubKey.IsUnspendable()) { + COutPoint out(hash, o); + ColorIdentifier cid = GetColorIdFromScript(tx.vout[o].scriptPubKey); + Coin coin; + bool is_spent = view.SpendCoin(out, &coin); + ColorIdentifier coincid = GetColorIdFromScript(coin.out.scriptPubKey); + if (!is_spent || tx.vout[o] != coin.out || pindex->nHeight != coin.nHeight || is_coinbase != coin.fCoinBase || cid != coincid ) { + fClean = false; // transaction output mismatch + } + } + } + + // restore inputs + if (i > 0) { // not coinbases + CTxUndo &txundo = blockUndo.vtxundo[i-1]; + if (txundo.vprevout.size() != tx.vin.size()) { + error("DisconnectBlock(): transaction and undo data inconsistent"); + return DISCONNECT_FAILED; + } + for (unsigned int j = tx.vin.size(); j-- > 0;) { + const COutPoint &out = tx.vin[j].prevout; + int res = ApplyTxInUndo(std::move(txundo.vprevout[j]), view, out); + if (res == DISCONNECT_FAILED) return DISCONNECT_FAILED; + fClean = fClean && res != DISCONNECT_UNCLEAN; + } + // At this point, all of txundo.vprevout should have been moved out. + } + } + + // move best block pointer to prevout block + view.SetBestBlock(pindex->pprev->GetBlockHash()); + + return fClean ? DISCONNECT_OK : DISCONNECT_UNCLEAN; +} + +static bool FindUndoPos(CValidationState &state, int nFile, CDiskBlockPos &pos, unsigned int nAddSize) +{ + pos.nFile = nFile; + + LOCK(cs_LastBlockFile); + + unsigned int nNewSize; + pos.nPos = vinfoBlockFile[nFile].nUndoSize; + nNewSize = vinfoBlockFile[nFile].nUndoSize += nAddSize; + setDirtyFileInfo.insert(nFile); + + unsigned int nOldChunks = (pos.nPos + UNDOFILE_CHUNK_SIZE - 1) / UNDOFILE_CHUNK_SIZE; + unsigned int nNewChunks = (nNewSize + UNDOFILE_CHUNK_SIZE - 1) / UNDOFILE_CHUNK_SIZE; + if (nNewChunks > nOldChunks) { + if (fPruneMode) + fCheckForPruning = true; + if (CheckDiskSpace(nNewChunks * UNDOFILE_CHUNK_SIZE - pos.nPos, true)) { + FILE *file = OpenUndoFile(pos); + if (file) { + LogPrintf("Pre-allocating up to position 0x%x in rev%05u.dat\n", nNewChunks * UNDOFILE_CHUNK_SIZE, pos.nFile); + AllocateFileRange(file, pos.nPos, nNewChunks * UNDOFILE_CHUNK_SIZE - pos.nPos); + fclose(file); + } + } + else + return state.Error("out of disk space"); + } + + return true; +} + +bool UndoWriteToDisk(const CBlockUndo& blockundo, CDiskBlockPos& pos, const uint256& hashBlock, const CMessageHeader::MessageStartChars& messageStart) +{ + // Open history file to append + CAutoFile fileout(OpenUndoFile(pos), SER_DISK, CLIENT_VERSION); + if (fileout.IsNull()) + return error("%s: OpenUndoFile failed", __func__); + + // Write index header + unsigned int nSize = GetSerializeSize(fileout, blockundo); + fileout << messageStart << nSize; + + // Write undo data + long fileOutPos = ftell(fileout.Get()); + if (fileOutPos < 0) + return error("%s: ftell failed", __func__); + pos.nPos = (unsigned int)fileOutPos; + fileout << blockundo; + + // calculate & write checksum + CHashWriter hasher(SER_GETHASH, PROTOCOL_VERSION); + hasher << hashBlock; + hasher << blockundo; + fileout << hasher.GetHash(); + + return true; +} + +static bool WriteUndoDataForBlock(const CBlockUndo& blockundo, CValidationState& state, CBlockIndex* pindex) +{ + // Write undo information to disk + if (pindex->GetUndoPos().IsNull()) { + CDiskBlockPos _pos; + if (!FindUndoPos(state, pindex->nFile, _pos, ::GetSerializeSize(blockundo, SER_DISK, CLIENT_VERSION) + 40)) + return error("ConnectBlock(): FindUndoPos failed"); + if (!UndoWriteToDisk(blockundo, _pos, pindex->pprev->GetBlockHash(), FederationParams().MessageStart())) + return AbortNode(state, "Failed to write undo data"); + + // update nUndoPos in block index + pindex->nUndoPos = _pos.nPos; + pindex->nStatus |= BLOCK_HAVE_UNDO; + setDirtyBlockIndex.insert(pindex); + } + + return true; +} +/** Check warning conditions and do some notifications on new chain tip set. */ +void static UpdateTip(const CBlockIndex *pindexNew) +{ + AssertLockHeld(::cs_main); + + // New best block + mempool.AddTransactionsUpdated(1); + + { + WaitableLock lock(g_best_block_mutex); + g_best_block = pindexNew->GetBlockHash(); + g_best_block_cv.notify_all(); + } + + LogPrintf("%s: new best=%s height=%d version=0x%08x tx=%lu date='%s' progress=%f cache=%.1fMiB(%utxo)", __func__, /* Continued */ + pindexNew->GetBlockHash().ToString(), pindexNew->nHeight, pindexNew->nFeatures, + (unsigned long)pindexNew->nChainTx, FormatISO8601DateTime(pindexNew->GetBlockTime()), + GuessVerificationProgress(Params().TxData(), pindexNew), pcoinsTip->DynamicMemoryUsage() * (1.0 / (1<<20)), pcoinsTip->GetCacheSize()); + LogPrintf("\n"); + +} + +static void CheckForkWarningConditionsOnNewFork(CBlockIndex* pindexNewForkTip) +{ + AssertLockHeld(cs_main); + // If we are on a fork that is sufficiently large, set a warning flag + CBlockIndex* pfork = pindexNewForkTip; + CBlockIndex* plonger = chainActive.Tip(); + while (pfork && pfork != plonger) + { + while (plonger && plonger->nHeight > pfork->nHeight) + plonger = plonger->pprev; + if (pfork == plonger) + break; + pfork = pfork->pprev; + } + + // We define a condition where we should warn the user about as a fork of at least 2 blocks + // with a tip within 72 blocks (+/- 12 hours if no one mines it) of ours + // In SignedBlocks, forks are not going to be raise. It shows chain governance is broken. + // So that, we use 2 blocks not 7(in bitcoin core, that uses 7). 1 block fork will be able to happen. + // Because of block hash malleability. + // or a chain that is entirely longer than ours and invalid (note that this should be detected by both) + // We define it this way because it allows us to only store the highest fork tip (+ base) which meets + // the 7-block condition and from this always have the most-likely-to-cause-warning fork + if (pfork && (!pindexBestForkTip || pindexNewForkTip->nHeight > pindexBestForkTip->nHeight) && + pindexNewForkTip->nHeight - pfork->nHeight > 2 && chainActive.Height() - pindexNewForkTip->nHeight < 72) + { + pindexBestForkTip = pindexNewForkTip; + pindexBestForkBase = pfork; + } + + CheckForkWarningConditions(); +} + + +/* Make mempool consistent after a reorg, by re-adding or recursively erasing + * disconnected block transactions from the mempool, and also removing any + * other transactions from the mempool that are no longer valid given the new + * tip/height. + * + * Note: we assume that disconnectpool only contains transactions that are NOT + * confirmed in the current chain nor already in the mempool (otherwise, + * in-mempool descendants of such transactions would be removed). + * + * Passing fAddToMempool=false will skip trying to add the transactions back, + * and instead just erase from the mempool as needed. + */ + +static void UpdateMempoolForReorg(DisconnectedBlockTransactions &disconnectpool, bool fAddToMempool) +{ + AssertLockHeld(cs_main); + std::vector vHashUpdate; + // disconnectpool's insertion_order index sorts the entries from + // oldest to newest, but the oldest entry will be the last tx from the + // latest mined block that was disconnected. + // Iterate disconnectpool in reverse, so that we add transactions + // back to the mempool starting with the earliest transaction that had + // been previously seen in a block. + auto it = disconnectpool.queuedTx.get().rbegin(); + while (it != disconnectpool.queuedTx.get().rend()) { + // ignore validation errors in resurrected transactions + CTxMempoolAcceptanceOptions opt; + opt.flags = MempoolAcceptanceFlags::BYPASSS_LIMITS; + if (!fAddToMempool || (*it)->IsCoinBase() || + !AcceptToMemoryPool(*it, opt) ){ + // If the transaction doesn't make it in to the mempool, remove any + // transactions that depend on it (which would now be orphans). + mempool.removeRecursive(**it, MemPoolRemovalReason::REORG); + } else if (mempool.exists((*it)->GetHashMalFix())) { + vHashUpdate.push_back((*it)->GetHashMalFix()); + } + ++it; + } + disconnectpool.queuedTx.clear(); + // AcceptToMemoryPool/addUnchecked all assume that new mempool entries have + // no in-mempool children, which is generally not true when adding + // previously-confirmed transactions back to the mempool. + // UpdateTransactionsFromBlock finds descendants of any transactions in + // the disconnectpool that were added back and cleans up the mempool state. + mempool.UpdateTransactionsFromBlock(vHashUpdate); + + // We also need to remove any now-immature transactions + mempool.removeForReorg(pcoinsTip.get(), chainActive.Tip()->nHeight + 1, STANDARD_LOCKTIME_VERIFY_FLAGS); + // Re-limit mempool size, in case we added any transactions + LimitMempoolSize(mempool, gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000, gArgs.GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY) * 60 * 60); +} + +/** Apply the effects of this block (with given index) on the UTXO set represented by coins. + * Validity checks that depend on the UTXO set are also done; ConnectBlock() + * can fail if those validity checks fail (among other reasons). */ +bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pindex, + CCoinsViewCache& view, bool fJustCheck) +{ + AssertLockHeld(cs_main); + assert(pindex); + uint256 block_hash{block.GetHash()}; + assert(*pindex->phashBlock == block_hash); + + int64_t nTimeStart = GetTimeMicros(); + + // Check it again in case a previous version let a bad block in + // NOTE: We don't currently (re-)invoke ContextualCheckBlock() or + // ContextualCheckBlockHeader() here. This means that if we add a new + // consensus rule that is enforced in one of those two functions, then we + // may have let in a block that violates the rule prior to updating the + // software, and we would NOT be enforcing the rule here. Fully solving + // upgrade from one software version to the next after a consensus rule + // change is potentially tricky and issue-specific (see RewindBlockIndex() + // for one general approach that was used for BIP 141 deployment). + // Also, currently the rule against blocks more than 2 hours in the future + // is enforced in ContextualCheckBlockHeader(); we wouldn't want to + // re-enforce that rule here (at least until we make it impossible for + // GetAdjustedTime() to go backward). + if (!CheckBlock(block, state, !fJustCheck, !fJustCheck)) { + if (state.CorruptionPossible()) { + // We don't write down blocks to disk if they may have been + // corrupted, so this should be impossible unless we're having hardware + // problems. + return AbortNode(state, "Corrupt block found indicating potential hardware failure; shutting down"); + } + return error("%s: Consensus::CheckBlock: %s", __func__, FormatStateMessage(state)); + } + + // verify that the view's current state corresponds to the previous block + uint256 hashPrevBlock = pindex->pprev == nullptr ? uint256() : pindex->pprev->GetBlockHash(); + assert(hashPrevBlock == view.GetBestBlock()); + + // Special case for the genesis block, skipping connection of its transactions + // (its coinbase is unspendable) + if (block.GetHash() == FederationParams().GenesisBlock().GetHash()) { + if (!fJustCheck) + view.SetBestBlock(pindex->GetBlockHash()); + return true; + } + + nBlocksTotal++; + + bool fScriptChecks = true; + if (!hashAssumeValid.IsNull()) { + // We've been configured with the hash of a block which has been externally verified to have a valid history. + // A suitable default value is included with the software and updated from time to time. Because validity + // relative to a piece of software is an objective fact these defaults can be easily reviewed. + // This setting doesn't force the selection of any particular chain but makes validating some faster by + // effectively caching the result of part of the verification. + BlockMap::const_iterator it = mapBlockIndex.find(hashAssumeValid); + if (it != mapBlockIndex.end()) { + if (it->second->GetAncestor(pindex->nHeight) == pindex && + pindexBestHeader->GetAncestor(pindex->nHeight) == pindex) { + // This block is a member of the assumed verified chain and an ancestor of the best header. + // The equivalent time check discourages hash power from extorting the network via DOS attack + // into accepting an invalid block through telling users they must manually set assumevalid. + // Requiring a software change or burying the invalid block, regardless of the setting, makes + // it hard to hide the implication of the demand. This also avoids having release candidates + // that are hardly doing any signature verification at all in testing without having to + // artificially set the default assumed verified block further back. + fScriptChecks = (pindexBestHeader->GetBlockTime() - pindex->GetBlockTime() <= 60 * 60 * 24 * 7 * 2); + } + } + } + + int64_t nTime1 = GetTimeMicros(); nTimeCheck += nTime1 - nTimeStart; + LogPrint(BCLog::BENCH, " - Sanity checks: %.2fms [%.2fs (%.2fms/blk)]\n", MILLI * (nTime1 - nTimeStart), nTimeCheck * MICRO, nTimeCheck * MILLI / nBlocksTotal); + + // Start enforcing BIP68 (sequence locks) and BIP112 (CHECKSEQUENCEVERIFY) using versionbits logic. + int nLockTimeFlags = LOCKTIME_VERIFY_SEQUENCE; + + int64_t nTime2 = GetTimeMicros(); nTimeForks += nTime2 - nTime1; + LogPrint(BCLog::BENCH, " - Fork checks: %.2fms [%.2fs (%.2fms/blk)]\n", MILLI * (nTime2 - nTime1), nTimeForks * MICRO, nTimeForks * MILLI / nBlocksTotal); + + CBlockUndo blockundo; + + CCheckQueueControl control(fScriptChecks && nScriptCheckThreads ? scriptcheckqueue.get() : nullptr); + + std::vector prevheights; + CAmount nFees = 0; + int nInputs = 0; + int64_t nSigOpsCost = 0; + blockundo.vtxundo.reserve(block.vtx.size() - 1); + std::vector txdata; + txdata.reserve(block.vtx.size()); // Required so that pointers to individual PrecomputedTransactionData don't get invalidated + for (unsigned int i = 0; i < block.vtx.size(); i++) + { + const CTransaction &tx = *(block.vtx[i]); + TxColoredCoinBalancesMap inColoredCoinBalances; + + nInputs += tx.vin.size(); + + if (!tx.IsCoinBase()) + { + CAmount txfee = 0; + if (!Consensus::CheckTxInputs(tx, state, view, pindex->nHeight, txfee)) { + return error("%s: Consensus::CheckTxInputs: %s, %s", __func__, tx.GetHashMalFix().ToString(), FormatStateMessage(state)); + } + nFees += txfee; + if (!MoneyRange(nFees)) { + return state.DoS(100, error("%s: accumulated fee in the block out of range.", __func__), + REJECT_INVALID, "bad-txns-accumulated-fee-outofrange"); + } + + // Check that transaction is BIP68 final + // BIP68 lock checks (as opposed to nLockTime checks) must + // be in ConnectBlock because they require the UTXO set + prevheights.resize(tx.vin.size()); + for (size_t j = 0; j < tx.vin.size(); j++) { + prevheights[j] = view.AccessCoin(tx.vin[j].prevout).nHeight; + } + + if (!SequenceLocks(tx, nLockTimeFlags, &prevheights, *pindex)) { + return state.DoS(100, error("%s: contains a non-BIP68-final transaction", __func__), + REJECT_INVALID, "bad-txns-nonfinal"); + } + } + + // Get the script flags for this block + unsigned int flags = GetBlockScriptFlags(pindex); + + // GetTransactionSigOps counts 3 types of sigops: + // * legacy (always) + // * p2sh (when P2SH enabled in flags and excludes coinbase) + // * witness (when witness enabled in flags and excludes coinbase) + nSigOpsCost += GetTransactionSigOps(tx, view, SCRIPT_VERIFY_NONE); + if (nSigOpsCost > GetMaxBlockSigops()) + return state.DoS(100, error("ConnectBlock(): too many sigops"), + REJECT_INVALID, "bad-blk-sigops"); + + txdata.emplace_back(tx); + if (!tx.IsCoinBase()) + { + std::vector vChecks; + bool fCacheResults = fJustCheck; /* Don't cache results if we're actually connecting blocks (still consult the cache, though) */ + if (!CheckInputs(tx, state, view, fScriptChecks, SCRIPT_VERIFY_NONE, fCacheResults, fCacheResults, txdata[i], inColoredCoinBalances, nScriptCheckThreads ? &vChecks : nullptr)) + return error("ConnectBlock(): CheckInputs on %s failed with %s", + tx.GetHashMalFix().ToString(), FormatStateMessage(state)); + control.Add(std::move(vChecks)); + } + + CTxUndo undoDummy; + if (i > 0) { + blockundo.vtxundo.push_back(CTxUndo()); + } + UpdateCoins(tx, view, i == 0 ? undoDummy : blockundo.vtxundo.back(), pindex->nHeight); + } + int64_t nTime3 = GetTimeMicros(); nTimeConnect += nTime3 - nTime2; + LogPrint(BCLog::BENCH, " - Connect %u transactions: %.2fms (%.3fms/tx, %.3fms/txin) [%.2fs (%.2fms/blk)]\n", (unsigned)block.vtx.size(), MILLI * (nTime3 - nTime2), MILLI * (nTime3 - nTime2) / block.vtx.size(), nInputs <= 1 ? 0 : MILLI * (nTime3 - nTime2) / (nInputs-1), nTimeConnect * MICRO, nTimeConnect * MILLI / nBlocksTotal); + + CAmount blockReward = nFees + GetBlockSubsidy(pindex->nHeight, Params().GetConsensus()); + if (block.vtx[0]->GetValueOut(ColorIdentifier()) > blockReward) + return state.DoS(100, + error("ConnectBlock(): coinbase pays too much (actual=%d vs limit=%d)", + block.vtx[0]->GetValueOut(ColorIdentifier()), blockReward), + REJECT_INVALID, "bad-cb-amount"); + + if (!control.Wait()) + return state.DoS(100, error("%s: CheckQueue failed", __func__), REJECT_INVALID, "block-validation-failed"); + int64_t nTime4 = GetTimeMicros(); nTimeVerify += nTime4 - nTime2; + LogPrint(BCLog::BENCH, " - Verify %u txins: %.2fms (%.3fms/txin) [%.2fs (%.2fms/blk)]\n", nInputs - 1, MILLI * (nTime4 - nTime2), nInputs <= 1 ? 0 : MILLI * (nTime4 - nTime2) / (nInputs-1), nTimeVerify * MICRO, nTimeVerify * MILLI / nBlocksTotal); + + if (fJustCheck) + return true; + + if (!WriteUndoDataForBlock(blockundo, state, pindex)) + return false; + + if (!pindex->IsValid(BLOCK_VALID_SCRIPTS)) { + pindex->RaiseValidity(BLOCK_VALID_SCRIPTS); + setDirtyBlockIndex.insert(pindex); + } + + assert(pindex->phashBlock); + // add this block to the view's block chain + view.SetBestBlock(pindex->GetBlockHash()); + + int64_t nTime5 = GetTimeMicros(); nTimeIndex += nTime5 - nTime4; + LogPrint(BCLog::BENCH, " - Index writing: %.2fms [%.2fs (%.2fms/blk)]\n", MILLI * (nTime5 - nTime4), nTimeIndex * MICRO, nTimeIndex * MILLI / nBlocksTotal); + + int64_t nTime6 = GetTimeMicros(); nTimeCallbacks += nTime6 - nTime5; + LogPrint(BCLog::BENCH, " - Callbacks: %.2fms [%.2fs (%.2fms/blk)]\n", MILLI * (nTime6 - nTime5), nTimeCallbacks * MICRO, nTimeCallbacks * MILLI / nBlocksTotal); + + TRACE6(validation, block_connected, + block_hash.begin(), + pindex->nHeight, + block.vtx.size(), + nInputs, + nSigOpsCost, + nTime5 - nTimeStart // in microseconds (µs) + ); + + return true; +} + + +/** Disconnect chainActive's tip. + * After calling, the mempool will be in an inconsistent state, with + * transactions from disconnected blocks being added to disconnectpool. You + * should make the mempool consistent again by calling UpdateMempoolForReorg. + * with cs_main held. + * + * If disconnectpool is nullptr, then no disconnected transactions are added to + * disconnectpool (note that the caller is responsible for mempool consistency + * in any case). + */ +bool CChainState::DisconnectTip(CValidationState& state, DisconnectedBlockTransactions *disconnectpool) +{ + AssertLockHeld(cs_main); + + CBlockIndex *pindexDelete = chainActive.Tip(); + assert(pindexDelete); + // Read block from disk. + std::shared_ptr pblock = std::make_shared(); + CBlock& block = *pblock; + if (!ReadBlockFromDisk(block, pindexDelete)) + return AbortNode(state, "Failed to read block"); + // Apply the block atomically to the chain state. + int64_t nStart = GetTimeMicros(); + { + CCoinsViewCache view(pcoinsTip.get()); + assert(view.GetBestBlock() == pindexDelete->GetBlockHash()); + if (DisconnectBlock(block, pindexDelete, view) != DISCONNECT_OK) + return error("DisconnectTip(): DisconnectBlock %s failed", pindexDelete->GetBlockHash().ToString()); + bool flushed = view.Flush(); + assert(flushed); + } + LogPrint(BCLog::BENCH, "- Disconnect block: %.2fms\n", (GetTimeMicros() - nStart) * MILLI); + // Write the chain state to disk, if necessary. + if (!FlushStateToDisk(state, FlushStateMode::IF_NEEDED)) + return false; + + if (disconnectpool) { + // Save transactions to re-add to mempool at end of reorg + for (auto it = block.vtx.rbegin(); it != block.vtx.rend(); ++it) { + disconnectpool->addTransaction(*it); + } + while (disconnectpool->DynamicMemoryUsage() > MAX_DISCONNECTED_TX_POOL_SIZE * 1000) { + // Drop the earliest entry, and remove its children from the mempool. + auto it = disconnectpool->queuedTx.get().begin(); + mempool.removeRecursive(**it, MemPoolRemovalReason::REORG); + disconnectpool->removeEntry(it); + } + } + + chainActive.SetTip(pindexDelete->pprev); + + UpdateTip(pindexDelete->pprev); + + //TODO: if xfield information change is being discarded during this reorg remove it from xFieldHistory here + + // Let wallets know transactions went from 1-confirmed to + // 0-confirmed or conflicted: + GetMainSignals().BlockDisconnected(pblock); + return true; +} + + +/** + * Connect a new block to chainActive. pblock is either nullptr or a pointer to a CBlock + * corresponding to pindexNew, to bypass loading it again from disk. + * + * The block is added to connectTrace if connection succeeds. + */ +bool CChainState::ConnectTip(CValidationState& state, CBlockIndex* pindexNew, const std::shared_ptr& pblock, ConnectTrace& connectTrace, DisconnectedBlockTransactions &disconnectpool) +{ + AssertLockHeld(cs_main); + + assert(pindexNew->pprev == chainActive.Tip()); + // Read block from disk. + int64_t nTime1 = GetTimeMicros(); + std::shared_ptr pthisBlock; + if (!pblock) { + std::shared_ptr pblockNew = std::make_shared(); + if (!ReadBlockFromDisk(*pblockNew, pindexNew)) + return AbortNode(state, "Failed to read block"); + pthisBlock = pblockNew; + } else { + pthisBlock = pblock; + } + const CBlock& blockConnecting = *pthisBlock; + // Apply the block atomically to the chain state. + int64_t nTime2 = GetTimeMicros(); nTimeReadFromDisk += nTime2 - nTime1; + int64_t nTime3; + LogPrint(BCLog::BENCH, " - Load block from disk: %.2fms [%.2fs]\n", (nTime2 - nTime1) * MILLI, nTimeReadFromDisk * MICRO); + { + CCoinsViewCache view(pcoinsTip.get()); + bool rv = ConnectBlock(blockConnecting, state, pindexNew, view); + GetMainSignals().BlockChecked(blockConnecting, state); + if (!rv) { + if (state.IsInvalid()) + InvalidBlockFound(pindexNew, state); + return error("ConnectTip(): ConnectBlock %s failed", pindexNew->GetBlockHash().ToString()); + } + + // if the block was added successfully and it is a federation block, + // make sure that the xfield from this block is added to xFieldHistory + CXFieldHistory xfieldHistory; + if(blockConnecting.xfield.IsValid() + && blockConnecting.GetHeight() > 0 + && IsXFieldNew(blockConnecting.xfield, &xfieldHistory)) + { + XFieldChange newChange(blockConnecting.xfield.xfieldValue, blockConnecting.GetHeight() + 1, blockConnecting.GetHash()); + xfieldHistory.Add(blockConnecting.xfield.xfieldType, newChange); + pblocktree->WriteXField(newChange); + } + + nTime3 = GetTimeMicros(); nTimeConnectTotal += nTime3 - nTime2; + LogPrint(BCLog::BENCH, " - Connect total: %.2fms [%.2fs (%.2fms/blk)]\n", (nTime3 - nTime2) * MILLI, nTimeConnectTotal * MICRO, nTimeConnectTotal * MILLI / nBlocksTotal); + bool flushed = view.Flush(); + assert(flushed); + } + + int64_t nTime4 = GetTimeMicros(); nTimeFlush += nTime4 - nTime3; + LogPrint(BCLog::BENCH, " - Flush: %.2fms [%.2fs (%.2fms/blk)]\n", (nTime4 - nTime3) * MILLI, nTimeFlush * MICRO, nTimeFlush * MILLI / nBlocksTotal); + // Write the chain state to disk, if necessary. + if (!FlushStateToDisk(state, FlushStateMode::IF_NEEDED)) + return false; + int64_t nTime5 = GetTimeMicros(); nTimeChainState += nTime5 - nTime4; + LogPrint(BCLog::BENCH, " - Writing chainstate: %.2fms [%.2fs (%.2fms/blk)]\n", (nTime5 - nTime4) * MILLI, nTimeChainState * MICRO, nTimeChainState * MILLI / nBlocksTotal); + // Remove conflicting transactions from the mempool.; + mempool.removeForBlock(blockConnecting.vtx, pindexNew->nHeight); + disconnectpool.removeForBlock(blockConnecting.vtx); + // Update chainActive & related variables. + chainActive.SetTip(pindexNew); + UpdateTip(pindexNew); + + int64_t nTime6 = GetTimeMicros(); nTimePostConnect += nTime6 - nTime5; nTimeTotal += nTime6 - nTime1; + LogPrint(BCLog::BENCH, " - Connect postprocess: %.2fms [%.2fs (%.2fms/blk)]\n", (nTime6 - nTime5) * MILLI, nTimePostConnect * MICRO, nTimePostConnect * MILLI / nBlocksTotal); + LogPrint(BCLog::BENCH, "- Connect block: %.2fms [%.2fs (%.2fms/blk)]\n", (nTime6 - nTime1) * MILLI, nTimeTotal * MICRO, nTimeTotal * MILLI / nBlocksTotal); + + connectTrace.BlockConnected(pindexNew, std::move(pthisBlock)); + return true; +} + + +/** + * Return the tip of the chain with the most work in it, that isn't + * known to be invalid (it's however far from certain to be valid). + */ +CBlockIndex* CChainState::FindMostWorkChain() { + do { + CBlockIndex *pindexNew = nullptr; + + // Find the best candidate header. + { + std::set::reverse_iterator it = setBlockIndexCandidates.rbegin(); + if (it == setBlockIndexCandidates.rend()) + return nullptr; + pindexNew = *it; + } + + // Check whether all blocks on the path between the currently active chain and the candidate are valid. + // Just going until the active chain is an optimization, as we know all blocks in it are valid already. + CBlockIndex *pindexTest = pindexNew; + bool fInvalidAncestor = false; + while (pindexTest && !chainActive.Contains(pindexTest)) { + assert(pindexTest->nChainTx || pindexTest->nHeight == 0); + + // Pruned nodes may have entries in setBlockIndexCandidates for + // which block files have been deleted. Remove those as candidates + // for the most work chain if we come across them; we can't switch + // to a chain unless we have all the non-active-chain parent blocks. + bool fFailedChain = pindexTest->nStatus & BLOCK_FAILED_MASK; + bool fMissingData = !(pindexTest->nStatus & BLOCK_HAVE_DATA); + if (fFailedChain || fMissingData) { + // Candidate chain is not usable (either invalid or missing data) + if (fFailedChain && (pindexBestInvalid == nullptr || pindexNew->nHeight > pindexBestInvalid->nHeight)) + pindexBestInvalid = pindexNew; + CBlockIndex *pindexFailed = pindexNew; + // Remove the entire chain from the set. + while (pindexTest != pindexFailed) { + if (fFailedChain) { + pindexFailed->nStatus |= BLOCK_FAILED_CHILD; + } else if (fMissingData) { + // If we're missing data, then add back to mapBlocksUnlinked, + // so that if the block arrives in the future we can try adding + // to setBlockIndexCandidates again. + mapBlocksUnlinked.insert(std::make_pair(pindexFailed->pprev, pindexFailed)); + } + setBlockIndexCandidates.erase(pindexFailed); + pindexFailed = pindexFailed->pprev; + } + setBlockIndexCandidates.erase(pindexTest); + fInvalidAncestor = true; + break; + } + pindexTest = pindexTest->pprev; + } + if (!fInvalidAncestor) + return pindexNew; + } while(true); +} + +/** Delete all entries in setBlockIndexCandidates that are worse than the current tip. */ +void CChainState::PruneBlockIndexCandidates() { + // Note that we can't delete the current block itself, as we may need to return to it later in case a + // reorganization to a better block fails. + std::set::iterator it = setBlockIndexCandidates.begin(); + while (it != setBlockIndexCandidates.end() && setBlockIndexCandidates.value_comp()(*it, chainActive.Tip())) { + setBlockIndexCandidates.erase(it++); + } + // Either the current tip or a successor of it we're working towards is left in setBlockIndexCandidates. + assert(!setBlockIndexCandidates.empty()); +} + +/** + * Try to make some progress towards making pindexMostWork the active block. + * pblock is either nullptr or a pointer to a CBlock corresponding to pindexMostWork. + */ +bool CChainState::ActivateBestChainStep(CValidationState& state, CBlockIndex* pindexMostWork, const std::shared_ptr& pblock, bool& fInvalidFound, ConnectTrace& connectTrace) +{ + AssertLockHeld(cs_main); + + const CBlockIndex *pindexOldTip = chainActive.Tip(); + const CBlockIndex *pindexFork = chainActive.FindFork(pindexMostWork); + + // Disconnect active blocks which are no longer in the best chain. + bool fBlocksDisconnected = false; + DisconnectedBlockTransactions disconnectpool; + while (chainActive.Tip() && chainActive.Tip() != pindexFork) { + if (!DisconnectTip(state, &disconnectpool)) { + // This is likely a fatal error, but keep the mempool consistent, + // just in case. Only remove from the mempool in this case. + UpdateMempoolForReorg(disconnectpool, false); + return false; + } + fBlocksDisconnected = true; + } + + // Build list of new blocks to connect. + std::vector vpindexToConnect; + bool fContinue = true; + int nHeight = pindexFork ? pindexFork->nHeight : -1; + while (fContinue && nHeight != pindexMostWork->nHeight) { + // Don't iterate the entire list of potential improvements toward the best tip, as we likely only need + // a few blocks along the way. + int nTargetHeight = std::min(nHeight + 32, pindexMostWork->nHeight); + vpindexToConnect.clear(); + vpindexToConnect.reserve(nTargetHeight - nHeight); + CBlockIndex *pindexIter = pindexMostWork->GetAncestor(nTargetHeight); + while (pindexIter && pindexIter->nHeight != nHeight) { + vpindexToConnect.push_back(pindexIter); + pindexIter = pindexIter->pprev; + } + nHeight = nTargetHeight; + + // Connect new blocks. + for (CBlockIndex *pindexConnect : reverse_iterate(vpindexToConnect)) { + if (!ConnectTip(state, pindexConnect, pindexConnect == pindexMostWork ? pblock : std::shared_ptr(), connectTrace, disconnectpool)) { + if (state.IsInvalid()) { + // The block violates a consensus rule. + if (!state.CorruptionPossible()) { + InvalidChainFound(vpindexToConnect.front()); + } + state = CValidationState(); + fInvalidFound = true; + fContinue = false; + break; + } else { + // A system error occurred (disk space, database error, ...). + // Make the mempool consistent with the current tip, just in case + // any observers try to use it before shutdown. + UpdateMempoolForReorg(disconnectpool, false); + return false; + } + } else { + PruneBlockIndexCandidates(); + if (!pindexOldTip || chainActive.Tip()->nHeight > pindexOldTip->nHeight) { + // We're in a better position than we were. Return temporarily to release the lock. + fContinue = false; + break; + } + } + } + } + + if (fBlocksDisconnected) { + // If any blocks were disconnected, disconnectpool may be non empty. Add + // any disconnected transactions back to the mempool. + UpdateMempoolForReorg(disconnectpool, true); + } + mempool.check(pcoinsTip.get()); + + // Callbacks/notifications for a new best chain. + if (fInvalidFound) + CheckForkWarningConditionsOnNewFork(vpindexToConnect.back()); + else + CheckForkWarningConditions(); + + return true; +} + + +/** + * Make the best chain active, in multiple steps. The result is either failure + * or an activated best chain. pblock is either nullptr or a pointer to a block + * that is already loaded (to avoid loading it again from disk). + * + * ActivateBestChain is split into steps (see ActivateBestChainStep) so that + * we avoid holding cs_main for an extended period of time; the length of this + * call may be quite long during reindexing or a substantial reorg. + */ +bool CChainState::ActivateBestChain(CValidationState &state, std::shared_ptr pblock) { + // Note that while we're often called here from ProcessNewBlock, this is + // far from a guarantee. Things in the P2P/RPC will often end up calling + // us in the middle of ProcessNewBlock - do not assume pblock is set + // sanely for performance or correctness! + AssertLockNotHeld(cs_main); + + // ABC maintains a fair degree of expensive-to-calculate internal state + // because this function periodically releases cs_main so that it does not lock up other threads for too long + // during large connects - and to allow for e.g. the callback queue to drain + // we use m_cs_chainstate to enforce mutual exclusion so that only one caller may execute this function at a time + LOCK(m_cs_chainstate); + + CBlockIndex *pindexMostWork = nullptr; + CBlockIndex *pindexNewTip = nullptr; + int nStopAtHeight = gArgs.GetArg("-stopatheight", DEFAULT_STOPATHEIGHT); + do { + if (GetMainSignals().CallbacksPending() > 10) { + // Block until the validation queue drains. This should largely + // never happen in normal operation, however may happen during + // reindex, causing memory blowup if we run too far ahead. + // Note that if a validationinterface callback ends up calling + // ActivateBestChain this may lead to a deadlock! We should + // probably have a DEBUG_LOCKORDER test for this in the future. + SyncWithValidationInterfaceQueue(); + } + + { + LOCK(cs_main); + CBlockIndex* starting_tip = chainActive.Tip(); + bool blocks_connected = false; + do { + // We absolutely may not unlock cs_main until we've made forward progress + // (with the exception of shutdown due to hardware issues, low disk space, etc). + ConnectTrace connectTrace(mempool); // Destructed before cs_main is unlocked + + if (pindexMostWork == nullptr) { + pindexMostWork = FindMostWorkChain(); + } + + // Whether we have anything to do at all. + if (pindexMostWork == nullptr || pindexMostWork == chainActive.Tip()) { + break; + } + + bool fInvalidFound = false; + std::shared_ptr nullBlockPtr; + if (!ActivateBestChainStep(state, pindexMostWork, pblock && pblock->GetHash() == pindexMostWork->GetBlockHash() ? pblock : nullBlockPtr, fInvalidFound, connectTrace)) + return false; + blocks_connected = true; + + if (fInvalidFound) { + // Wipe cache, we may need another branch now. + pindexMostWork = nullptr; + } + pindexNewTip = chainActive.Tip(); + + for (const PerBlockConnectTrace& trace : connectTrace.GetBlocksConnected()) { + assert(trace.pblock && trace.pindex); + GetMainSignals().BlockConnected(trace.pblock, trace.pindex, trace.conflictedTxs); + } + } while (!chainActive.Tip() || (starting_tip && CBlockIndexWorkComparator()(chainActive.Tip(), starting_tip))); + if (!blocks_connected) return true; + + const CBlockIndex* pindexFork = chainActive.FindFork(starting_tip); + bool fInitialDownload = IsInitialBlockDownload(); + + // Notify external listeners about the new tip. + // Enqueue while holding cs_main to ensure that UpdatedBlockTip is called in the order in which blocks are connected + if (pindexFork != pindexNewTip) { + // Notify ValidationInterface subscribers + GetMainSignals().UpdatedBlockTip(pindexNewTip, pindexFork, fInitialDownload); + + // Always notify the UI if a new block tip was connected + uiInterface.NotifyBlockTip(fInitialDownload, pindexNewTip); + } + } + // When we reach this point, we switched to a new tip (stored in pindexNewTip). + + if (nStopAtHeight && pindexNewTip && pindexNewTip->nHeight >= nStopAtHeight) StartShutdown(); + + // We check shutdown only after giving ActivateBestChainStep a chance to run once so that we + // never shutdown before connecting the genesis block during LoadChainTip(). Previously this + // caused an assert() failure during shutdown in such cases as the UTXO DB flushing checks + // that the best block hash is non-null. + if (ShutdownRequested()) + break; + } while (pindexNewTip != pindexMostWork); + CheckBlockIndex(); + + // Write changes periodically to disk, after relay. + if (!FlushStateToDisk(state, FlushStateMode::PERIODIC)) { + return false; + } + + return true; +} + + +bool CChainState::InvalidateBlock(CValidationState& state, CBlockIndex *pindex) +{ + // We first disconnect backwards and then mark the blocks as invalid. + // This prevents a case where pruned nodes may fail to invalidateblock + // and be left unable to start as they have no tip candidates (as there + // are no blocks that meet the "have data and are not invalid per + // nStatus" criteria for inclusion in setBlockIndexCandidates). + + bool pindex_was_in_chain = false; + CBlockIndex *invalid_walk_tip = chainActive.Tip(); + + DisconnectedBlockTransactions disconnectpool; + while (chainActive.Contains(pindex)) { + pindex_was_in_chain = true; + // ActivateBestChain considers blocks already in chainActive + // unconditionally valid already, so force disconnect away from it. + if (!DisconnectTip(state, &disconnectpool)) { + // It's probably hopeless to try to make the mempool consistent + // here if DisconnectTip failed, but we can try. + UpdateMempoolForReorg(disconnectpool, false); + return false; + } + } + + // Now mark the blocks we just disconnected as descendants invalid + // (note this may not be all descendants). + while (pindex_was_in_chain && invalid_walk_tip != pindex) { + invalid_walk_tip->nStatus |= BLOCK_FAILED_CHILD; + setDirtyBlockIndex.insert(invalid_walk_tip); + setBlockIndexCandidates.erase(invalid_walk_tip); + invalid_walk_tip = invalid_walk_tip->pprev; + } + + // Mark the block itself as invalid. + pindex->nStatus |= BLOCK_FAILED_VALID; + setDirtyBlockIndex.insert(pindex); + setBlockIndexCandidates.erase(pindex); + m_failed_blocks.insert(pindex); + + // DisconnectTip will add transactions to disconnectpool; try to add these + // back to the mempool. + UpdateMempoolForReorg(disconnectpool, true); + + // The resulting new best tip may not be in setBlockIndexCandidates anymore, so + // add it again. + BlockMap::iterator it = mapBlockIndex.begin(); + while (it != mapBlockIndex.end()) { + if (it->second->IsValid(BLOCK_VALID_TRANSACTIONS) && it->second->nChainTx && !setBlockIndexCandidates.value_comp()(it->second, chainActive.Tip())) { + setBlockIndexCandidates.insert(it->second); + } + it++; + } + + InvalidChainFound(pindex); + + // Only notify about a new block tip if the active chain was modified. + if (pindex_was_in_chain) { + uiInterface.NotifyBlockTip(IsInitialBlockDownload(), pindex->pprev); + } + return true; +} + + +void CChainState::ResetBlockFailureFlags(CBlockIndex *pindex) { + AssertLockHeld(cs_main); + + int nHeight = pindex->nHeight; + + // Remove the invalidity flag from this block and all its descendants. + BlockMap::iterator it = mapBlockIndex.begin(); + while (it != mapBlockIndex.end()) { + if (!it->second->IsValid() && it->second->GetAncestor(nHeight) == pindex) { + it->second->nStatus &= ~BLOCK_FAILED_MASK; + setDirtyBlockIndex.insert(it->second); + if (it->second->IsValid(BLOCK_VALID_TRANSACTIONS) && it->second->nChainTx && setBlockIndexCandidates.value_comp()(chainActive.Tip(), it->second)) { + setBlockIndexCandidates.insert(it->second); + } + if (it->second == pindexBestInvalid) { + // Reset invalid block marker if it was pointing to one of those. + pindexBestInvalid = nullptr; + } + m_failed_blocks.erase(it->second); + } + it++; + } + + // Remove the invalidity flag from all ancestors too. + while (pindex != nullptr) { + if (pindex->nStatus & BLOCK_FAILED_MASK) { + pindex->nStatus &= ~BLOCK_FAILED_MASK; + setDirtyBlockIndex.insert(pindex); + m_failed_blocks.erase(pindex); + } + pindex = pindex->pprev; + } +} + + +CBlockIndex* CChainState::AddToBlockIndex(const CBlockHeader& block) +{ + AssertLockHeld(cs_main); + + // Check for duplicate + uint256 hash = block.GetHash(); + BlockMap::iterator it = mapBlockIndex.find(hash); + if (it != mapBlockIndex.end()) + return it->second; + + // Construct new block index object + CBlockIndex* pindexNew = new CBlockIndex(block); + // We assign the sequence id to blocks only when the full data is available, + // to avoid miners withholding blocks but broadcasting headers, to get a + // competitive advantage. + pindexNew->nSequenceId = 0; + BlockMap::iterator mi = mapBlockIndex.insert(std::make_pair(hash, pindexNew)).first; + pindexNew->phashBlock = &((*mi).first); + BlockMap::iterator miPrev = mapBlockIndex.find(block.hashPrevBlock); + if (miPrev != mapBlockIndex.end()) + { + pindexNew->pprev = (*miPrev).second; + pindexNew->nHeight = pindexNew->pprev->nHeight + 1; + pindexNew->BuildSkip(); + } + pindexNew->nTimeMax = (pindexNew->pprev ? std::max(pindexNew->pprev->nTimeMax, pindexNew->nTime) : pindexNew->nTime); + pindexNew->RaiseValidity(BLOCK_VALID_TREE); + if (pindexBestHeader == nullptr || pindexBestHeader->nHeight < pindexNew->nHeight) + pindexBestHeader = pindexNew; + + setDirtyBlockIndex.insert(pindexNew); + + return pindexNew; +} + +/** Mark a block as having its data received and checked (up to BLOCK_VALID_TRANSACTIONS). */ +void CChainState::ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pindexNew, const CDiskBlockPos& pos) +{ + pindexNew->nTx = block.vtx.size(); + pindexNew->nChainTx = 0; + pindexNew->nFile = pos.nFile; + pindexNew->nDataPos = pos.nPos; + pindexNew->nUndoPos = 0; + pindexNew->nStatus |= BLOCK_HAVE_DATA; + pindexNew->RaiseValidity(BLOCK_VALID_TRANSACTIONS); + setDirtyBlockIndex.insert(pindexNew); + + if (pindexNew->pprev == nullptr || pindexNew->pprev->nChainTx) { + // If pindexNew is the genesis block or all parents are BLOCK_VALID_TRANSACTIONS. + std::deque queue; + queue.push_back(pindexNew); + + // Recursively process any descendant blocks that now may be eligible to be connected. + while (!queue.empty()) { + CBlockIndex *pindex = queue.front(); + queue.pop_front(); + pindex->nChainTx = (pindex->pprev ? pindex->pprev->nChainTx : 0) + pindex->nTx; + { + LOCK(cs_nBlockSequenceId); + pindex->nSequenceId = nBlockSequenceId++; + } + if (chainActive.Tip() == nullptr || !setBlockIndexCandidates.value_comp()(pindex, chainActive.Tip())) { + setBlockIndexCandidates.insert(pindex); + } + std::pair::iterator, std::multimap::iterator> range = mapBlocksUnlinked.equal_range(pindex); + while (range.first != range.second) { + std::multimap::iterator it = range.first; + queue.push_back(it->second); + range.first++; + mapBlocksUnlinked.erase(it); + } + } + } else { + if (pindexNew->pprev && pindexNew->pprev->IsValid(BLOCK_VALID_TREE)) { + mapBlocksUnlinked.insert(std::make_pair(pindexNew->pprev, pindexNew)); + } + } +} + + +bool CChainState::AcceptBlockHeader(const CBlockHeader& block, CValidationState& state, CBlockIndex** ppindex, CXFieldHistoryMap* pxfieldHistory) +{ + AssertLockHeld(cs_main); + // Check for duplicate + uint256 hash = block.GetHash(); + BlockMap::iterator miSelf = mapBlockIndex.find(hash); + CBlockIndex *pindex = nullptr; + if (hash != FederationParams().GenesisBlock().GetHash()) { + if (miSelf != mapBlockIndex.end()) { + // Block header is already known. + pindex = miSelf->second; + if (ppindex) + *ppindex = pindex; + if (pindex->nStatus & BLOCK_FAILED_MASK) + return state.Invalid(error("%s: block %s is marked invalid", __func__, hash.ToString()), 0, "duplicate"); + return true; + } + + if (!CheckBlockHeader(block, state, pxfieldHistory)) + return error("%s: Consensus::CheckBlockHeader: %s, %s", __func__, hash.ToString(), FormatStateMessage(state)); + + // Get prev block index + CBlockIndex* pindexPrev = nullptr; + BlockMap::iterator mi = mapBlockIndex.find(block.hashPrevBlock); + if (mi == mapBlockIndex.end()) + return state.DoS(10, error("%s: prev block not found", __func__), 0, "prev-blk-not-found"); + pindexPrev = (*mi).second; + if (pindexPrev->nStatus & BLOCK_FAILED_MASK) + return state.DoS(100, error("%s: prev block invalid", __func__), REJECT_INVALID, "bad-prevblk"); + if (!ContextualCheckBlockHeader(block, state, pindexPrev, GetAdjustedTime())) + return error("%s: Consensus::ContextualCheckBlockHeader: %s, %s", __func__, hash.ToString(), FormatStateMessage(state)); + + // If the previous block index isn't valid, determine if it descends from any block which + // has been found invalid (m_failed_blocks), then mark pindexPrev and any blocks + // between them as failed. + if (!pindexPrev->IsValid(BLOCK_VALID_SCRIPTS)) { + for (const CBlockIndex* failedit : m_failed_blocks) { + if (pindexPrev->GetAncestor(failedit->nHeight) == failedit) { + assert(failedit->nStatus & BLOCK_FAILED_VALID); + CBlockIndex* invalid_walk = pindexPrev; + while (invalid_walk != failedit) { + invalid_walk->nStatus |= BLOCK_FAILED_CHILD; + setDirtyBlockIndex.insert(invalid_walk); + invalid_walk = invalid_walk->pprev; + } + return state.DoS(100, error("%s: prev block invalid", __func__), REJECT_INVALID, "bad-prevblk"); + } + } + } + } + if (pindex == nullptr) + pindex = AddToBlockIndex(block); + + //if this header was valid and has any xfield change remember it until we finish processing the headers message + if(pxfieldHistory + && block.xfield.IsValid() + && pindex->nHeight > 0 + && IsXFieldNew(block.xfield, pxfieldHistory)) + { + XFieldChange newChange(block.xfield.xfieldValue, pindex->nHeight + 1, block.GetHash()); + pxfieldHistory->Add(block.xfield.xfieldType, newChange); + } + + if (ppindex) + *ppindex = pindex; + + CheckBlockIndex(); + + return true; +} + + +/** Store block on disk. If dbp is non-nullptr, the file is known to already reside on disk */ +bool CChainState::AcceptBlock(const std::shared_ptr& pblock, CValidationState& state, CBlockIndex** ppindex, bool fRequested, const CDiskBlockPos* dbp, bool* fNewBlock, CXFieldHistoryMap* pxfieldHistory) +{ + const CBlock& block = *pblock; + + if (fNewBlock) *fNewBlock = false; + AssertLockHeld(cs_main); + + CBlockIndex *pindexDummy = nullptr; + CBlockIndex *&pindex = ppindex ? *ppindex : pindexDummy; + + if (!AcceptBlockHeader(block, state, &pindex, pxfieldHistory)) + return false; + + // Try to process all requested blocks that we don't have, but only + // process an unrequested block if it's new and has enough work to + // advance our tip, and isn't too many blocks ahead. + bool fAlreadyHave = pindex->nStatus & BLOCK_HAVE_DATA; + bool fHasMoreOrSameWork = (chainActive.Tip() ? pindex->nHeight >= chainActive.Tip()->nHeight : true); + // Blocks that are too out-of-order needlessly limit the effectiveness of + // pruning, because pruning will not delete block files that contain any + // blocks which are too close in height to the tip. Apply this test + // regardless of whether pruning is enabled; it should generally be safe to + // not process unrequested blocks. + bool fTooFarAhead = (pindex->nHeight > int(chainActive.Height() + MIN_BLOCKS_TO_KEEP)); + + // TODO: Decouple this function from the block download logic by removing fRequested + // This requires some new chain data structure to efficiently look up if a + // block is in a chain leading to a candidate for best tip, despite not + // being such a candidate itself. + + // TODO: deal better with return value and error conditions for duplicate + // and unrequested blocks. + if (fAlreadyHave) return true; + if (!fRequested) { // If we didn't ask for it: + if (pindex->nTx != 0) return true; // This is a previously-processed block that was pruned + if (!fHasMoreOrSameWork) return true; // Don't process less-work chains + if (fTooFarAhead) return true; // Block height is too high + } + + if (!CheckBlock(block, state, false, true, pxfieldHistory) || + !ContextualCheckBlock(block, state, pindex->pprev)) { + if (state.IsInvalid() && !state.CorruptionPossible()) { + pindex->nStatus |= BLOCK_FAILED_VALID; + setDirtyBlockIndex.insert(pindex); + } + return error("%s: %s", __func__, FormatStateMessage(state)); + } + + // Header is valid/has work, merkle tree and segwit merkle tree are good...RELAY NOW + // (but if it does not build on our best tip, let the SendMessages loop relay it) + if (!IsInitialBlockDownload() && chainActive.Tip() == pindex->pprev) + GetMainSignals().NewValidBlock(pindex, pblock); + + // Write block to history file + if (fNewBlock) *fNewBlock = true; + try { + CDiskBlockPos blockPos = SaveBlockToDisk(block, pindex->nHeight, dbp); + if (blockPos.IsNull()) { + state.Error(strprintf("%s: Failed to find position to write new block to disk", __func__)); + return false; + } + ReceivedBlockTransactions(block, pindex, blockPos); + } catch (const std::runtime_error& e) { + return AbortNode(state, std::string("System error: ") + e.what()); + } + + FlushStateToDisk(state, FlushStateMode::NONE); + + CheckBlockIndex(); + + return true; +} + + +CBlockIndex * CChainState::InsertBlockIndex(const uint256& hash) +{ + AssertLockHeld(cs_main); + + if (hash.IsNull()) + return nullptr; + + // Return existing + BlockMap::iterator mi = mapBlockIndex.find(hash); + if (mi != mapBlockIndex.end()) + return (*mi).second; + + // Create new + CBlockIndex* pindexNew = new CBlockIndex(); + mi = mapBlockIndex.insert(std::make_pair(hash, pindexNew)).first; + pindexNew->phashBlock = &((*mi).first); + + return pindexNew; +} + +bool CChainState::LoadBlockIndex(CBlockTreeDB& blocktree) +{ + if (!blocktree.LoadBlockIndexGuts([this](const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { return this->InsertBlockIndex(hash); })) + return false; + + std::vector > vSortedByHeight; + vSortedByHeight.reserve(mapBlockIndex.size()); + for (const std::pair& item : mapBlockIndex) + { + CBlockIndex* pindex = item.second; + vSortedByHeight.push_back(std::make_pair(pindex->nHeight, pindex)); + } + sort(vSortedByHeight.begin(), vSortedByHeight.end()); + for (const std::pair& item : vSortedByHeight) + { + CBlockIndex* pindex = item.second; + pindex->nTimeMax = (pindex->pprev ? std::max(pindex->pprev->nTimeMax, pindex->nTime) : pindex->nTime); + // We can link the chain of blocks for which we've received transactions at some point. + // Pruned nodes may have deleted the block. + if (pindex->nTx > 0) { + if (pindex->pprev) { + if (pindex->pprev->nChainTx) { + pindex->nChainTx = pindex->pprev->nChainTx + pindex->nTx; + } else { + pindex->nChainTx = 0; + mapBlocksUnlinked.insert(std::make_pair(pindex->pprev, pindex)); + } + } else { + pindex->nChainTx = pindex->nTx; + } + } + if (!(pindex->nStatus & BLOCK_FAILED_MASK) && pindex->pprev && (pindex->pprev->nStatus & BLOCK_FAILED_MASK)) { + pindex->nStatus |= BLOCK_FAILED_CHILD; + setDirtyBlockIndex.insert(pindex); + } + if (pindex->IsValid(BLOCK_VALID_TRANSACTIONS) && (pindex->nChainTx || pindex->pprev == nullptr)) + setBlockIndexCandidates.insert(pindex); + if (pindex->nStatus & BLOCK_FAILED_MASK && (!pindexBestInvalid || pindex->nHeight > pindexBestInvalid->nHeight)) + pindexBestInvalid = pindex; + if (pindex->pprev) + pindex->BuildSkip(); + if (pindex->IsValid(BLOCK_VALID_TREE) && (pindexBestHeader == nullptr || CBlockIndexWorkComparator()(pindexBestHeader, pindex))) + pindexBestHeader = pindex; + } + + return true; +} + + +/** Apply the effects of a block on the utxo cache, ignoring that it may already have been applied. */ +bool CChainState::RollforwardBlock(const CBlockIndex* pindex, CCoinsViewCache& inputs) +{ + AssertLockHeld(cs_main); + // TODO: merge with ConnectBlock + CBlock block; + if (!ReadBlockFromDisk(block, pindex)) { + return error("ReplayBlock(): ReadBlockFromDisk failed at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); + } + + for (const CTransactionRef& tx : block.vtx) { + if (!tx->IsCoinBase()) { + for (const CTxIn &txin : tx->vin) { + inputs.SpendCoin(txin.prevout); + } + } + // Pass check = true as every addition may be an overwrite. + AddCoins(inputs, *tx, pindex->nHeight, true); + } + return true; +} + +bool CChainState::ReplayBlocks(CCoinsView* view) +{ + LOCK(cs_main); + + CCoinsViewCache cache(view); + + std::vector hashHeads = view->GetHeadBlocks(); + if (hashHeads.empty()) return true; // We're already in a consistent state. + if (hashHeads.size() != 2) return error("ReplayBlocks(): unknown inconsistent state"); + + uiInterface.ShowProgress(_("Replaying blocks..."), 0, false); + LogPrintf("Replaying blocks\n"); + + const CBlockIndex* pindexOld = nullptr; // Old tip during the interrupted flush. + const CBlockIndex* pindexNew; // New tip during the interrupted flush. + const CBlockIndex* pindexFork = nullptr; // Latest block common to both the old and the new tip. + + if (mapBlockIndex.count(hashHeads[0]) == 0) { + return error("ReplayBlocks(): reorganization to unknown block requested"); + } + pindexNew = mapBlockIndex[hashHeads[0]]; + + if (!hashHeads[1].IsNull()) { // The old tip is allowed to be 0, indicating it's the first flush. + if (mapBlockIndex.count(hashHeads[1]) == 0) { + return error("ReplayBlocks(): reorganization from unknown block requested"); + } + pindexOld = mapBlockIndex[hashHeads[1]]; + pindexFork = LastCommonAncestor(pindexOld, pindexNew); + assert(pindexFork != nullptr); + } + + // Rollback along the old branch. + while (pindexOld != pindexFork) { + if (pindexOld->nHeight > 0) { // Never disconnect the genesis block. + CBlock block; + if (!ReadBlockFromDisk(block, pindexOld)) { + return error("RollbackBlock(): ReadBlockFromDisk() failed at %d, hash=%s", pindexOld->nHeight, pindexOld->GetBlockHash().ToString()); + } + LogPrintf("Rolling back %s (%i)\n", pindexOld->GetBlockHash().ToString(), pindexOld->nHeight); + DisconnectResult res = DisconnectBlock(block, pindexOld, cache); + if (res == DISCONNECT_FAILED) { + return error("RollbackBlock(): DisconnectBlock failed at %d, hash=%s", pindexOld->nHeight, pindexOld->GetBlockHash().ToString()); + } + // If DISCONNECT_UNCLEAN is returned, it means a non-existing UTXO was deleted, or an existing UTXO was + // overwritten. It corresponds to cases where the block-to-be-disconnect never had all its operations + // applied to the UTXO set. However, as both writing a UTXO and deleting a UTXO are idempotent operations, + // the result is still a version of the UTXO set with the effects of that block undone. + } + pindexOld = pindexOld->pprev; + } + + // Roll forward from the forking point to the new tip. + int nForkHeight = pindexFork ? pindexFork->nHeight : 0; + for (int nHeight = nForkHeight + 1; nHeight <= pindexNew->nHeight; ++nHeight) { + const CBlockIndex* pindex = pindexNew->GetAncestor(nHeight); + LogPrintf("Rolling forward %s (%i)\n", pindex->GetBlockHash().ToString(), nHeight); + if (!RollforwardBlock(pindex, cache)) return false; + } + + cache.SetBestBlock(pindexNew->GetBlockHash()); + cache.Flush(); + uiInterface.ShowProgress("", 100, false); + return true; +} + + +bool CChainState::RewindBlockIndex() +{ + LOCK(cs_main); + + // Note that during -reindex-chainstate we are called with an empty chainActive! + + int nHeight = 1; + while (nHeight <= chainActive.Height()) { + // Although SCRIPT_VERIFY_WITNESS is now generally enforced on all + // blocks in ConnectBlock, we don't need to go back and + // re-download/re-verify blocks from before segwit actually activated. + nHeight++; + } + + // nHeight is now the height of the first insufficiently-validated block, or tipheight + 1 + CValidationState state; + CBlockIndex* pindex = chainActive.Tip(); + while (chainActive.Height() >= nHeight) { + if (fPruneMode && !(chainActive.Tip()->nStatus & BLOCK_HAVE_DATA)) { + // If pruning, don't try rewinding past the HAVE_DATA point; + // since older blocks can't be served anyway, there's + // no need to walk further, and trying to DisconnectTip() + // will fail (and require a needless reindex/redownload + // of the blockchain). + break; + } + if (!DisconnectTip(state, nullptr)) { + return error("RewindBlockIndex: unable to disconnect block at height %i (%s)", pindex->nHeight, FormatStateMessage(state)); + } + // Occasionally flush state to disk. + if (!FlushStateToDisk(state, FlushStateMode::PERIODIC)) { + LogPrintf("RewindBlockIndex: unable to flush state to disk (%s)\n", FormatStateMessage(state)); + return false; + } + } + + // Reduce validity flag and have-data flags. + // We do this after actual disconnecting, otherwise we'll end up writing the lack of data + // to disk before writing the chainstate, resulting in a failure to continue if interrupted. + for (const auto& entry : mapBlockIndex) { + CBlockIndex* pindexIter = entry.second; + + // Note: If we encounter an insufficiently validated block that + // is on chainActive, it must be because we are a pruning node, and + // this block or some successor doesn't HAVE_DATA, so we were unable to + // rewind all the way. Blocks remaining on chainActive at this point + // must not have their validity reduced. + if (pindexIter->IsValid(BLOCK_VALID_TRANSACTIONS) && pindexIter->nChainTx) { + setBlockIndexCandidates.insert(pindexIter); + } + } + + if (chainActive.Tip() != nullptr) { + // We can't prune block index candidates based on our tip if we have + // no tip due to chainActive being empty! + PruneBlockIndexCandidates(); + + CheckBlockIndex(); + } + + return true; +} + +void CChainState::UnloadBlockIndex() +{ + AssertLockHeld(::cs_main); + nBlockSequenceId = 1; + m_failed_blocks.clear(); + setBlockIndexCandidates.clear(); +} + + +bool CChainState::LoadGenesisBlock() +{ + LOCK(cs_main); + + // Check whether we're already initialized by checking for genesis in + // mapBlockIndex. Note that we can't use chainActive here, since it is + // set based on the coins db, not the block index db, which is the only + // thing loaded at this point. + if (mapBlockIndex.count(FederationParams().GenesisBlock().GetHash())) + return true; + + try { + CBlock &block = const_cast(FederationParams().GenesisBlock()); + CDiskBlockPos blockPos = SaveBlockToDisk(block, 0, nullptr); + if (blockPos.IsNull()) + return error("%s: writing genesis block to disk failed", __func__); + CBlockIndex *pindex = AddToBlockIndex(block); + ReceivedBlockTransactions(block, pindex, blockPos); + } catch (const std::runtime_error& e) { + return error("%s: failed to write genesis block: %s", __func__, e.what()); + } + + //initialize xfield history + CXFieldHistory history(FederationParams().GenesisBlock()); + + return true; +} + + +void CChainState::CheckBlockIndex() +{ + if (!fCheckBlockIndex) { + return; + } + + LOCK(cs_main); + + // During a reindex, we read the genesis block and call CheckBlockIndex before ActivateBestChain, + // so we have the genesis block in mapBlockIndex but no active chain. (A few of the tests when + // iterating the block tree require that chainActive has been initialized.) + if (chainActive.Height() < 0) { + assert(mapBlockIndex.size() <= 1); + return; + } + + // Build forward-pointing map of the entire block tree. + std::multimap forward; + for (auto& entry : mapBlockIndex) { + forward.insert(std::make_pair(entry.second->pprev, entry.second)); + } + + assert(forward.size() == mapBlockIndex.size()); + + std::pair::iterator,std::multimap::iterator> rangeGenesis = forward.equal_range(nullptr); + CBlockIndex *pindex = rangeGenesis.first->second; + rangeGenesis.first++; + assert(rangeGenesis.first == rangeGenesis.second); // There is only one index entry with parent nullptr. + + // Iterate over the entire block tree, using depth-first search. + // Along the way, remember whether there are blocks on the path from genesis + // block being explored which are the first to have certain properties. + size_t nNodes = 0; + int nHeight = 0; + CBlockIndex* pindexFirstInvalid = nullptr; // Oldest ancestor of pindex which is invalid. + CBlockIndex* pindexFirstMissing = nullptr; // Oldest ancestor of pindex which does not have BLOCK_HAVE_DATA. + CBlockIndex* pindexFirstNeverProcessed = nullptr; // Oldest ancestor of pindex for which nTx == 0. + CBlockIndex* pindexFirstNotTreeValid = nullptr; // Oldest ancestor of pindex which does not have BLOCK_VALID_TREE (regardless of being valid or not). + CBlockIndex* pindexFirstNotTransactionsValid = nullptr; // Oldest ancestor of pindex which does not have BLOCK_VALID_TRANSACTIONS (regardless of being valid or not). + CBlockIndex* pindexFirstNotChainValid = nullptr; // Oldest ancestor of pindex which does not have BLOCK_VALID_CHAIN (regardless of being valid or not). + CBlockIndex* pindexFirstNotScriptsValid = nullptr; // Oldest ancestor of pindex which does not have BLOCK_VALID_SCRIPTS (regardless of being valid or not). + while (pindex != nullptr) { + nNodes++; + if (pindexFirstInvalid == nullptr && pindex->nStatus & BLOCK_FAILED_VALID) pindexFirstInvalid = pindex; + if (pindexFirstMissing == nullptr && !(pindex->nStatus & BLOCK_HAVE_DATA)) pindexFirstMissing = pindex; + if (pindexFirstNeverProcessed == nullptr && pindex->nTx == 0) pindexFirstNeverProcessed = pindex; + if (pindex->pprev != nullptr && pindexFirstNotTreeValid == nullptr && (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_TREE) pindexFirstNotTreeValid = pindex; + if (pindex->pprev != nullptr && pindexFirstNotTransactionsValid == nullptr && (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_TRANSACTIONS) pindexFirstNotTransactionsValid = pindex; + if (pindex->pprev != nullptr && pindexFirstNotChainValid == nullptr && (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_CHAIN) pindexFirstNotChainValid = pindex; + if (pindex->pprev != nullptr && pindexFirstNotScriptsValid == nullptr && (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_SCRIPTS) pindexFirstNotScriptsValid = pindex; + + // Begin: actual consistency checks. + if (pindex->pprev == nullptr) { + // Genesis block checks. + assert(pindex->GetBlockHash() == FederationParams().GenesisBlock().GetHash()); // Genesis block's hash must match. + assert(pindex == chainActive.Genesis()); // The current active chain's genesis block must be this block. + } + if (pindex->nChainTx == 0) assert(pindex->nSequenceId <= 0); // nSequenceId can't be set positive for blocks that aren't linked (negative is used for preciousblock) + // VALID_TRANSACTIONS is equivalent to nTx > 0 for all nodes (whether or not pruning has occurred). + // HAVE_DATA is only equivalent to nTx > 0 (or VALID_TRANSACTIONS) if no pruning has occurred. + if (!fHavePruned) { + // If we've never pruned, then HAVE_DATA should be equivalent to nTx > 0 + assert(!(pindex->nStatus & BLOCK_HAVE_DATA) == (pindex->nTx == 0)); + assert(pindexFirstMissing == pindexFirstNeverProcessed); + } else { + // If we have pruned, then we can only say that HAVE_DATA implies nTx > 0 + if (pindex->nStatus & BLOCK_HAVE_DATA) assert(pindex->nTx > 0); + } + if (pindex->nStatus & BLOCK_HAVE_UNDO) assert(pindex->nStatus & BLOCK_HAVE_DATA); + assert(((pindex->nStatus & BLOCK_VALID_MASK) >= BLOCK_VALID_TRANSACTIONS) == (pindex->nTx > 0)); // This is pruning-independent. + // All parents having had data (at some point) is equivalent to all parents being VALID_TRANSACTIONS, which is equivalent to nChainTx being set. + assert((pindexFirstNeverProcessed != nullptr) == (pindex->nChainTx == 0)); // nChainTx != 0 is used to signal that all parent blocks have been processed (but may have been pruned). + assert((pindexFirstNotTransactionsValid != nullptr) == (pindex->nChainTx == 0)); + assert(pindex->nHeight == nHeight); // nHeight must be consistent. + assert(pindex->pprev == nullptr || pindex->nHeight >= pindex->pprev->nHeight); // For every block except the genesis block, the block height must be larger than the parent's. + assert(nHeight < 2 || (pindex->pskip && (pindex->pskip->nHeight < nHeight))); // The pskip pointer must point back for all but the first 2 blocks. + assert(pindexFirstNotTreeValid == nullptr); // All mapBlockIndex entries must at least be TREE valid + if ((pindex->nStatus & BLOCK_VALID_MASK) >= BLOCK_VALID_TREE) assert(pindexFirstNotTreeValid == nullptr); // TREE valid implies all parents are TREE valid + if ((pindex->nStatus & BLOCK_VALID_MASK) >= BLOCK_VALID_CHAIN) assert(pindexFirstNotChainValid == nullptr); // CHAIN valid implies all parents are CHAIN valid + if ((pindex->nStatus & BLOCK_VALID_MASK) >= BLOCK_VALID_SCRIPTS) assert(pindexFirstNotScriptsValid == nullptr); // SCRIPTS valid implies all parents are SCRIPTS valid + if (pindexFirstInvalid == nullptr) { + // Checks for not-invalid blocks. + assert((pindex->nStatus & BLOCK_FAILED_MASK) == 0); // The failed mask cannot be set for blocks without invalid parents. + } + if (!CBlockIndexWorkComparator()(pindex, chainActive.Tip()) && pindexFirstNeverProcessed == nullptr) { + if (pindexFirstInvalid == nullptr) { + // If this block sorts at least as good as the current tip and + // is valid and we have all data for its parents, it must be in + // setBlockIndexCandidates. chainActive.Tip() must also be there + // even if some data has been pruned. + if (pindexFirstMissing == nullptr || pindex == chainActive.Tip()) { + assert(setBlockIndexCandidates.count(pindex)); + } + // If some parent is missing, then it could be that this block was in + // setBlockIndexCandidates but had to be removed because of the missing data. + // In this case it must be in mapBlocksUnlinked -- see test below. + } + } else { // If this block sorts worse than the current tip or some ancestor's block has never been seen, it cannot be in setBlockIndexCandidates. + assert(setBlockIndexCandidates.count(pindex) == 0); + } + // Check whether this block is in mapBlocksUnlinked. + std::pair::iterator,std::multimap::iterator> rangeUnlinked = mapBlocksUnlinked.equal_range(pindex->pprev); + bool foundInUnlinked = false; + while (rangeUnlinked.first != rangeUnlinked.second) { + assert(rangeUnlinked.first->first == pindex->pprev); + if (rangeUnlinked.first->second == pindex) { + foundInUnlinked = true; + break; + } + rangeUnlinked.first++; + } + if (pindex->pprev && (pindex->nStatus & BLOCK_HAVE_DATA) && pindexFirstNeverProcessed != nullptr && pindexFirstInvalid == nullptr) { + // If this block has block data available, some parent was never received, and has no invalid parents, it must be in mapBlocksUnlinked. + assert(foundInUnlinked); + } + if (!(pindex->nStatus & BLOCK_HAVE_DATA)) assert(!foundInUnlinked); // Can't be in mapBlocksUnlinked if we don't HAVE_DATA + if (pindexFirstMissing == nullptr) assert(!foundInUnlinked); // We aren't missing data for any parent -- cannot be in mapBlocksUnlinked. + if (pindex->pprev && (pindex->nStatus & BLOCK_HAVE_DATA) && pindexFirstNeverProcessed == nullptr && pindexFirstMissing != nullptr) { + // We HAVE_DATA for this block, have received data for all parents at some point, but we're currently missing data for some parent. + assert(fHavePruned); // We must have pruned. + // This block may have entered mapBlocksUnlinked if: + // - it has a descendant that at some point had more work than the + // tip, and + // - we tried switching to that descendant but were missing + // data for some intermediate block between chainActive and the + // tip. + // So if this block is itself better than chainActive.Tip() and it wasn't in + // setBlockIndexCandidates, then it must be in mapBlocksUnlinked. + if (!CBlockIndexWorkComparator()(pindex, chainActive.Tip()) && setBlockIndexCandidates.count(pindex) == 0) { + if (pindexFirstInvalid == nullptr) { + assert(foundInUnlinked); + } + } + } + // assert(pindex->GetBlockHash() == pindex->GetBlockHeader().GetHash()); // Perhaps too slow + // End: actual consistency checks. + + // Try descending into the first subnode. + std::pair::iterator,std::multimap::iterator> range = forward.equal_range(pindex); + if (range.first != range.second) { + // A subnode was found. + pindex = range.first->second; + nHeight++; + continue; + } + // This is a leaf node. + // Move upwards until we reach a node of which we have not yet visited the last child. + while (pindex) { + // We are going to either move to a parent or a sibling of pindex. + // If pindex was the first with a certain property, unset the corresponding variable. + if (pindex == pindexFirstInvalid) pindexFirstInvalid = nullptr; + if (pindex == pindexFirstMissing) pindexFirstMissing = nullptr; + if (pindex == pindexFirstNeverProcessed) pindexFirstNeverProcessed = nullptr; + if (pindex == pindexFirstNotTreeValid) pindexFirstNotTreeValid = nullptr; + if (pindex == pindexFirstNotTransactionsValid) pindexFirstNotTransactionsValid = nullptr; + if (pindex == pindexFirstNotChainValid) pindexFirstNotChainValid = nullptr; + if (pindex == pindexFirstNotScriptsValid) pindexFirstNotScriptsValid = nullptr; + // Find our parent. + CBlockIndex* pindexPar = pindex->pprev; + // Find which child we just visited. + std::pair::iterator,std::multimap::iterator> rangePar = forward.equal_range(pindexPar); + while (rangePar.first->second != pindex) { + assert(rangePar.first != rangePar.second); // Our parent must have at least the node we're coming from as child. + rangePar.first++; + } + // Proceed to the next one. + rangePar.first++; + if (rangePar.first != rangePar.second) { + // Move to the sibling. + pindex = rangePar.first->second; + break; + } else { + // Move up further. + pindex = pindexPar; + nHeight--; + continue; + } + } + } + + // Check that we actually traversed the entire map. + assert(nNodes == forward.size()); +} + diff --git a/src/chainstate.h b/src/chainstate.h new file mode 100644 index 0000000000..d35b5791a3 --- /dev/null +++ b/src/chainstate.h @@ -0,0 +1,187 @@ +// Copyright (c) 2024 Chaintope Inc. +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_CHAINSTATE_H +#define BITCOIN_CHAINSTATE_H + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +enum DisconnectResult +{ + DISCONNECT_OK, // All good. + DISCONNECT_UNCLEAN, // Rolled back, but UTXO set was inconsistent with block. + DISCONNECT_FAILED // Something else went wrong. +}; + +struct CBlockIndexWorkComparator +{ + bool operator()(const CBlockIndex *pa, const CBlockIndex *pb) const { + // First sort by most total work, ... + if (pa->nHeight > pb->nHeight) return false; + if (pa->nHeight < pb->nHeight) return true; + + // ... then by earliest time received, ... + if (pa->nSequenceId < pb->nSequenceId) return false; + if (pa->nSequenceId > pb->nSequenceId) return true; + + // Use pointer address as tie breaker (should only happen with blocks + // loaded from disk, as those all have id 0). + if (pa < pb) return false; + if (pa > pb) return true; + + // Identical blocks. + return false; + } +}; + + +struct BlockHasher +{ + size_t operator()(const uint256& hash) const { return hash.GetCheapHash(); } +}; + +typedef std::unordered_map BlockMap; + +/** + * CChainState stores and provides an API to update our local knowledge of the + * current best chain and header tree. + * + * It generally provides access to the current block tree, as well as functions + * to provide new data, which it will appropriately validate and incorporate in + * its state as necessary. + * + * Eventually, the API here is targeted at being exposed externally as a + * consumable libconsensus library, so any functions added must only call + * other class member functions, pure functions in other parts of the consensus + * library, callbacks via the validation interface, or read/write-to-disk + * functions (eventually this will also be via callbacks). + */ +class CChainState { +private: + /** + * The set of all CBlockIndex entries with BLOCK_VALID_TRANSACTIONS (for itself and all ancestors) and + * as good as our current tip or better. Entries may be failed, though, and pruning nodes may be + * missing the data for the block. + */ + std::set setBlockIndexCandidates; + + /** + * Every received block is assigned a unique and increasing identifier, so we + * know which one to give priority in case of a fork. + */ + Mutex cs_nBlockSequenceId; + /** Blocks loaded from disk are assigned id 0, so start the counter at 1. */ + int32_t nBlockSequenceId = 1; + /** Decreasing counter (used by subsequent preciousblock calls). */ + int32_t nBlockReverseSequenceId = -1; + /** block height for the last block that preciousblock has been applied to. */ + int nLastPreciousHeight = 0; + + /** In order to efficiently track invalidity of headers, we keep the set of + * blocks which we tried to connect and found to be invalid here (ie which + * were set to BLOCK_FAILED_VALID since the last restart). We can then + * walk this set and check if a new header is a descendant of something in + * this set, preventing us from having to walk mapBlockIndex when we try + * to connect a bad block and fail. + * + * While this is more complicated than marking everything which descends + * from an invalid block as invalid at the time we discover it to be + * invalid, doing so would require walking all of mapBlockIndex to find all + * descendants. Since this case should be very rare, keeping track of all + * BLOCK_FAILED_VALID blocks in a set should be just fine and work just as + * well. + * + * Because we already walk mapBlockIndex in height-order at startup, we go + * ahead and mark descendants of invalid blocks as FAILED_CHILD at that time, + * instead of putting things in this set. + */ + std::set m_failed_blocks; + + /** + * the ChainState CriticalSection + * A lock that must be held when modifying this ChainState - held in ActivateBestChain() + */ + Mutex m_cs_chainstate; + +public: + CChain chainActive; + BlockMap mapBlockIndex; + std::multimap mapBlocksUnlinked; + CBlockIndex *pindexBestInvalid = nullptr; + + std::unique_ptr< CCheckQueue >scriptcheckqueue; + + bool LoadBlockIndex(CBlockTreeDB& blocktree) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + + bool ActivateBestChain(CValidationState &state, std::shared_ptr pblock); + + /** + * If a block header hasn't already been seen, call CheckBlockHeader on it, ensure + * that it doesn't descend from an invalid block, and then add it to mapBlockIndex. + */ + bool AcceptBlockHeader(const CBlockHeader& block, CValidationState& state, CBlockIndex** ppindex, CXFieldHistoryMap* pxfieldHistory = nullptr) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + bool AcceptBlock(const std::shared_ptr& pblock, CValidationState& state, CBlockIndex** ppindex, bool fRequested, const CDiskBlockPos* dbp, bool* fNewBlock, CXFieldHistoryMap* pxfieldHistory = nullptr) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + + // Block (dis)connection on a given view: + DisconnectResult DisconnectBlock(const CBlock& block, const CBlockIndex* pindex, CCoinsViewCache& view); + bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pindex, + CCoinsViewCache& view, bool fJustCheck = false); + + // Block disconnection on our pcoinsTip: + bool DisconnectTip(CValidationState& state, DisconnectedBlockTransactions *disconnectpool); + + // Manual block validity manipulation: + bool PreciousBlock(CValidationState& state, CBlockIndex* pindex) LOCKS_EXCLUDED(cs_main); + bool InvalidateBlock(CValidationState& state, CBlockIndex* pindex) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + void ResetBlockFailureFlags(CBlockIndex* pindex) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + + bool ReplayBlocks(CCoinsView* view); + bool RewindBlockIndex(); + bool LoadGenesisBlock(); + + void PruneBlockIndexCandidates(); + + void UnloadBlockIndex(); + +private: + bool ActivateBestChainStep(CValidationState& state, CBlockIndex* pindexMostWork, const std::shared_ptr& pblock, bool& fInvalidFound, ConnectTrace& connectTrace); + bool ConnectTip(CValidationState& state, CBlockIndex* pindexNew, const std::shared_ptr& pblock, ConnectTrace& connectTrace, DisconnectedBlockTransactions &disconnectpool); + + CBlockIndex* AddToBlockIndex(const CBlockHeader& block) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + /** Create a new block index entry for a given block hash */ + CBlockIndex* InsertBlockIndex(const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + /** + * Make various assertions about the state of the block index. + * + * By default this only executes fully when using the Regtest chain; see: fCheckBlockIndex. + */ + void CheckBlockIndex(); + + void InvalidBlockFound(CBlockIndex *pindex, const CValidationState &state); + CBlockIndex* FindMostWorkChain() EXCLUSIVE_LOCKS_REQUIRED(cs_main); + void ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pindexNew, const CDiskBlockPos& pos) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + + + bool RollforwardBlock(const CBlockIndex* pindex, CCoinsViewCache& inputs) EXCLUSIVE_LOCKS_REQUIRED(cs_main); +} ; + +extern CChainState g_chainstate; + +void NotifyHeaderTip() LOCKS_EXCLUDED(cs_main); + +bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex *pindex); + +#endif //BITCOIN_CHAINSTATE_H \ No newline at end of file diff --git a/src/connecttrace.h b/src/connecttrace.h new file mode 100644 index 0000000000..43f0c19dbe --- /dev/null +++ b/src/connecttrace.h @@ -0,0 +1,79 @@ +// Copyright (c) 2024 Chaintope Inc. +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_CONNECTTRACE_H +#define BITCOIN_CONNECTTRACE_H + +#include +#include +#include +#include +#include +#include + +struct PerBlockConnectTrace { + CBlockIndex* pindex = nullptr; + std::shared_ptr pblock; + std::shared_ptr> conflictedTxs; + PerBlockConnectTrace() : conflictedTxs(std::make_shared>()) {} +}; + +/** + * Used to track blocks whose transactions were applied to the UTXO state as a + * part of a single ActivateBestChainStep call. + * + * This class also tracks transactions that are removed from the mempool as + * conflicts (per block) and can be used to pass all those transactions + * through SyncTransaction. + * + * This class assumes (and asserts) that the conflicted transactions for a given + * block are added via mempool callbacks prior to the BlockConnected() associated + * with those transactions. If any transactions are marked conflicted, it is + * assumed that an associated block will always be added. + * + * This class is single-use, once you call GetBlocksConnected() you have to throw + * it away and make a new one. + */ +class ConnectTrace { +private: + std::vector blocksConnected; + CTxMemPool &pool; + boost::signals2::scoped_connection m_connNotifyEntryRemoved; + +public: + explicit ConnectTrace(CTxMemPool &_pool) : blocksConnected(1), pool(_pool) { + m_connNotifyEntryRemoved = pool.NotifyEntryRemoved.connect(std::bind(&ConnectTrace::NotifyEntryRemoved, this, std::placeholders::_1, std::placeholders::_2)); + } + + + void BlockConnected(CBlockIndex* pindex, std::shared_ptr pblock) { + assert(!blocksConnected.back().pindex); + assert(pindex); + assert(pblock); + blocksConnected.back().pindex = pindex; + blocksConnected.back().pblock = std::move(pblock); + blocksConnected.emplace_back(); + } + + std::vector& GetBlocksConnected() { + // We always keep one extra block at the end of our list because + // blocks are added after all the conflicted transactions have + // been filled in. Thus, the last entry should always be an empty + // one waiting for the transactions from the next block. We pop + // the last entry here to make sure the list we return is sane. + assert(!blocksConnected.back().pindex); + assert(blocksConnected.back().conflictedTxs->empty()); + blocksConnected.pop_back(); + return blocksConnected; + } + + void NotifyEntryRemoved(CTransactionRef txRemoved, MemPoolRemovalReason reason) { + assert(!blocksConnected.back().pindex); + if (reason == MemPoolRemovalReason::CONFLICT) { + blocksConnected.back().conflictedTxs->emplace_back(std::move(txRemoved)); + } + } +}; + +#endif //BITCOIN_CONNECTTRACE_H \ No newline at end of file diff --git a/src/cs_main.cpp b/src/cs_main.cpp new file mode 100644 index 0000000000..be4cb972b4 --- /dev/null +++ b/src/cs_main.cpp @@ -0,0 +1,10 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Copyright (c) 2024 Chaintope Inc. +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include + +RecursiveMutex cs_main; +RecursiveMutex cs_LastBlockFile; diff --git a/src/cs_main.h b/src/cs_main.h new file mode 100644 index 0000000000..da39c19590 --- /dev/null +++ b/src/cs_main.h @@ -0,0 +1,24 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Copyright (c) 2024 Chaintope Inc. +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_CS_MAIN_H +#define BITCOIN_CS_MAIN_H + +#include + +/** + * Mutex to guard access to validation specific variables, such as reading + * or changing the chainstate. + * + * This may also need to be locked when updating the transaction pool, e.g. on + * AcceptToMemoryPool. See CTxMemPool::cs comment for details. + * + * The transaction pool has a separate lock to allow reading from it and the + * chainstate at the same time. + */ +extern RecursiveMutex cs_main; +extern RecursiveMutex cs_LastBlockFile; + +#endif // BITCOIN_CS_MAIN_H diff --git a/src/file_io.cpp b/src/file_io.cpp new file mode 100644 index 0000000000..129a631ad9 --- /dev/null +++ b/src/file_io.cpp @@ -0,0 +1,619 @@ +// Copyright (c) 2024 Chaintope Inc. +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include +#include +#include +#include +#include + +static const uint64_t MEMPOOL_DUMP_VERSION = 1; + +bool LoadMempool(void) +{ + int64_t nExpiryTimeout = gArgs.GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY) * 60 * 60; + FILE* filestr = fsbridge::fopen(GetDataDir() / "mempool.dat", "rb"); + CAutoFile file(filestr, SER_DISK, CLIENT_VERSION); + if (file.IsNull()) { + LogPrintf("Failed to open mempool file from disk. Continuing anyway.\n"); + return false; + } + + int64_t count = 0; + int64_t expired = 0; + int64_t failed = 0; + int64_t already_there = 0; + int64_t nNow = GetTime(); + + try { + uint64_t version; + file >> version; + if (version != MEMPOOL_DUMP_VERSION) { + return false; + } + uint64_t num; + file >> num; + while (num--) { + CTransactionRef tx; + int64_t nTime; + int64_t nFeeDelta; + file >> tx; + file >> nTime; + file >> nFeeDelta; + + CAmount amountdelta = nFeeDelta; + if (amountdelta) { + mempool.PrioritiseTransaction(tx->GetHashMalFix(), amountdelta); + } + CTxMempoolAcceptanceOptions opt; + if (nTime + nExpiryTimeout > nNow) { + LOCK(cs_main); + opt.nAcceptTime = nTime; + AcceptToMemoryPoolWorker(tx, opt); + if (opt.state.IsValid()) { + ++count; + } else { + // mempool may contain the transaction already, e.g. from + // wallet(s) having loaded it while we were processing + // mempool transactions; consider these as valid, instead of + // failed, but mark them as 'already there' + if (mempool.exists(tx->GetHashMalFix())) { + ++already_there; + } else { + ++failed; + } + } + } else { + ++expired; + } + if (ShutdownRequested()) + return false; + } + std::map mapDeltas; + file >> mapDeltas; + + for (const auto& i : mapDeltas) { + mempool.PrioritiseTransaction(i.first, i.second); + } + } catch (const std::exception& e) { + LogPrintf("Failed to deserialize mempool data on disk: %s. Continuing anyway.\n", e.what()); + return false; + } + + LogPrintf("Imported mempool transactions from disk: %i succeeded, %i failed, %i expired, %i already there\n", count, failed, expired, already_there); + return true; +} + +bool DumpMempool(void) +{ + int64_t start = GetTimeMicros(); + + std::map mapDeltas; + std::vector vinfo; + + { + LOCK(mempool.cs); + for (const auto &i : mempool.mapDeltas) { + mapDeltas[i.first] = i.second; + } + vinfo = mempool.infoAll(); + } + + int64_t mid = GetTimeMicros(); + + try { + FILE* filestr = fsbridge::fopen(GetDataDir() / "mempool.dat.new", "wb"); + if (!filestr) { + return false; + } + + CAutoFile file(filestr, SER_DISK, CLIENT_VERSION); + + uint64_t version = MEMPOOL_DUMP_VERSION; + file << version; + + file << (uint64_t)vinfo.size(); + for (const auto& i : vinfo) { + file << *(i.tx); + file << (int64_t)i.nTime; + file << (int64_t)i.nFeeDelta; + mapDeltas.erase(i.tx->GetHashMalFix()); + } + + file << mapDeltas; + if (!FileCommit(file.Get())) + throw std::runtime_error("FileCommit failed"); + file.fclose(); + RenameOver(GetDataDir() / "mempool.dat.new", GetDataDir() / "mempool.dat"); + int64_t last = GetTimeMicros(); + LogPrintf("Dumped mempool: %gs to copy, %gs to dump\n", (mid-start)*MICRO, (last-mid)*MICRO); + } catch (const std::exception& e) { + LogPrintf("Failed to dump mempool: %s. Continuing anyway.\n", e.what()); + return false; + } + return true; +} + + +static bool FindBlockPos(CDiskBlockPos &pos, unsigned int nAddSize, unsigned int nHeight, uint64_t nTime, bool fKnown = false) +{ + LOCK(cs_LastBlockFile); + + unsigned int nFile = fKnown ? pos.nFile : nLastBlockFile; + if (vinfoBlockFile.size() <= nFile) { + vinfoBlockFile.resize(nFile + 1); + } + + if (!fKnown) { + while (vinfoBlockFile[nFile].nSize + nAddSize >= MAX_BLOCKFILE_SIZE) { + nFile++; + if (vinfoBlockFile.size() <= nFile) { + vinfoBlockFile.resize(nFile + 1); + } + } + pos.nFile = nFile; + pos.nPos = vinfoBlockFile[nFile].nSize; + } + + if ((int)nFile != nLastBlockFile) { + if (!fKnown) { + LogPrintf("Leaving block file %i: %s\n", nLastBlockFile, vinfoBlockFile[nLastBlockFile].ToString()); + } + FlushBlockFile(!fKnown); + nLastBlockFile = nFile; + } + + vinfoBlockFile[nFile].AddBlock(nHeight, nTime); + if (fKnown) + vinfoBlockFile[nFile].nSize = std::max(pos.nPos + nAddSize, vinfoBlockFile[nFile].nSize); + else + vinfoBlockFile[nFile].nSize += nAddSize; + + if (!fKnown) { + unsigned int nOldChunks = (pos.nPos + BLOCKFILE_CHUNK_SIZE - 1) / BLOCKFILE_CHUNK_SIZE; + unsigned int nNewChunks = (vinfoBlockFile[nFile].nSize + BLOCKFILE_CHUNK_SIZE - 1) / BLOCKFILE_CHUNK_SIZE; + if (nNewChunks > nOldChunks) { + if (fPruneMode) + fCheckForPruning = true; + if (CheckDiskSpace(nNewChunks * BLOCKFILE_CHUNK_SIZE - pos.nPos, true)) { + FILE *file = OpenBlockFile(pos); + if (file) { + LogPrintf("Pre-allocating up to position 0x%x in blk%05u.dat\n", nNewChunks * BLOCKFILE_CHUNK_SIZE, pos.nFile); + AllocateFileRange(file, pos.nPos, nNewChunks * BLOCKFILE_CHUNK_SIZE - pos.nPos); + fclose(file); + } + } + else + return error("out of disk space"); + } + } + + setDirtyFileInfo.insert(nFile); + return true; +} + +bool LoadExternalBlockFile(FILE* fileIn, CDiskBlockPos *dbp, CXFieldHistoryMap* pxfieldHistory) +{ + // Map of disk positions for blocks with unknown parent (only used for reindex) + static std::multimap mapBlocksUnknownParent; + int64_t nStart = GetTimeMillis(); + + int nLoaded = 0; + try { + // This takes over fileIn and calls fclose() on it in the CBufferedFile destructor + CBufferedFile blkdat(fileIn, 2*GetCurrentMaxBlockSize(), GetCurrentMaxBlockSize()+8, SER_DISK, CLIENT_VERSION); + uint64_t nRewind = blkdat.GetPos(); + while (!blkdat.eof()) { + blkdat.SetPos(nRewind); + nRewind++; // start one byte further next time, in case of failure + blkdat.SetLimit(); // remove former limit + unsigned int nSize = 0; + try { + // locate a header + unsigned char buf[CMessageHeader::MESSAGE_START_SIZE]; + blkdat.FindByte(FederationParams().MessageStart()[0]); + nRewind = blkdat.GetPos()+1; + blkdat >> buf; + if (memcmp(buf, FederationParams().MessageStart(), CMessageHeader::MESSAGE_START_SIZE)) + continue; + // read size + blkdat >> nSize; + if (nSize < 80 || nSize > GetCurrentMaxBlockSize()) + continue; + } catch (const std::exception&) { + // no valid block header found; don't complain + break; + } + try { + // read block + uint64_t nBlockPos = blkdat.GetPos(); + if (dbp) + dbp->nPos = nBlockPos; + blkdat.SetLimit(nBlockPos + nSize); + blkdat.SetPos(nBlockPos); + std::shared_ptr pblock = std::make_shared(); + CBlock& block = *pblock; + blkdat >> block; + nRewind = blkdat.GetPos(); + + uint256 hash = block.GetHash(); + { + LOCK(cs_main); + // detect out of order blocks, and store them for later + if (hash != FederationParams().GenesisBlock().GetHash() && !LookupBlockIndex(block.hashPrevBlock)) { + LogPrint(BCLog::REINDEX, "%s: Out of order block %s, parent %s not known\n", __func__, hash.ToString(), + block.hashPrevBlock.ToString()); + if (dbp) + mapBlocksUnknownParent.insert(std::make_pair(block.hashPrevBlock, *dbp)); + continue; + } + + // process in case the block isn't known yet + CBlockIndex* pindex = LookupBlockIndex(hash); + if (!pindex || (pindex->nStatus & BLOCK_HAVE_DATA) == 0) { + CValidationState state; + if (g_chainstate.AcceptBlock(pblock, state, nullptr, true, dbp, nullptr, pxfieldHistory)) { + nLoaded++; + } + if (state.IsError()) { + break; + } + } else if (hash != FederationParams().GenesisBlock().GetHash() && pindex->nHeight % 1000 == 0) { + LogPrint(BCLog::REINDEX, "%s Block Import: already had block %s at height %d\n", __func__, hash.ToString(), pindex->nHeight); + } + } + + // Activate the genesis block so normal node progress can continue + if (hash == FederationParams().GenesisBlock().GetHash()) { + CValidationState state; + if (!ActivateBestChain(state)) { + break; + } + } + + NotifyHeaderTip(); + + // Recursively process earlier encountered successors of this block + std::deque queue; + queue.push_back(hash); + while (!queue.empty()) { + uint256 head = queue.front(); + queue.pop_front(); + std::pair::iterator, std::multimap::iterator> range = mapBlocksUnknownParent.equal_range(head); + while (range.first != range.second) { + std::multimap::iterator it = range.first; + std::shared_ptr pblockrecursive = std::make_shared(); + + BlockMap::iterator miSelf = mapBlockIndex.find(hash); + CBlockIndex *pindex = nullptr; + pindex = miSelf->second; + + if (ReadBlockFromDisk(*pblockrecursive, it->second, pindex->nHeight)) + { + LogPrint(BCLog::REINDEX, "%s: Processing out of order child %s of %s\n", __func__, pblockrecursive->GetHash().ToString(), + head.ToString()); + LOCK(cs_main); + CValidationState dummy; + if (g_chainstate.AcceptBlock(pblockrecursive, dummy, nullptr, true, &it->second, nullptr)) + { + nLoaded++; + queue.push_back(pblockrecursive->GetHash()); + } + } + range.first++; + mapBlocksUnknownParent.erase(it); + NotifyHeaderTip(); + } + } + } catch (const std::exception& e) { + LogPrintf("%s: Deserialize or I/O error - %s\n", __func__, e.what()); + } + } + } catch (const std::runtime_error& e) { + AbortNode(std::string("System error: ") + e.what()); + } + if (nLoaded > 0) + LogPrintf("Loaded %i blocks from external file in %dms\n", nLoaded, GetTimeMillis() - nStart); + return nLoaded > 0; +} + +static bool WriteBlockToDisk(const CBlock& block, CDiskBlockPos& pos, const CMessageHeader::MessageStartChars& messageStart) +{ + // Open history file to append + CAutoFile fileout(OpenBlockFile(pos), SER_DISK, CLIENT_VERSION); + if (fileout.IsNull()) + return error("WriteBlockToDisk: OpenBlockFile failed"); + + // Write index header + unsigned int nSize = GetSerializeSize(fileout, block); + fileout << messageStart << nSize; + + // Write block + long fileOutPos = ftell(fileout.Get()); + if (fileOutPos < 0) + return error("WriteBlockToDisk: ftell failed"); + pos.nPos = (unsigned int)fileOutPos; + fileout << block; + + return true; +} + +/** Store block on disk. If dbp is non-nullptr, the file is known to already reside on disk */ +CDiskBlockPos SaveBlockToDisk(const CBlock& block, int nHeight, const CDiskBlockPos* dbp) { + unsigned int nBlockSize = ::GetSerializeSize(block, SER_DISK, CLIENT_VERSION); + CDiskBlockPos blockPos; + if (dbp != nullptr) + blockPos = *dbp; + if (!FindBlockPos(blockPos, nBlockSize+8, nHeight, block.GetBlockTime(), dbp != nullptr)) { + error("%s: FindBlockPos failed", __func__); + return CDiskBlockPos(); + } + if (dbp == nullptr) { + if (!WriteBlockToDisk(block, blockPos, FederationParams().MessageStart())) { + AbortNode("Failed to write block"); + return CDiskBlockPos(); + } + } + return blockPos; +} + +bool FlushStateToDisk(CValidationState &state, FlushStateMode mode, int nManualPruneHeight) { + int64_t nMempoolUsage = mempool.DynamicMemoryUsage(); + LOCK(cs_main); + static int64_t nLastWrite = 0; + static int64_t nLastFlush = 0; + std::set setFilesToPrune; + bool full_flush_completed = false; + + const size_t coins_count = pcoinsTip->GetCacheSize(); + const size_t coins_mem_usage = pcoinsTip->DynamicMemoryUsage(); + + try { + { + bool fFlushForPrune = false; + bool fDoFullFlush = false; + LOCK(cs_LastBlockFile); + if (fPruneMode && (fCheckForPruning || nManualPruneHeight > 0) && !fReindex) { + if (nManualPruneHeight > 0) { + FindFilesToPruneManual(setFilesToPrune, nManualPruneHeight); + } else { + FindFilesToPrune(setFilesToPrune, Params().PruneAfterHeight()); + fCheckForPruning = false; + } + if (!setFilesToPrune.empty()) { + fFlushForPrune = true; + if (!fHavePruned) { + pblocktree->WriteFlag("prunedblockfiles", true); + fHavePruned = true; + } + } + } + int64_t nNow = GetTimeMicros(); + // Avoid writing/flushing immediately after startup. + if (nLastWrite == 0) { + nLastWrite = nNow; + } + if (nLastFlush == 0) { + nLastFlush = nNow; + } + int64_t nMempoolSizeMax = gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; + int64_t cacheSize = pcoinsTip->DynamicMemoryUsage(); + int64_t nTotalSpace = nCoinCacheUsage + std::max(nMempoolSizeMax - nMempoolUsage, 0); + // The cache is large and we're within 10% and 10 MiB of the limit, but we have time now (not in the middle of a block processing). + bool fCacheLarge = mode == FlushStateMode::PERIODIC && cacheSize > std::max((9 * nTotalSpace) / 10, nTotalSpace - MAX_BLOCK_COINSDB_USAGE * 1024 * 1024); + // The cache is over the limit, we have to write now. + bool fCacheCritical = mode == FlushStateMode::IF_NEEDED && cacheSize > nTotalSpace; + // It's been a while since we wrote the block index to disk. Do this frequently, so we don't need to redownload after a crash. + bool fPeriodicWrite = mode == FlushStateMode::PERIODIC && nNow > nLastWrite + (int64_t)DATABASE_WRITE_INTERVAL * 1000000; + // It's been very long since we flushed the cache. Do this infrequently, to optimize cache usage. + bool fPeriodicFlush = mode == FlushStateMode::PERIODIC && nNow > nLastFlush + (int64_t)DATABASE_FLUSH_INTERVAL * 1000000; + // Combine all conditions that result in a full cache flush. + fDoFullFlush = (mode == FlushStateMode::ALWAYS) || fCacheLarge || fCacheCritical || fPeriodicFlush || fFlushForPrune; + // Write blocks and block index to disk. + if (fDoFullFlush || fPeriodicWrite) { + // Depend on nMinDiskSpace to ensure we can write block index + if (!CheckDiskSpace(0, true)) + return state.Error("out of disk space"); + // First make sure all block and undo data is flushed to disk. + FlushBlockFile(); + // Then update all block file information (which may refer to block and undo files). + { + std::vector > vFiles; + vFiles.reserve(setDirtyFileInfo.size()); + for (std::set::iterator it = setDirtyFileInfo.begin(); it != setDirtyFileInfo.end(); ) { + vFiles.push_back(std::make_pair(*it, &vinfoBlockFile[*it])); + setDirtyFileInfo.erase(it++); + } + std::vector vBlocks; + vBlocks.reserve(setDirtyBlockIndex.size()); + for (std::set::iterator it = setDirtyBlockIndex.begin(); it != setDirtyBlockIndex.end(); ) { + vBlocks.push_back(*it); + setDirtyBlockIndex.erase(it++); + } + if (!pblocktree->WriteBatchSync(vFiles, nLastBlockFile, vBlocks)) { + return AbortNode(state, "Failed to write to block index database"); + } + } + // Finally remove any pruned files + if (fFlushForPrune) + UnlinkPrunedFiles(setFilesToPrune); + nLastWrite = nNow; + } + // Flush best chain related state. This can only be done if the blocks / block index write was also done. + if (fDoFullFlush && !pcoinsTip->GetBestBlock().IsNull()) { + // Typical Coin structures on disk are around 48 bytes in size. + // Pushing a new one to the database can cause it to be written + // twice (once in the log, and once in the tables). This is already + // an overestimation, as most will delete an existing entry or + // overwrite one. Still, use a conservative safety factor of 2. + if (!CheckDiskSpace(48 * 2 * 2 * pcoinsTip->GetCacheSize())) + return state.Error("out of disk space"); + // Flush the chainstate (which may refer to block index entries). + if (!pcoinsTip->Flush()) + return AbortNode(state, "Failed to write to coin database"); + nLastFlush = nNow; + full_flush_completed = true; + TRACE5(utxocache, utxocache_flush, + (int64_t)(GetTimeMicros() - nNow), // in microseconds (µs) + (uint32_t)mode, + (uint64_t)coins_count, + (uint64_t)coins_mem_usage, + (bool)fFlushForPrune); + } + } + if (full_flush_completed) { + // Update best block in wallet (so we can detect restored wallets). + GetMainSignals().ChainStateFlushed(chainActive.GetLocator()); + } + } catch (const std::runtime_error& e) { + return AbortNode(state, std::string("System error while flushing: ") + e.what()); + } + return true; +} + +void FlushBlockFile(bool fFinalize) +{ + LOCK(cs_LastBlockFile); + + CDiskBlockPos posOld(nLastBlockFile, 0); + bool status = true; + + FILE *fileOld = OpenBlockFile(posOld); + if (fileOld) { + if (fFinalize) + status &= TruncateFile(fileOld, vinfoBlockFile[nLastBlockFile].nSize); + status &= FileCommit(fileOld); + fclose(fileOld); + } + + fileOld = OpenUndoFile(posOld); + if (fileOld) { + if (fFinalize) + status &= TruncateFile(fileOld, vinfoBlockFile[nLastBlockFile].nUndoSize); + status &= FileCommit(fileOld); + fclose(fileOld); + } + + if (!status) { + AbortNode("Flushing block file to disk failed. This is likely the result of an I/O error."); + } +} + +bool ReadRawBlockFromDisk(std::vector& block, const CDiskBlockPos& pos, const CMessageHeader::MessageStartChars& message_start) +{ + CDiskBlockPos hpos = pos; + hpos.nPos -= 8; // Seek back 8 bytes for meta header + CAutoFile filein(OpenBlockFile(hpos, true), SER_DISK, CLIENT_VERSION); + if (filein.IsNull()) { + return error("%s: OpenBlockFile failed for %s", __func__, pos.ToString()); + } + + try { + CMessageHeader::MessageStartChars blk_start; + unsigned int blk_size; + + filein >> blk_start >> blk_size; + + if (memcmp(blk_start, message_start, CMessageHeader::MESSAGE_START_SIZE)) { + return error("%s: Block magic mismatch for %s: %s versus expected %s", __func__, pos.ToString(), + HexStr(blk_start, blk_start + CMessageHeader::MESSAGE_START_SIZE), + HexStr(message_start, message_start + CMessageHeader::MESSAGE_START_SIZE)); + } + + if (blk_size > MAX_SIZE) { + return error("%s: Block data is larger than maximum deserialization size for %s: %s versus %s", __func__, pos.ToString(), + blk_size, MAX_SIZE); + } + + block.resize(blk_size); // Zeroing of memory is intentional here + filein.read((char*)block.data(), blk_size); + } catch(const std::exception& e) { + return error("%s: Read from block file failed: %s for %s", __func__, e.what(), pos.ToString()); + } + + return true; +} + + +bool ReadBlockFromDisk(CBlock& block, const CDiskBlockPos& pos, int nHeight) +{ + block.SetNull(); + + // Open history file to read + CAutoFile filein(OpenBlockFile(pos, true), SER_DISK, CLIENT_VERSION); + if (filein.IsNull()) + return error("ReadBlockFromDisk: OpenBlockFile failed for %s", pos.ToString()); + + // Read block + try { + filein >> block; + } + catch (const std::exception& e) { + return error("%s: Deserialize or I/O error - %s at %s", __func__, e.what(), pos.ToString()); + } + + CValidationState state; + if(!CheckBlockHeader(block.GetBlockHeader(), state, nullptr, nHeight, true)) + return error("%s: ReadBlockFromDisk: %s nHeight = %d", __func__, FormatStateMessage(state), nHeight); + + return true; +} + +bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex) +{ + CDiskBlockPos blockPos; + uint32_t height = 0; + { + LOCK(cs_main); + blockPos = pindex->GetBlockPos(); + height = pindex->nHeight; + } + + if (!ReadBlockFromDisk(block, blockPos, height)) + return false; + if (block.GetHash() != pindex->GetBlockHash()) + return error("ReadBlockFromDisk(CBlock&, CBlockIndex*): GetHash() doesn't match index for %s at %s", + pindex->ToString(), pindex->GetBlockPos().ToString()); + return true; +} + +bool ReadRawBlockFromDisk(std::vector& block, const CBlockIndex* pindex, const CMessageHeader::MessageStartChars& message_start) +{ + CDiskBlockPos block_pos; + { + LOCK(cs_main); + block_pos = pindex->GetBlockPos(); + } + + return ReadRawBlockFromDisk(block, block_pos, message_start); +} + +FILE* OpenDiskFile(const CDiskBlockPos &pos, const char *prefix, bool fReadOnly) +{ + if (pos.IsNull()) + return nullptr; + fs::path path = GetBlockPosFilename(pos, prefix); + fs::create_directories(path.parent_path()); + FILE* file = fsbridge::fopen(path, fReadOnly ? "rb": "rb+"); + if (!file && !fReadOnly) + file = fsbridge::fopen(path, "wb+"); + if (!file) { + LogPrintf("Unable to open file %s\n", path.string()); + return nullptr; + } + if (pos.nPos) { + if (fseek(file, pos.nPos, SEEK_SET)) { + LogPrintf("Unable to seek to position %u of %s\n", pos.nPos, path.string()); + fclose(file); + return nullptr; + } + } + return file; +} + +/** Open an undo file (rev?????.dat) */ +FILE* OpenUndoFile(const CDiskBlockPos &pos, bool fReadOnly) { + return OpenDiskFile(pos, "rev", fReadOnly); +} \ No newline at end of file diff --git a/src/file_io.h b/src/file_io.h new file mode 100644 index 0000000000..c50e106520 --- /dev/null +++ b/src/file_io.h @@ -0,0 +1,50 @@ +// Copyright (c) 2024 Chaintope Inc. +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_FILE_IO_H +#define BITCOIN_FILE_IO_H + +enum class FlushStateMode { + NONE, + IF_NEEDED, + PERIODIC, + ALWAYS +}; + +/** Load the mempool from disk. */ +bool LoadMempool(); + +/** Dump the mempool to disk. */ +bool DumpMempool(); + +/** Import blocks from an external file */ +bool LoadExternalBlockFile(FILE* fileIn, CDiskBlockPos *dbp = nullptr, CXFieldHistoryMap* pxfieldHistory = nullptr); + +/** + * Update the on-disk chain state. + * The caches and indexes are flushed depending on the mode we're called with + * if they're too large, if it's been a while since the last write, + * or always and in all cases if we're in prune mode and are deleting files. + * + * If FlushStateMode::NONE is used, then FlushStateToDisk(...) won't do anything + * besides checking if we need to prune. + */ +bool FlushStateToDisk(CValidationState &state, FlushStateMode mode, int nManualPruneHeight=0); + +void FlushBlockFile(bool fFinalize = false); + +/** Functions for disk access for blocks */ +bool ReadBlockFromDisk(CBlock& block, const CDiskBlockPos& pos, int height); +bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex); +bool ReadRawBlockFromDisk(std::vector& block, const CDiskBlockPos& pos, const CMessageHeader::MessageStartChars& message_start); +bool ReadRawBlockFromDisk(std::vector& block, const CBlockIndex* pindex, const CMessageHeader::MessageStartChars& message_start); + +FILE* OpenDiskFile(const CDiskBlockPos &pos, const char *prefix, bool fReadOnly); +/** Open an undo file (rev?????.dat) */ +FILE* OpenUndoFile(const CDiskBlockPos &pos, bool fReadOnly = false); + +/** Store block on disk. If dbp is non-nullptr, the file is known to already reside on disk */ +CDiskBlockPos SaveBlockToDisk(const CBlock& block, int nHeight, const CDiskBlockPos* dbp); + +#endif // BITCOIN_FILE_IO_H \ No newline at end of file diff --git a/src/index/base.cpp b/src/index/base.cpp index de2e4d535d..5116999318 100644 --- a/src/index/base.cpp +++ b/src/index/base.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include constexpr char DB_BEST_BLOCK = 'B'; diff --git a/src/init.cpp b/src/init.cpp index 464c2b2005..cc1607171b 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -51,6 +51,9 @@ #include #include #include +#include +#include +#include #ifndef WIN32 #include diff --git a/src/net_processing.cpp b/src/net_processing.cpp index d2c338206f..c6708a62f4 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -32,6 +32,7 @@ #include #include #include +#include #include #include diff --git a/src/policy/packages.cpp b/src/policy/packages.cpp index 9101d1749d..15339b73ff 100644 --- a/src/policy/packages.cpp +++ b/src/policy/packages.cpp @@ -66,7 +66,7 @@ bool CheckPackage(const Package& txns, CValidationState& state) return true; } -bool ArePackageTransactionsAccepted(PackageValidationState& results) +bool ArePackageTransactionsAccepted(const PackageValidationState& results) { for (const auto& r : results) { if(r.second.IsInvalid() || r.second.IsError() || r.second.missingInputs) { diff --git a/src/rest.cpp b/src/rest.cpp index 6c2b3821c0..31fc1aac3f 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 5f287e83ee..862d2c2dae 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -5,6 +5,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include +#include #include #include @@ -14,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -34,6 +36,8 @@ #include #include #include +#include +#include #include #include diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index 74f0314da0..b0c126216d 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -29,6 +29,8 @@ #include #include #include +#include + #ifdef ENABLE_WALLET #include #endif diff --git a/src/scriptcheck.h b/src/scriptcheck.h new file mode 100644 index 0000000000..844f8c53e6 --- /dev/null +++ b/src/scriptcheck.h @@ -0,0 +1,38 @@ +// Copyright (c) 2024 Chaintope Inc. +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_SCRIPT_CHECK_H +#define BITCOIN_SCRIPT_CHECK_H + +/** + * Closure representing one script verification + * Note that this stores references to the spending transaction + */ +class CScriptCheck +{ +private: + CTxOut m_tx_out; + const CTransaction *ptxTo; + unsigned int nIn; + unsigned int nFlags; + bool cacheStore; + ScriptError error; + PrecomputedTransactionData *txdata; + ColorIdentifier colorid; + +public: + CScriptCheck(): ptxTo(nullptr), nIn(0), nFlags(0), cacheStore(false), error(SCRIPT_ERR_UNKNOWN_ERROR), colorid(ColorIdentifier()) {} + CScriptCheck(const CTxOut& outIn, const CTransaction& txToIn, unsigned int nInIn, unsigned int nFlagsIn, bool cacheIn, PrecomputedTransactionData* txdataIn, ColorIdentifier coloridIn = ColorIdentifier()) : + m_tx_out(outIn), ptxTo(&txToIn), nIn(nInIn), nFlags(nFlagsIn), cacheStore(cacheIn), error(SCRIPT_ERR_UNKNOWN_ERROR), txdata(txdataIn), colorid(coloridIn) { } + + bool operator()(); + + ScriptError GetScriptError() const { return error; } + const ColorIdentifier& GetColorIdentifier() const { return colorid; } +}; + +/** Initializes the script-execution cache */ +void InitScriptExecutionCache(); + +#endif // BITCOIN_SCRIPT_CHECK_H \ No newline at end of file diff --git a/src/validation.cpp b/src/validation.cpp index e903217c2f..ed3073b59c 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -1,17 +1,14 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2018 The Bitcoin Core developers -// Copyright (c) 2019-2021 Chaintope Inc. +// Copyright (c) 2019-2024 Chaintope Inc. // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include +#include -#include -#include -#include #include #include -#include #include #include #include @@ -25,201 +22,23 @@ #include #include #include -#include -#include -#include