[DVT-510] Filtered dumpblocks (#46)

* add block filter

* add forging of filtered blocks

* add languages to readme

* update notify

* remove extra space

README.md

@@ -313,12 +313,57 @@ cat poa-core.0.to.100k | grep '"difficulty"' > poa-core.0.to.100k.blocks
 polycli forge --genesis genesis.json --mode json --blocks poa-core.0.to.100k.blocks --count 99999
 ```
 
-```
+```bash
 # To do the same with using proto instead of json:
-go run main.go dumpblocks http://127.0.0.1:8545 0 1000000 -f poa-core.0.to.100k.proto -r=false -m proto
-go run main.go forge --genesis genesis.json --mode proto --blocks poa-core.0.to.100k.proto --count 99999
+polycli dumpblocks http://127.0.0.1:8545 0 1000000 -f poa-core.0.to.100k.proto -r=false -m proto
+polycli forge --genesis genesis.json --mode proto --blocks poa-core.0.to.100k.proto --count 99999
+```
+
+## Forging Filtered Blocks
+
+Sometimes, it can be helpful to only import the blocks and transactions that are
+relevant. This can be done with `dumpblocks` by providing a `--filter` flag.
+
+```bash
+polycli dumpblocks http://127.0.0.1:8545/ 0 100000 \
+  --filename poa-core.0.to.100k.test \
+  --dump-blocks=true \
+  --dump-receipts=true \
+  --filter '{"to":["0xaf93ff8c6070c4880ca5abc4051f309aa19ec385","0x2d68f0161fcd778db31c7080f6c914657f4d240"],"from":["0xcf260ea317555637c55f70e55dba8d5ad8414cb0","0xaf93ff8c6070c4880ca5abc4051f309aa19ec385","0x2d68f0161fcd778db31c7080f6c914657f4d240"]}'
 ```
 
+To load the pruned blocks into Edge, a couple of flags need to be set. This will
+import only the blocks that are listed in the blocks file. This can be 
+non-consecutive blocks. If you receive a `not enough funds to cover gas costs` 
+error, be sure to fund those addresses in in the `genesis.json`.
+
+```bash
+polycli forge \
+  --genesis genesis.json \
+  --mode json \
+  --blocks poa-core.0.to.100k.test.blocks \
+  --receipts poa-core.0.to.100k.test.receipts \
+  --count 2 \
+  --tx-fees=true \
+  --base-block-reward 1000000000000000000 \
+  --read-first-block=true \
+  --rewrite-tx-nonces=true \
+  --verify-blocks=false \
+  --consecutive-blocks=false \
+  --process-blocks=false
+```
+
+Start the server with:
+```bash
+polygon-edge server --data-dir ./forged-data --chain genesis.json --grpc-address :10000 --libp2p :10001 --jsonrpc :10002
+```
+and query it with:
+```bash
+polycli rpc http://localhost:10002 eth_getBlockByNumber 2743 false | jq
+```
+You will notice that block numbers that have been skipped will return `null`. 
+
+
 # Metrics To Dash
 
 Given an openmetrics / prometheus response, create a json file that can

cmd/abi/abi.go

@@ -30,7 +30,7 @@ go run main.go abi --file ../zkevm-node/etherman/smartcontracts/abi/polygonzkevm
 
 go run main.go abi < ../zkevm-node/etherman/smartcontracts/abi/polygonzkevm.abi
 
- go run main.go abi --data 0x3c158267000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000063ed0f8f0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000006eec03843b9aca0082520894d2f852ec7b4e457f6e7ff8b243c49ff5692926ea87038d7ea4c68000808204c58080642dfe2cca094f2419aad1322ec68e3b37974bd9c918e0686b9bbf02b8bd1145622a3dd64202da71549c010494fd1475d3bf232aa9028204a872fd2e531abfd31c000000000000000000000000000000000000 < ../zkevm-node/etherman/smartcontracts/abi/polygonzkevm.abi
+go run main.go abi --data 0x3c158267000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000063ed0f8f0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000006eec03843b9aca0082520894d2f852ec7b4e457f6e7ff8b243c49ff5692926ea87038d7ea4c68000808204c58080642dfe2cca094f2419aad1322ec68e3b37974bd9c918e0686b9bbf02b8bd1145622a3dd64202da71549c010494fd1475d3bf232aa9028204a872fd2e531abfd31c000000000000000000000000000000000000 < ../zkevm-node/etherman/smartcontracts/abi/polygonzkevm.abi
 `,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		// it would be nice to have a generic reader

cmd/dumpblocks/dumpblocks.go

@@ -23,11 +23,13 @@ import (
 	"net/url"
 	"os"
 	"strconv"
+	"strings"
 	"sync"
 	"time"
 
 	ethrpc "github.com/ethereum/go-ethereum/rpc"
 	"github.com/maticnetwork/polygon-cli/proto/gen/pb"
+	"github.com/maticnetwork/polygon-cli/rpctypes"
 	"github.com/maticnetwork/polygon-cli/util"
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
@@ -47,6 +49,12 @@ type (
 		ShouldDumpReceipts bool
 		Filename           string
 		Mode               string
+		FilterStr          string
+		filter             Filter
+	}
+	Filter struct {
+		To   []string `json:"to"`
+		From []string `json:"from"`
 	}
 )
 
@@ -96,6 +104,8 @@ var DumpblocksCmd = &cobra.Command{
 						continue
 					}
 
+					blocks = filterBlocks(blocks)
+
 					if inputDumpblocks.ShouldDumpBlocks {
 						err = writeResponses(blocks, "block")
 						if err != nil {
@@ -170,6 +180,18 @@ var DumpblocksCmd = &cobra.Command{
 			return fmt.Errorf("output format must one of [json, proto]")
 		}
 
+		if err := json.Unmarshal([]byte(inputDumpblocks.FilterStr), &inputDumpblocks.filter); err != nil {
+			return fmt.Errorf("could not unmarshal filter string")
+		}
+
+		// Make sure the filters are all lowercase.
+		for i := 0; i < len(inputDumpblocks.filter.To); i++ {
+			inputDumpblocks.filter.To[i] = strings.ToLower(inputDumpblocks.filter.To[i])
+		}
+		for i := 0; i < len(inputDumpblocks.filter.From); i++ {
+			inputDumpblocks.filter.From[i] = strings.ToLower(inputDumpblocks.filter.From[i])
+		}
+
 		return nil
 	},
 }
@@ -181,6 +203,7 @@ func init() {
 	DumpblocksCmd.PersistentFlags().StringVarP(&inputDumpblocks.Filename, "filename", "f", "", "where to write the output to (default stdout)")
 	DumpblocksCmd.PersistentFlags().StringVarP(&inputDumpblocks.Mode, "mode", "m", "json", "the output format [json, proto]")
 	DumpblocksCmd.PersistentFlags().Uint64VarP(&inputDumpblocks.BatchSize, "batch-size", "b", 150, "the batch size. Realistically, this probably shouldn't be bigger than 999. Most providers seem to cap at 1000.")
+	DumpblocksCmd.PersistentFlags().StringVarP(&inputDumpblocks.FilterStr, "filter", "F", "{}", "filter output based on tx to and from, not setting a filter means all are allowed")
 }
 
 // writeResponses writes the data to either stdout or a file if one is provided.
@@ -274,3 +297,30 @@ func writeProto(out []byte) error {
 
 	return nil
 }
+
+// filterBlocks will filter blocks that having transactions with a matching to or
+// from field. If the to or from is an empty slice, then it will match all.
+func filterBlocks(blocks []*json.RawMessage) []*json.RawMessage {
+	// No filtering is done if there filters are not set.
+	if len(inputDumpblocks.filter.To) == 0 && len(inputDumpblocks.filter.From) == 0 {
+		return blocks
+	}
+
+	filtered := []*json.RawMessage{}
+	for _, msg := range blocks {
+		var block rpctypes.RawBlockResponse
+		if err := json.Unmarshal(*msg, &block); err != nil {
+			log.Error().Bytes("block", *msg).Msg("Unable to unmarshal block")
+			continue
+		}
+
+		for _, tx := range block.Transactions {
+			if (len(inputDumpblocks.filter.To) > 0 && slices.Contains(inputDumpblocks.filter.To, strings.ToLower(string(tx.To)))) ||
+				(len(inputDumpblocks.filter.From) > 0 && slices.Contains(inputDumpblocks.filter.From, strings.ToLower(string(tx.From)))) {
+				filtered = append(filtered, msg)
+			}
+		}
+	}
+
+	return filtered
+}

cmd/forge/forge.go

@@ -51,16 +51,21 @@ import (
 
 type (
 	forgeParams struct {
-		Client          string
-		DataDir         string
-		GenesisFile     string
-		Verifier        string
-		Mode            string
-		Count           uint64
-		BlocksFile      string
-		BaseBlockReward string
-		ReceiptsFile    string
-		IncludeTxFees   bool
+		Client                string
+		DataDir               string
+		GenesisFile           string
+		Verifier              string
+		Mode                  string
+		Count                 uint64
+		BlocksFile            string
+		BaseBlockReward       string
+		ReceiptsFile          string
+		IncludeTxFees         bool
+		ShouldReadFirstBlock  bool
+		ShouldVerifyBlocks    bool
+		ShouldRewriteTxNonces bool
+		HasConsecutiveBlocks  bool
+		ShouldProcessBlocks   bool
 
 		GenesisData []byte
 	}
@@ -144,6 +149,11 @@ func init() {
 	ForgeCmd.PersistentFlags().StringVarP(&inputForge.BaseBlockReward, "base-block-reward", "B", "2_000_000_000_000_000_000", "The amount rewarded for mining blocks")
 	ForgeCmd.PersistentFlags().StringVarP(&inputForge.ReceiptsFile, "receipts", "r", "", "A file of encoded receipts; the format of this file should match the mode")
 	ForgeCmd.PersistentFlags().BoolVarP(&inputForge.IncludeTxFees, "tx-fees", "t", false, "if the transaction fees should be included when computing block rewards")
+	ForgeCmd.PersistentFlags().BoolVarP(&inputForge.ShouldReadFirstBlock, "read-first-block", "R", false, "whether to read the first block, leave false if first block is genesis")
+	ForgeCmd.PersistentFlags().BoolVarP(&inputForge.ShouldVerifyBlocks, "verify-blocks", "V", true, "whether to verify blocks, set false if forging nonconsecutive blocks")
+	ForgeCmd.PersistentFlags().BoolVarP(&inputForge.ShouldRewriteTxNonces, "rewrite-tx-nonces", "", false, "whether to rewrite transaction nonces, set true if forging nonconsecutive blocks")
+	ForgeCmd.PersistentFlags().BoolVarP(&inputForge.HasConsecutiveBlocks, "consecutive-blocks", "", true, "whether the blocks file has consecutive blocks")
+	ForgeCmd.PersistentFlags().BoolVarP(&inputForge.ShouldProcessBlocks, "process-blocks", "p", true, "whether the transactions in blocks should be processed applied to the state")
 
 	if err := cobra.MarkFlagRequired(ForgeCmd.PersistentFlags(), "blocks"); err != nil {
 		log.Error().Err(err).Msg("Unable to mark blocks flag as required")
@@ -159,7 +169,6 @@ type edgeBlockchainHandle struct {
 }
 
 func NewEdgeBlockchain() (*edgeBlockchainHandle, error) {
-
 	var chainConfig edgechain.Chain
 	err := json.Unmarshal(inputForge.GenesisData, &chainConfig)
 	if err != nil {
@@ -247,24 +256,26 @@ func readAllBlocksToChain(bh *edgeBlockchainHandle, blockReader BlockReader, rec
 
 	parentBlock := genesisBlock
 
-	// this should probably be based on a flag, but in our current use case, we're going to assume the 0th block is
-	// a copy of the genesis block, so there's no point inserting it again
-	_, err := blockReader.ReadBlock()
-	if err != nil {
-		return fmt.Errorf("could not read off the genesis block from input: %w", err)
+	// Usually, the first block is the genesis block so it will skip it based on the flag.
+	var i uint64 = 0
+	if !inputForge.ShouldReadFirstBlock {
+		_, err := blockReader.ReadBlock()
+		if err != nil {
+			return fmt.Errorf("could not read off the genesis block from input: %w", err)
+		}
+		i++
 	}
 
 	// in practice, I ran into some issues where the dumps that I created had duplicate blocks, This map is used to
 	// detect and skip any kind of duplicates
 	blockHashSet := make(map[ethcommon.Hash]struct{}, 0)
 
-	// insertion into the chain will fail if blocks are numbered non-sequnetially. This is used to throw an error if we
+	// insertion into the chain will fail if blocks are numbered non-sequentially. This is used to throw an error if we
 	// encounter blocks out of order. In the future, we should have a flag if we want to use original numbering or if
 	// we want to create new numbering
 	var lastNumber uint64 = 0
-	var i uint64
 	var receipt *rpctypes.RawTxReceipt
-	for i = 1; i < blocksToRead; i = i + 1 {
+	for ; i < blocksToRead; i++ {
 		// read a polyblock which is a generic interface that can be marshalled into different formats
 		block, err := blockReader.ReadBlock()
 		if err != nil {
@@ -277,15 +288,25 @@ func readAllBlocksToChain(bh *edgeBlockchainHandle, blockReader BlockReader, rec
 		}
 		blockHashSet[block.Hash()] = struct{}{}
 
-		if block.Number().Uint64()-1 != lastNumber {
+		// There are instances where we can import nonconsecutive blocks, skip this
+		// error on those instances.
+		if inputForge.HasConsecutiveBlocks && block.Number().Uint64()-1 != lastNumber {
 			return fmt.Errorf("encountered non consecutive block numbers on input. Got %s and expected %d", block.Number().String(), lastNumber+1)
 		}
 		lastNumber = block.Number().Uint64()
 
 		// convert the generic rpc block into a block for edge. I suppose we'll need to think about other blockchain
-		// forging at somet poing, but for now edge & supernets seem to be the real use case
+		// forging at some point, but for now edge & supernets seem to be the real use case
 		edgeBlock := PolyBlockToEdge(block)
 
+		// The transactions nonces need to be rewritten or else there will be an error.
+		if inputForge.ShouldRewriteTxNonces {
+			for nonce, tx := range edgeBlock.Transactions {
+				tx.Nonce = uint64(nonce)
+				log.Logger.Debug().Int64("old nonce", int64(tx.Nonce)).Int64("new nonce", int64(nonce)).Str("tx hash", tx.Hash.String()).Msg("Rewrote tx nonce")
+			}
+		}
+
 		// The parent hash value will not make sense, so we'll overwrite this when the value from our local parent block.
 		edgeBlock.Header.ParentHash = parentBlock.Header.ComputeHash().Hash
 
@@ -297,10 +318,15 @@ func readAllBlocksToChain(bh *edgeBlockchainHandle, blockReader BlockReader, rec
 			return err
 		}
 
-		// This will execute the block and apply the transaction to the state
-		txn, err := bh.Executor.ProcessBlock(parentBlock.Header.StateRoot, edgeBlock, blockCreator)
+		var txn *edgestate.Transition
+		if inputForge.ShouldProcessBlocks {
+			// This will execute the block and apply the transaction to the state.
+			txn, err = bh.Executor.ProcessBlock(parentBlock.Header.StateRoot, edgeBlock, blockCreator)
+		} else {
+			txn, err = bh.Executor.BeginTxn(parentBlock.Header.StateRoot, edgeBlock.Header, blockCreator)
+		}
 		if err != nil {
-			return fmt.Errorf("unable to process block %d %s: %w", i, edgeBlock.Hash().String(), err)
+			return fmt.Errorf("unable to process block %d with hash %s at index %d: %w", block.Number().Int64(), edgeBlock.Hash().String(), i, err)
 		}
 
 		if err = bh.Blockchain.GetConsensus().PreCommitState(edgeBlock.Header, txn); err != nil {
@@ -314,10 +340,12 @@ func readAllBlocksToChain(bh *edgeBlockchainHandle, blockReader BlockReader, rec
 		edgeBlock.Header.StateRoot = newRoot
 		edgeBlock.Header.Hash = edgeBlock.Header.ComputeHash().Hash
 
-		// This is an optional step but helpful to catch some mistakes in implementation
-		err = bc.VerifyFinalizedBlock(edgeBlock)
-		if err != nil {
-			return fmt.Errorf("unable to verify finalized block: %w", err)
+		// This is an optional step but helpful to catch some mistakes in implementation.
+		if inputForge.ShouldVerifyBlocks {
+			err = bc.VerifyFinalizedBlock(edgeBlock)
+			if err != nil {
+				return fmt.Errorf("unable to verify finalized block: %w", err)
+			}
 		}
 
 		// This might be worth putting behind a flag at some point, but we need some way to distribute native token

go.mod

@@ -170,7 +170,7 @@ require (
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/tsdb v0.7.1 // indirect
-	github.com/rjeczalik/notify v0.9.1 // indirect
+	github.com/rjeczalik/notify v0.9.3 // indirect
 	github.com/rs/cors v1.8.2 // indirect
 	github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect
 	github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 // indirect

go.sum

@@ -616,6 +616,8 @@ github.com/raulk/go-watchdog v1.3.0 h1:oUmdlHxdkXRJlwfG0O9omj8ukerm8MEQavSiDTEtB
 github.com/raulk/go-watchdog v1.3.0/go.mod h1:fIvOnLbF0b0ZwkB9YU4mOW9Did//4vPZtDqv66NfsMU=
 github.com/rjeczalik/notify v0.9.1 h1:CLCKso/QK1snAlnhNR/CNvNiFU2saUtjV0bx3EwNeCE=
 github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho=
+github.com/rjeczalik/notify v0.9.3 h1:6rJAzHTGKXGj76sbRgDiDcYj/HniypXmSJo1SWakZeY=
+github.com/rjeczalik/notify v0.9.3/go.mod h1:gF3zSOrafR9DQEWSE8TjfI9NkooDxbyT4UgRGKZA0lc=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k=
 github.com/rs/cors v1.8.2 h1:KCooALfAYGs415Cwu5ABvv9n9509fSiG5SQJn/AQo4U=
@@ -911,6 +913,7 @@ golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180926160741-c2ed4eda69e7/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=