From e7ca1181503ffed22f62a8589d3992e70e28e144 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Thu, 21 Nov 2024 11:42:55 -0800 Subject: [PATCH 01/45] add verify evm offchain replay util cmd --- cmd/util/cmd/root.go | 2 + .../cmd/verify-evm-offchain-replay/main.go | 87 +++++++++ .../cmd/verify-evm-offchain-replay/verify.go | 93 ++++++++++ fvm/evm/offchain/utils/collection_test.go | 48 ++--- fvm/evm/offchain/utils/verify.go | 168 ++++++++++++++++++ 5 files changed, 374 insertions(+), 24 deletions(-) create mode 100644 cmd/util/cmd/verify-evm-offchain-replay/main.go create mode 100644 cmd/util/cmd/verify-evm-offchain-replay/verify.go create mode 100644 fvm/evm/offchain/utils/verify.go diff --git a/cmd/util/cmd/root.go b/cmd/util/cmd/root.go index cefd8db691d..146fb2b5af8 100644 --- a/cmd/util/cmd/root.go +++ b/cmd/util/cmd/root.go @@ -41,6 +41,7 @@ import ( "github.com/onflow/flow-go/cmd/util/cmd/snapshot" system_addresses "github.com/onflow/flow-go/cmd/util/cmd/system-addresses" truncate_database "github.com/onflow/flow-go/cmd/util/cmd/truncate-database" + verify_evm_offchain_replay "github.com/onflow/flow-go/cmd/util/cmd/verify-evm-offchain-replay" "github.com/onflow/flow-go/cmd/util/cmd/version" "github.com/onflow/flow-go/module/profiler" ) @@ -126,6 +127,7 @@ func addCommands() { rootCmd.AddCommand(debug_script.Cmd) rootCmd.AddCommand(generate_authorization_fixes.Cmd) rootCmd.AddCommand(evm_state_exporter.Cmd) + rootCmd.AddCommand(verify_evm_offchain_replay.Cmd) } func initConfig() { diff --git a/cmd/util/cmd/verify-evm-offchain-replay/main.go b/cmd/util/cmd/verify-evm-offchain-replay/main.go new file mode 100644 index 00000000000..9f56587306e --- /dev/null +++ b/cmd/util/cmd/verify-evm-offchain-replay/main.go @@ -0,0 +1,87 @@ +package verify + +import ( + "fmt" + "strconv" + "strings" + + "github.com/rs/zerolog/log" + "github.com/spf13/cobra" + + "github.com/onflow/flow-go/model/flow" +) + +var ( + flagDatadir string + flagExecutionDataDir string + flagEVMStateGobDir string + flagChain string + flagFromTo string +) + +// usage example +// +// ./util verify-evm-offchain-replay --chain flow-testnet --from-to 211176671-211177000 +// --datadir /var/flow/data/protocol --execution_data_dir /var/flow/data/execution_data +var Cmd = &cobra.Command{ + Use: "verify-evm-offchain-replay", + Short: "verify evm offchain replay with execution data", + Run: run, +} + +func init() { + Cmd.Flags().StringVar(&flagChain, "chain", "", "Chain name") + _ = Cmd.MarkFlagRequired("chain") + + Cmd.Flags().StringVar(&flagDatadir, "datadir", "/var/flow/data/protocol", + "directory that stores the protocol state") + + Cmd.Flags().StringVar(&flagExecutionDataDir, "execution_data_dir", "/var/flow/data/execution_data", + "directory that stores the execution state") + + Cmd.Flags().StringVar(&flagFromTo, "from_to", "", + "the flow height range to verify blocks, i.e, 1-1000, 1000-2000, 2000-3000, etc.") + + Cmd.Flags().StringVar(&flagEVMStateGobDir, "evm_state_gob_dir", "/var/flow/data/evm_state_gob", + "directory that stores the evm state gob files as checkpoint") +} + +func run(*cobra.Command, []string) { + _ = flow.ChainID(flagChain).Chain() + + from, to, err := parseFromTo(flagFromTo) + if err != nil { + log.Fatal().Err(err).Msg("could not parse from_to") + } + + log.Info().Msgf("verifying range from %d to %d", from, to) + err = Verify(from, to, flow.Testnet, flagDatadir, flagExecutionDataDir) + if err != nil { + log.Fatal().Err(err).Msg("could not verify last k height") + } + log.Info().Msgf("successfully verified range from %d to %d", from, to) + +} + +func parseFromTo(fromTo string) (from, to uint64, err error) { + parts := strings.Split(fromTo, "-") + if len(parts) != 2 { + return 0, 0, fmt.Errorf("invalid format: expected 'from-to', got '%s'", fromTo) + } + + from, err = strconv.ParseUint(strings.TrimSpace(parts[0]), 10, 64) + if err != nil { + return 0, 0, fmt.Errorf("invalid 'from' value: %w", err) + } + + to, err = strconv.ParseUint(strings.TrimSpace(parts[1]), 10, 64) + if err != nil { + return 0, 0, fmt.Errorf("invalid 'to' value: %w", err) + } + + if from > to { + return 0, 0, fmt.Errorf("'from' value (%d) must be less than or equal to 'to' value (%d)", from, to) + } + + return from, to, nil +} diff --git a/cmd/util/cmd/verify-evm-offchain-replay/verify.go b/cmd/util/cmd/verify-evm-offchain-replay/verify.go new file mode 100644 index 00000000000..1a907be669a --- /dev/null +++ b/cmd/util/cmd/verify-evm-offchain-replay/verify.go @@ -0,0 +1,93 @@ +package verify + +import ( + "fmt" + "io" + "os" + "path/filepath" + + "github.com/dgraph-io/badger/v2" + badgerds "github.com/ipfs/go-ds-badger2" + + "github.com/onflow/flow-go/cmd/util/cmd/common" + "github.com/onflow/flow-go/fvm/environment" + "github.com/onflow/flow-go/fvm/evm" + "github.com/onflow/flow-go/fvm/evm/offchain/utils" + "github.com/onflow/flow-go/fvm/evm/testutils" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/blobs" + "github.com/onflow/flow-go/module/executiondatasync/execution_data" + "github.com/onflow/flow-go/storage" +) + +func Verify(from uint64, to uint64, chainID flow.ChainID, dataDir string, executionDataDir string, evmStateGobDir string) error { + db, storages, executionDataStore, dsStore, err := initStorages(chainID, dataDir, executionDataDir) + if err != nil { + return fmt.Errorf("could not initialize storages: %w", err) + } + + defer db.Close() + defer dsStore.Close() + + var store *testutils.TestValueStore + isRoot := isEVMRootHeight(chainID, from) + if isRoot { + store = testutils.GetSimpleValueStore() + as := environment.NewAccountStatus() + rootAddr := evm.StorageAccountAddress(chainID) + err = store.SetValue(rootAddr[:], []byte(flow.AccountStatusKey), as.ToBytes()) + if err != nil { + return err + } + } else { + // TODO: recover from gob + } + + return utils.OffchainReplayBackwardCompatibilityTest( + chainID, + from, + to, + storages.Headers, + storages.Results, + executionDataStore, + store, + ) +} + +func initStorages(chainID flow.ChainID, dataDir string, executionDataDir string) ( + *badger.DB, + *storage.All, + execution_data.ExecutionDataGetter, + io.Closer, + error, +) { + db := common.InitStorage(dataDir) + + storages := common.InitStorages(db) + + datastoreDir := filepath.Join(executionDataDir, "blobstore") + err := os.MkdirAll(datastoreDir, 0700) + if err != nil { + return nil, nil, nil, nil, err + } + dsOpts := &badgerds.DefaultOptions + ds, err := badgerds.NewDatastore(datastoreDir, dsOpts) + if err != nil { + return nil, nil, nil, nil, err + } + + executionDataBlobstore := blobs.NewBlobstore(ds) + executionDataStore := execution_data.NewExecutionDataStore(executionDataBlobstore, execution_data.DefaultSerializer) + + return db, storages, executionDataStore, ds, nil +} + +// EVM Root Height is the first block that has EVM Block Event where the EVM block height is 1 +func isEVMRootHeight(chainID flow.ChainID, flowHeight uint64) bool { + if chainID == flow.Testnet { + return flowHeight == 211176671 + } else if chainID == flow.Mainnet { + return flowHeight == 85981136 + } + return flowHeight == 1 +} diff --git a/fvm/evm/offchain/utils/collection_test.go b/fvm/evm/offchain/utils/collection_test.go index a18ce4a81ac..827bb918601 100644 --- a/fvm/evm/offchain/utils/collection_test.go +++ b/fvm/evm/offchain/utils/collection_test.go @@ -41,7 +41,7 @@ func TestTestnetBackwardCompatibility(t *testing.T) { // > ~/Downloads/events_devnet51_1.jsonl // ... // - // 2) comment the above t.Skip, and update the events file paths and checkpoint dir + // 2) comment the above t.Skip, and update the events file paths and evmStateGob dir // to run the tests BackwardCompatibleSinceEVMGenesisBlock( t, flow.Testnet, []string{ @@ -65,47 +65,47 @@ func TestTestnetBackwardCompatibility(t *testing.T) { // --start 211176670 --end 211176770 --network testnet --host access-001.devnet51.nodes.onflow.org:9000 // // During the replay process, it will generate `values_.gob` and -// `allocators_.gob` checkpoint files for each height. If these checkpoint files exist, +// `allocators_.gob` checkpoint files for each height. If these checkpoint gob files exist, // the corresponding event JSON files will be skipped to optimize replay. func BackwardCompatibleSinceEVMGenesisBlock( t *testing.T, chainID flow.ChainID, eventsFilePaths []string, // ordered EVM events in JSONL format - checkpointDir string, - checkpointEndHeight uint64, // EVM height of an EVM state that a checkpoint was created for + evmStateGob string, + evmStateEndHeight uint64, // EVM height of an EVM state that a evmStateGob file was created for ) { // ensure that event files is not an empty array require.True(t, len(eventsFilePaths) > 0) - log.Info().Msgf("replaying EVM events from %v to %v, with checkpoints in %s, and checkpointEndHeight: %v", + log.Info().Msgf("replaying EVM events from %v to %v, with evmStateGob file in %s, and evmStateEndHeight: %v", eventsFilePaths[0], eventsFilePaths[len(eventsFilePaths)-1], - checkpointDir, checkpointEndHeight) + evmStateGob, evmStateEndHeight) - store, checkpointEndHeightOrZero := initStorageWithCheckpoints(t, chainID, checkpointDir, checkpointEndHeight) + store, evmStateEndHeightOrZero := initStorageWithEVMStateGob(t, chainID, evmStateGob, evmStateEndHeight) // the events to replay - nextHeight := checkpointEndHeightOrZero + 1 + nextHeight := evmStateEndHeightOrZero + 1 // replay each event files for _, eventsFilePath := range eventsFilePaths { log.Info().Msgf("replaying events from %v, nextHeight: %v", eventsFilePath, nextHeight) - checkpointEndHeight := replayEvents(t, chainID, store, eventsFilePath, checkpointDir, nextHeight) - nextHeight = checkpointEndHeight + 1 + evmStateEndHeight := replayEvents(t, chainID, store, eventsFilePath, evmStateGob, nextHeight) + nextHeight = evmStateEndHeight + 1 } log.Info(). Msgf("succhessfully replayed all events and state changes are consistent with onchain state change. nextHeight: %v", nextHeight) } -func initStorageWithCheckpoints(t *testing.T, chainID flow.ChainID, checkpointDir string, checkpointEndHeight uint64) ( +func initStorageWithEVMStateGob(t *testing.T, chainID flow.ChainID, evmStateGob string, evmStateEndHeight uint64) ( *TestValueStore, uint64, ) { rootAddr := evm.StorageAccountAddress(chainID) - // if there is no checkpoint, create a empty store and initialize the account status, + // if there is no evmStateGob file, create a empty store and initialize the account status, // return 0 as the genesis height - if checkpointEndHeight == 0 { + if evmStateEndHeight == 0 { store := GetSimpleValueStore() as := environment.NewAccountStatus() require.NoError(t, store.SetValue(rootAddr[:], []byte(flow.AccountStatusKey), as.ToBytes())) @@ -113,19 +113,19 @@ func initStorageWithCheckpoints(t *testing.T, chainID flow.ChainID, checkpointDi return store, 0 } - valueFileName, allocatorFileName := checkpointFileNamesByEndHeight(checkpointDir, checkpointEndHeight) + valueFileName, allocatorFileName := evmStateGobFileNamesByEndHeight(evmStateGob, evmStateEndHeight) values, err := deserialize(valueFileName) require.NoError(t, err) allocators, err := deserializeAllocator(allocatorFileName) require.NoError(t, err) store := GetSimpleValueStorePopulated(values, allocators) - return store, checkpointEndHeight + return store, evmStateEndHeight } func replayEvents( t *testing.T, chainID flow.ChainID, - store *TestValueStore, eventsFilePath string, checkpointDir string, initialNextHeight uint64) uint64 { + store *TestValueStore, eventsFilePath string, evmStateGob string, initialNextHeight uint64) uint64 { rootAddr := evm.StorageAccountAddress(chainID) @@ -172,22 +172,22 @@ func replayEvents( return nil }) - checkpointEndHeight := nextHeight - 1 + evmStateEndHeight := nextHeight - 1 - log.Info().Msgf("finished replaying events from %v to %v, creating checkpoint", initialNextHeight, checkpointEndHeight) - valuesFile, allocatorsFile := dumpCheckpoint(t, store, checkpointDir, checkpointEndHeight) - log.Info().Msgf("checkpoint created: %v, %v", valuesFile, allocatorsFile) + log.Info().Msgf("finished replaying events from %v to %v, creating evm state gobs", initialNextHeight, evmStateEndHeight) + valuesFile, allocatorsFile := dumpEVMStateToGobFiles(t, store, evmStateGob, evmStateEndHeight) + log.Info().Msgf("evm state gobs created: %v, %v", valuesFile, allocatorsFile) - return checkpointEndHeight + return evmStateEndHeight } -func checkpointFileNamesByEndHeight(dir string, endHeight uint64) (string, string) { +func evmStateGobFileNamesByEndHeight(dir string, endHeight uint64) (string, string) { return filepath.Join(dir, fmt.Sprintf("values_%d.gob", endHeight)), filepath.Join(dir, fmt.Sprintf("allocators_%d.gob", endHeight)) } -func dumpCheckpoint(t *testing.T, store *TestValueStore, dir string, checkpointEndHeight uint64) (string, string) { - valuesFileName, allocatorsFileName := checkpointFileNamesByEndHeight(dir, checkpointEndHeight) +func dumpEVMStateToGobFiles(t *testing.T, store *TestValueStore, dir string, evmStateEndHeight uint64) (string, string) { + valuesFileName, allocatorsFileName := evmStateGobFileNamesByEndHeight(dir, evmStateEndHeight) values, allocators := store.Dump() require.NoError(t, serialize(valuesFileName, values)) diff --git a/fvm/evm/offchain/utils/verify.go b/fvm/evm/offchain/utils/verify.go new file mode 100644 index 00000000000..bf3ed506adf --- /dev/null +++ b/fvm/evm/offchain/utils/verify.go @@ -0,0 +1,168 @@ +package utils + +import ( + "context" + "errors" + "strings" + + "github.com/rs/zerolog/log" + + "github.com/onflow/cadence" + "github.com/onflow/cadence/encoding/ccf" + + "github.com/onflow/flow-go/fvm/environment" + "github.com/onflow/flow-go/fvm/evm" + "github.com/onflow/flow-go/fvm/evm/events" + "github.com/onflow/flow-go/fvm/evm/offchain/blocks" + evmStorage "github.com/onflow/flow-go/fvm/evm/offchain/storage" + "github.com/onflow/flow-go/fvm/evm/offchain/sync" + "github.com/onflow/flow-go/fvm/evm/testutils" + "github.com/onflow/flow-go/ledger" + "github.com/onflow/flow-go/ledger/common/convert" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/executiondatasync/execution_data" + "github.com/onflow/flow-go/storage" +) + +func OffchainReplayBackwardCompatibilityTest( + chainID flow.ChainID, + flowStartHeight uint64, + flowEndHeight uint64, + headers storage.Headers, + results storage.ExecutionResults, + executionDataStore execution_data.ExecutionDataGetter, + store environment.ValueStore, +) error { + rootAddr := evm.StorageAccountAddress(chainID) + rootAddrStr := string(rootAddr.Bytes()) + + bpStorage := evmStorage.NewEphemeralStorage(store) + bp, err := blocks.NewBasicProvider(chainID, bpStorage, rootAddr) + if err != nil { + return err + } + + for height := flowStartHeight; height <= flowEndHeight; height++ { + blockID, err := headers.BlockIDByHeight(height) + if err != nil { + return err + } + + result, err := results.ByBlockID(blockID) + if err != nil { + return err + } + + executionData, err := executionDataStore.Get(context.Background(), result.ExecutionDataID) + if err != nil { + return err + } + + events := flow.EventsList{} + payloads := []*ledger.Payload{} + + for _, chunkData := range executionData.ChunkExecutionDatas { + events = append(events, chunkData.Events...) + payloads = append(payloads, chunkData.TrieUpdate.Payloads...) + } + + updates := make(map[flow.RegisterID]flow.RegisterValue, len(payloads)) + for i := len(payloads) - 1; i >= 0; i-- { + regID, regVal, err := convert.PayloadToRegister(payloads[i]) + if err != nil { + return err + } + + // skip non-evm-account registers + if regID.Owner != rootAddrStr { + continue + } + + // when iterating backwards, duplicated register updates are stale updates, + // so skipping them + if _, ok := updates[regID]; !ok { + updates[regID] = regVal + } + } + + // parse events + evmBlockEvent, evmTxEvents, err := parseEVMEvents(events) + if err != nil { + return err + } + + err = bp.OnBlockReceived(evmBlockEvent) + if err != nil { + return err + } + + sp := testutils.NewTestStorageProvider(store, evmBlockEvent.Height) + cr := sync.NewReplayer(chainID, rootAddr, sp, bp, log.Logger, nil, true) + res, err := cr.ReplayBlock(evmTxEvents, evmBlockEvent) + if err != nil { + return err + } + + // commit all changes + for k, v := range res.StorageRegisterUpdates() { + err = store.SetValue([]byte(k.Owner), []byte(k.Key), v) + if err != nil { + return err + } + } + + err = bp.OnBlockExecuted(evmBlockEvent.Height, res) + if err != nil { + return err + } + + // verify and commit all block hash list changes + for k, v := range bpStorage.StorageRegisterUpdates() { + // verify the block hash list changes are included in the trie update + + err = store.SetValue([]byte(k.Owner), []byte(k.Key), v) + if err != nil { + return err + } + } + } + + return nil +} + +func parseEVMEvents(evts flow.EventsList) (*events.BlockEventPayload, []events.TransactionEventPayload, error) { + var blockEvent *events.BlockEventPayload + txEvents := make([]events.TransactionEventPayload, 0) + + for _, e := range evts { + evtType := string(e.Type) + if strings.Contains(evtType, "BlockExecuted") { + if blockEvent != nil { + return nil, nil, errors.New("multiple block events in a single block") + } + + ev, err := ccf.Decode(nil, e.Payload) + if err != nil { + return nil, nil, err + } + + blockEventPayload, err := events.DecodeBlockEventPayload(ev.(cadence.Event)) + if err != nil { + return nil, nil, err + } + blockEvent = blockEventPayload + } else if strings.Contains(evtType, "TransactionExecuted") { + ev, err := ccf.Decode(nil, e.Payload) + if err != nil { + return nil, nil, err + } + txEv, err := events.DecodeTransactionEventPayload(ev.(cadence.Event)) + if err != nil { + return nil, nil, err + } + txEvents = append(txEvents, *txEv) + } + } + + return blockEvent, txEvents, nil +} From b560792abe1239e44d4354ead3367436b909f707 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Thu, 21 Nov 2024 14:49:32 -0800 Subject: [PATCH 02/45] refactor serailization with gob --- .../cmd/verify-evm-offchain-replay/main.go | 2 +- .../cmd/verify-evm-offchain-replay/verify.go | 38 +++++++- fvm/evm/offchain/utils/collection_test.go | 91 +------------------ fvm/evm/testutils/gob.go | 88 ++++++++++++++++++ 4 files changed, 129 insertions(+), 90 deletions(-) create mode 100644 fvm/evm/testutils/gob.go diff --git a/cmd/util/cmd/verify-evm-offchain-replay/main.go b/cmd/util/cmd/verify-evm-offchain-replay/main.go index 9f56587306e..76581e8a471 100644 --- a/cmd/util/cmd/verify-evm-offchain-replay/main.go +++ b/cmd/util/cmd/verify-evm-offchain-replay/main.go @@ -55,7 +55,7 @@ func run(*cobra.Command, []string) { } log.Info().Msgf("verifying range from %d to %d", from, to) - err = Verify(from, to, flow.Testnet, flagDatadir, flagExecutionDataDir) + err = Verify(from, to, flow.Testnet, flagDatadir, flagExecutionDataDir, flagEVMStateGobDir) if err != nil { log.Fatal().Err(err).Msg("could not verify last k height") } diff --git a/cmd/util/cmd/verify-evm-offchain-replay/verify.go b/cmd/util/cmd/verify-evm-offchain-replay/verify.go index 1a907be669a..bbdd9911c21 100644 --- a/cmd/util/cmd/verify-evm-offchain-replay/verify.go +++ b/cmd/util/cmd/verify-evm-offchain-replay/verify.go @@ -40,10 +40,21 @@ func Verify(from uint64, to uint64, chainID flow.ChainID, dataDir string, execut return err } } else { - // TODO: recover from gob + prev := from - 1 + valueFileName, allocatorFileName := evmStateGobFileNamesByEndHeight(evmStateGobDir, prev) + values, err := testutils.DeserializeState(valueFileName) + if err != nil { + return err + } + + allocators, err := testutils.DeserializeAllocator(allocatorFileName) + if err != nil { + return err + } + store = testutils.GetSimpleValueStorePopulated(values, allocators) } - return utils.OffchainReplayBackwardCompatibilityTest( + err = utils.OffchainReplayBackwardCompatibilityTest( chainID, from, to, @@ -52,6 +63,23 @@ func Verify(from uint64, to uint64, chainID flow.ChainID, dataDir string, execut executionDataStore, store, ) + + if err != nil { + return err + } + + valueFileName, allocatorFileName := evmStateGobFileNamesByEndHeight(evmStateGobDir, to) + values, allocators := store.Dump() + err = testutils.SerializeState(valueFileName, values) + if err != nil { + return err + } + err = testutils.SerializeAllocator(allocatorFileName, allocators) + if err != nil { + return err + } + + return nil } func initStorages(chainID flow.ChainID, dataDir string, executionDataDir string) ( @@ -91,3 +119,9 @@ func isEVMRootHeight(chainID flow.ChainID, flowHeight uint64) bool { } return flowHeight == 1 } + +func evmStateGobFileNamesByEndHeight(evmStateGobDir string, endHeight uint64) (string, string) { + valueFileName := filepath.Join(evmStateGobDir, fmt.Sprintf("values-%d.gob", endHeight)) + allocatorFileName := filepath.Join(evmStateGobDir, fmt.Sprintf("allocators-%d.gob", endHeight)) + return valueFileName, allocatorFileName +} diff --git a/fvm/evm/offchain/utils/collection_test.go b/fvm/evm/offchain/utils/collection_test.go index 827bb918601..e5b3059661b 100644 --- a/fvm/evm/offchain/utils/collection_test.go +++ b/fvm/evm/offchain/utils/collection_test.go @@ -2,7 +2,6 @@ package utils_test import ( "bufio" - "encoding/gob" "encoding/hex" "encoding/json" "fmt" @@ -114,9 +113,9 @@ func initStorageWithEVMStateGob(t *testing.T, chainID flow.ChainID, evmStateGob } valueFileName, allocatorFileName := evmStateGobFileNamesByEndHeight(evmStateGob, evmStateEndHeight) - values, err := deserialize(valueFileName) + values, err := DeserializeState(valueFileName) require.NoError(t, err) - allocators, err := deserializeAllocator(allocatorFileName) + allocators, err := DeserializeAllocator(allocatorFileName) require.NoError(t, err) store := GetSimpleValueStorePopulated(values, allocators) return store, evmStateEndHeight @@ -190,8 +189,8 @@ func dumpEVMStateToGobFiles(t *testing.T, store *TestValueStore, dir string, evm valuesFileName, allocatorsFileName := evmStateGobFileNamesByEndHeight(dir, evmStateEndHeight) values, allocators := store.Dump() - require.NoError(t, serialize(valuesFileName, values)) - require.NoError(t, serializeAllocator(allocatorsFileName, allocators)) + require.NoError(t, SerializeState(valuesFileName, values)) + require.NoError(t, SerializeAllocator(allocatorsFileName, allocators)) return valuesFileName, allocatorsFileName } @@ -244,85 +243,3 @@ func scanEventFilesAndRun( t.Fatal(err) } } - -// Serialize function: saves map data to a file -func serialize(filename string, data map[string][]byte) error { - // Create a file to save data - file, err := os.Create(filename) - if err != nil { - return err - } - defer file.Close() - - // Use gob to encode data - encoder := gob.NewEncoder(file) - err = encoder.Encode(data) - if err != nil { - return err - } - - return nil -} - -// Deserialize function: reads map data from a file -func deserialize(filename string) (map[string][]byte, error) { - // Open the file for reading - file, err := os.Open(filename) - if err != nil { - return nil, err - } - defer file.Close() - - // Prepare the map to store decoded data - var data map[string][]byte - - // Use gob to decode data - decoder := gob.NewDecoder(file) - err = decoder.Decode(&data) - if err != nil { - return nil, err - } - - return data, nil -} - -// Serialize function: saves map data to a file -func serializeAllocator(filename string, data map[string]uint64) error { - // Create a file to save data - file, err := os.Create(filename) - if err != nil { - return err - } - defer file.Close() - - // Use gob to encode data - encoder := gob.NewEncoder(file) - err = encoder.Encode(data) - if err != nil { - return err - } - - return nil -} - -// Deserialize function: reads map data from a file -func deserializeAllocator(filename string) (map[string]uint64, error) { - // Open the file for reading - file, err := os.Open(filename) - if err != nil { - return nil, err - } - defer file.Close() - - // Prepare the map to store decoded data - var data map[string]uint64 - - // Use gob to decode data - decoder := gob.NewDecoder(file) - err = decoder.Decode(&data) - if err != nil { - return nil, err - } - - return data, nil -} diff --git a/fvm/evm/testutils/gob.go b/fvm/evm/testutils/gob.go new file mode 100644 index 00000000000..1c944a1e9e3 --- /dev/null +++ b/fvm/evm/testutils/gob.go @@ -0,0 +1,88 @@ +package testutils + +import ( + "encoding/gob" + "os" +) + +// Serialize function: saves map data to a file +func SerializeState(filename string, data map[string][]byte) error { + // Create a file to save data + file, err := os.Create(filename) + if err != nil { + return err + } + defer file.Close() + + // Use gob to encode data + encoder := gob.NewEncoder(file) + err = encoder.Encode(data) + if err != nil { + return err + } + + return nil +} + +// Deserialize function: reads map data from a file +func DeserializeState(filename string) (map[string][]byte, error) { + // Open the file for reading + file, err := os.Open(filename) + if err != nil { + return nil, err + } + defer file.Close() + + // Prepare the map to store decoded data + var data map[string][]byte + + // Use gob to decode data + decoder := gob.NewDecoder(file) + err = decoder.Decode(&data) + if err != nil { + return nil, err + } + + return data, nil +} + +// Serialize function: saves map data to a file +func SerializeAllocator(filename string, data map[string]uint64) error { + // Create a file to save data + file, err := os.Create(filename) + if err != nil { + return err + } + defer file.Close() + + // Use gob to encode data + encoder := gob.NewEncoder(file) + err = encoder.Encode(data) + if err != nil { + return err + } + + return nil +} + +// Deserialize function: reads map data from a file +func DeserializeAllocator(filename string) (map[string]uint64, error) { + // Open the file for reading + file, err := os.Open(filename) + if err != nil { + return nil, err + } + defer file.Close() + + // Prepare the map to store decoded data + var data map[string]uint64 + + // Use gob to decode data + decoder := gob.NewDecoder(file) + err = decoder.Decode(&data) + if err != nil { + return nil, err + } + + return data, nil +} From 626747690801913f433774995bcc24f78f2a11fd Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Thu, 21 Nov 2024 15:20:56 -0800 Subject: [PATCH 03/45] add logging --- .../cmd/verify-evm-offchain-replay/main.go | 2 +- .../cmd/verify-evm-offchain-replay/verify.go | 23 ++++++++++++++++--- fvm/evm/offchain/utils/verify.go | 8 ++++--- 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/cmd/util/cmd/verify-evm-offchain-replay/main.go b/cmd/util/cmd/verify-evm-offchain-replay/main.go index 76581e8a471..2459a35cd59 100644 --- a/cmd/util/cmd/verify-evm-offchain-replay/main.go +++ b/cmd/util/cmd/verify-evm-offchain-replay/main.go @@ -55,7 +55,7 @@ func run(*cobra.Command, []string) { } log.Info().Msgf("verifying range from %d to %d", from, to) - err = Verify(from, to, flow.Testnet, flagDatadir, flagExecutionDataDir, flagEVMStateGobDir) + err = Verify(log.Logger, from, to, flow.Testnet, flagDatadir, flagExecutionDataDir, flagEVMStateGobDir) if err != nil { log.Fatal().Err(err).Msg("could not verify last k height") } diff --git a/cmd/util/cmd/verify-evm-offchain-replay/verify.go b/cmd/util/cmd/verify-evm-offchain-replay/verify.go index bbdd9911c21..f75cd8278b6 100644 --- a/cmd/util/cmd/verify-evm-offchain-replay/verify.go +++ b/cmd/util/cmd/verify-evm-offchain-replay/verify.go @@ -8,6 +8,7 @@ import ( "github.com/dgraph-io/badger/v2" badgerds "github.com/ipfs/go-ds-badger2" + "github.com/rs/zerolog" "github.com/onflow/flow-go/cmd/util/cmd/common" "github.com/onflow/flow-go/fvm/environment" @@ -20,7 +21,16 @@ import ( "github.com/onflow/flow-go/storage" ) -func Verify(from uint64, to uint64, chainID flow.ChainID, dataDir string, executionDataDir string, evmStateGobDir string) error { +// Verify verifies the offchain replay of EVM blocks from the given height range +// and updates the EVM state gob files with the latest state +func Verify(log zerolog.Logger, from uint64, to uint64, chainID flow.ChainID, dataDir string, executionDataDir string, evmStateGobDir string) error { + log.Info(). + Str("chain", chainID.String()). + Str("dataDir", dataDir). + Str("executionDataDir", executionDataDir). + Str("evmStateGobDir", evmStateGobDir). + Msgf("verifying range from %d to %d", from, to) + db, storages, executionDataStore, dsStore, err := initStorages(chainID, dataDir, executionDataDir) if err != nil { return fmt.Errorf("could not initialize storages: %w", err) @@ -32,6 +42,8 @@ func Verify(from uint64, to uint64, chainID flow.ChainID, dataDir string, execut var store *testutils.TestValueStore isRoot := isEVMRootHeight(chainID, from) if isRoot { + log.Info().Msgf("initializing EVM state for root height %d", from) + store = testutils.GetSimpleValueStore() as := environment.NewAccountStatus() rootAddr := evm.StorageAccountAddress(chainID) @@ -41,20 +53,23 @@ func Verify(from uint64, to uint64, chainID flow.ChainID, dataDir string, execut } } else { prev := from - 1 + log.Info().Msgf("loading EVM state from previous height %d", prev) + valueFileName, allocatorFileName := evmStateGobFileNamesByEndHeight(evmStateGobDir, prev) values, err := testutils.DeserializeState(valueFileName) if err != nil { - return err + return fmt.Errorf("could not deserialize state %v: %w", valueFileName, err) } allocators, err := testutils.DeserializeAllocator(allocatorFileName) if err != nil { - return err + return fmt.Errorf("could not deserialize allocator %v: %w", allocatorFileName, err) } store = testutils.GetSimpleValueStorePopulated(values, allocators) } err = utils.OffchainReplayBackwardCompatibilityTest( + log, chainID, from, to, @@ -79,6 +94,8 @@ func Verify(from uint64, to uint64, chainID flow.ChainID, dataDir string, execut return err } + log.Info().Msgf("saved EVM state to %s and %s", valueFileName, allocatorFileName) + return nil } diff --git a/fvm/evm/offchain/utils/verify.go b/fvm/evm/offchain/utils/verify.go index bf3ed506adf..ae99e827acb 100644 --- a/fvm/evm/offchain/utils/verify.go +++ b/fvm/evm/offchain/utils/verify.go @@ -5,10 +5,9 @@ import ( "errors" "strings" - "github.com/rs/zerolog/log" - "github.com/onflow/cadence" "github.com/onflow/cadence/encoding/ccf" + "github.com/rs/zerolog" "github.com/onflow/flow-go/fvm/environment" "github.com/onflow/flow-go/fvm/evm" @@ -25,6 +24,7 @@ import ( ) func OffchainReplayBackwardCompatibilityTest( + log zerolog.Logger, chainID flow.ChainID, flowStartHeight uint64, flowEndHeight uint64, @@ -97,7 +97,7 @@ func OffchainReplayBackwardCompatibilityTest( } sp := testutils.NewTestStorageProvider(store, evmBlockEvent.Height) - cr := sync.NewReplayer(chainID, rootAddr, sp, bp, log.Logger, nil, true) + cr := sync.NewReplayer(chainID, rootAddr, sp, bp, log, nil, true) res, err := cr.ReplayBlock(evmTxEvents, evmBlockEvent) if err != nil { return err @@ -125,6 +125,8 @@ func OffchainReplayBackwardCompatibilityTest( return err } } + + log.Info().Msgf("verified block %d", height) } return nil From fd24bb89877939fc90ade8bcec2881c4a5705f28 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Fri, 22 Nov 2024 06:28:14 -0800 Subject: [PATCH 04/45] update error message --- cmd/util/cmd/verify-evm-offchain-replay/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/util/cmd/verify-evm-offchain-replay/main.go b/cmd/util/cmd/verify-evm-offchain-replay/main.go index 2459a35cd59..0bc6eef8187 100644 --- a/cmd/util/cmd/verify-evm-offchain-replay/main.go +++ b/cmd/util/cmd/verify-evm-offchain-replay/main.go @@ -57,7 +57,7 @@ func run(*cobra.Command, []string) { log.Info().Msgf("verifying range from %d to %d", from, to) err = Verify(log.Logger, from, to, flow.Testnet, flagDatadir, flagExecutionDataDir, flagEVMStateGobDir) if err != nil { - log.Fatal().Err(err).Msg("could not verify last k height") + log.Fatal().Err(err).Msg("could not verify height") } log.Info().Msgf("successfully verified range from %d to %d", from, to) From c2560bc05c0f03395607dee11576d6da9850f5b6 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Fri, 22 Nov 2024 08:44:36 -0800 Subject: [PATCH 05/45] add register checks --- fvm/evm/offchain/utils/verify.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/fvm/evm/offchain/utils/verify.go b/fvm/evm/offchain/utils/verify.go index ae99e827acb..2045de36f22 100644 --- a/fvm/evm/offchain/utils/verify.go +++ b/fvm/evm/offchain/utils/verify.go @@ -1,8 +1,10 @@ package utils import ( + "bytes" "context" "errors" + "fmt" "strings" "github.com/onflow/cadence" @@ -124,6 +126,21 @@ func OffchainReplayBackwardCompatibilityTest( if err != nil { return err } + + expectedUpdate, ok := updates[k] + if !ok { + return fmt.Errorf("missing update for register %v, %v", k, expectedUpdate) + } + + if !bytes.Equal(expectedUpdate, v) { + return fmt.Errorf("unexpected update for register %v, expected %v, got %v", k, expectedUpdate, v) + } + + delete(updates, k) + } + + if len(updates) > 0 { + return fmt.Errorf("missing updates for registers %v", updates) } log.Info().Msgf("verified block %d", height) From 52f7f6c02b043c4f1dbe3f40cc1ec4bef45d7332 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Fri, 22 Nov 2024 10:30:11 -0800 Subject: [PATCH 06/45] store block proposal in replay --- fvm/evm/offchain/blocks/provider.go | 50 +++++++++++++++++++++-- fvm/evm/offchain/sync/replay.go | 37 +++++++++-------- fvm/evm/offchain/sync/replayer.go | 18 ++++---- fvm/evm/offchain/sync/replayer_test.go | 4 +- fvm/evm/offchain/utils/collection_test.go | 2 +- fvm/evm/offchain/utils/verify.go | 39 +++++++++++++++++- 6 files changed, 119 insertions(+), 31 deletions(-) diff --git a/fvm/evm/offchain/blocks/provider.go b/fvm/evm/offchain/blocks/provider.go index 9111be4ac64..b9da39bd468 100644 --- a/fvm/evm/offchain/blocks/provider.go +++ b/fvm/evm/offchain/blocks/provider.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/onflow/flow-go/fvm/evm/events" + "github.com/onflow/flow-go/fvm/evm/handler" "github.com/onflow/flow-go/fvm/evm/types" "github.com/onflow/flow-go/model/flow" ) @@ -13,7 +14,10 @@ import ( // a OnBlockReceived call before block execution and // a follow up OnBlockExecuted call after block execution. type BasicProvider struct { + chainID flow.ChainID blks *Blocks + rootAddr flow.Address + storage types.BackendStorage latestBlockPayload *events.BlockEventPayload } @@ -28,7 +32,12 @@ func NewBasicProvider( if err != nil { return nil, err } - return &BasicProvider{blks: blks}, nil + return &BasicProvider{ + chainID: chainID, + blks: blks, + rootAddr: rootAddr, + storage: storage, + }, nil } // GetSnapshotAt returns a block snapshot at the given height @@ -61,14 +70,49 @@ func (p *BasicProvider) OnBlockReceived(blockEvent *events.BlockEventPayload) er // OnBlockExecuted should be called after executing blocks. func (p *BasicProvider) OnBlockExecuted( height uint64, - resCol types.ReplayResultCollector) error { + resCol types.ReplayResultCollector, + blockProposal *types.BlockProposal, +) error { // we push the block hash after execution, so the behaviour of the blockhash is // identical to the evm.handler. if p.latestBlockPayload.Height != height { return fmt.Errorf("active block height doesn't match expected: %d, got: %d", p.latestBlockPayload.Height, height) } + + blockBytes, err := blockProposal.Block.ToBytes() + if err != nil { + return types.NewFatalError(err) + } + + // do the same as handler.CommitBlockProposal + err = p.storage.SetValue( + p.rootAddr[:], + []byte(handler.BlockStoreLatestBlockKey), + blockBytes, + ) + if err != nil { + return err + } + + blockProposalBytes, err := blockProposal.ToBytes() + if err != nil { + return types.NewFatalError(err) + } + + hash := p.latestBlockPayload.Hash + // update block proposal + err = p.storage.SetValue( + p.rootAddr[:], + []byte(handler.BlockStoreLatestBlockProposalKey), + blockProposalBytes, + ) + if err != nil { + return err + } + + // update block hash list return p.blks.PushBlockHash( p.latestBlockPayload.Height, - p.latestBlockPayload.Hash, + hash, ) } diff --git a/fvm/evm/offchain/sync/replay.go b/fvm/evm/offchain/sync/replay.go index 4516f37007d..e85fc21658c 100644 --- a/fvm/evm/offchain/sync/replay.go +++ b/fvm/evm/offchain/sync/replay.go @@ -30,25 +30,26 @@ func ReplayBlockExecution( transactionEvents []events.TransactionEventPayload, blockEvent *events.BlockEventPayload, validateResults bool, -) error { +) ([]*types.Result, error) { // check the passed block event if blockEvent == nil { - return fmt.Errorf("nil block event has been passed") + return nil, fmt.Errorf("nil block event has been passed") } // create a base block context for all transactions // tx related context values will be replaced during execution ctx, err := blockSnapshot.BlockContext() if err != nil { - return err + return nil, err } // update the tracer ctx.Tracer = tracer gasConsumedSoFar := uint64(0) txHashes := make(types.TransactionHashes, len(transactionEvents)) + results := make([]*types.Result, 0, len(transactionEvents)) for idx, tx := range transactionEvents { - err = replayTransactionExecution( + result, err := replayTransactionExecution( rootAddr, ctx, uint(idx), @@ -58,28 +59,30 @@ func ReplayBlockExecution( validateResults, ) if err != nil { - return fmt.Errorf("transaction execution failed, txIndex: %d, err: %w", idx, err) + return nil, fmt.Errorf("transaction execution failed, txIndex: %d, err: %w", idx, err) } gasConsumedSoFar += tx.GasConsumed txHashes[idx] = tx.Hash + + results = append(results, result) } if validateResults { // check transaction inclusion txHashRoot := gethTypes.DeriveSha(txHashes, gethTrie.NewStackTrie(nil)) if txHashRoot != blockEvent.TransactionHashRoot { - return fmt.Errorf("transaction root hash doesn't match [%x] != [%x]", txHashRoot, blockEvent.TransactionHashRoot) + return nil, fmt.Errorf("transaction root hash doesn't match [%x] != [%x]", txHashRoot, blockEvent.TransactionHashRoot) } // check total gas used if blockEvent.TotalGasUsed != gasConsumedSoFar { - return fmt.Errorf("total gas used doesn't match [%d] != [%d]", gasConsumedSoFar, blockEvent.TotalGasUsed) + return nil, fmt.Errorf("total gas used doesn't match [%d] != [%d]", gasConsumedSoFar, blockEvent.TotalGasUsed) } // no need to check the receipt root hash given we have checked the logs and other // values during tx execution. } - return nil + return results, nil } func replayTransactionExecution( @@ -90,7 +93,7 @@ func replayTransactionExecution( ledger atree.Ledger, txEvent *events.TransactionEventPayload, validate bool, -) error { +) (*types.Result, error) { // create emulator em := emulator.NewEmulator(ledger, rootAddr) @@ -102,7 +105,7 @@ func replayTransactionExecution( if len(txEvent.PrecompiledCalls) > 0 { pcs, err := types.AggregatedPrecompileCallsFromEncoded(txEvent.PrecompiledCalls) if err != nil { - return fmt.Errorf("error decoding precompiled calls [%x]: %w", txEvent.Payload, err) + return nil, fmt.Errorf("error decoding precompiled calls [%x]: %w", txEvent.Payload, err) } ctx.ExtraPrecompiledContracts = precompiles.AggregatedPrecompiledCallsToPrecompiledContracts(pcs) } @@ -110,7 +113,7 @@ func replayTransactionExecution( // create a new block view bv, err := em.NewBlockView(ctx) if err != nil { - return err + return nil, err } var res *types.Result @@ -119,31 +122,31 @@ func replayTransactionExecution( if txEvent.TransactionType == types.DirectCallTxType { call, err := types.DirectCallFromEncoded(txEvent.Payload) if err != nil { - return fmt.Errorf("failed to RLP-decode direct call [%x]: %w", txEvent.Payload, err) + return nil, fmt.Errorf("failed to RLP-decode direct call [%x]: %w", txEvent.Payload, err) } res, err = bv.DirectCall(call) if err != nil { - return fmt.Errorf("failed to execute direct call [%x]: %w", txEvent.Hash, err) + return nil, fmt.Errorf("failed to execute direct call [%x]: %w", txEvent.Hash, err) } } else { gethTx := &gethTypes.Transaction{} if err := gethTx.UnmarshalBinary(txEvent.Payload); err != nil { - return fmt.Errorf("failed to RLP-decode transaction [%x]: %w", txEvent.Payload, err) + return nil, fmt.Errorf("failed to RLP-decode transaction [%x]: %w", txEvent.Payload, err) } res, err = bv.RunTransaction(gethTx) if err != nil { - return fmt.Errorf("failed to run transaction [%x]: %w", txEvent.Hash, err) + return nil, fmt.Errorf("failed to run transaction [%x]: %w", txEvent.Hash, err) } } // validate results if validate { if err := ValidateResult(res, txEvent); err != nil { - return fmt.Errorf("transaction replay failed (txHash %x): %w", txEvent.Hash, err) + return nil, fmt.Errorf("transaction replay failed (txHash %x): %w", txEvent.Hash, err) } } - return nil + return res, nil } func ValidateResult( diff --git a/fvm/evm/offchain/sync/replayer.go b/fvm/evm/offchain/sync/replayer.go index 25ccdc10cbf..33411b7c133 100644 --- a/fvm/evm/offchain/sync/replayer.go +++ b/fvm/evm/offchain/sync/replayer.go @@ -46,7 +46,11 @@ func NewReplayer( // ReplayBlock replays the execution of the transactions of an EVM block // using the provided transactionEvents and blockEvents, -// which include all the context data for re-executing the transactions, and returns the replay result. +// which include all the context data for re-executing the transactions, and returns +// the replay result and the result of each transaction. +// the replay result contains the register updates, and the result of each transaction +// contains the execution result of each transaction, which is useful for recontstructing +// the EVM block proposal. // this method can be called concurrently if underlying storage // tracer and block snapshot provider support concurrency. // @@ -56,11 +60,11 @@ func NewReplayer( func (cr *Replayer) ReplayBlock( transactionEvents []events.TransactionEventPayload, blockEvent *events.BlockEventPayload, -) (types.ReplayResultCollector, error) { +) (types.ReplayResultCollector, []*types.Result, error) { // prepare storage st, err := cr.storageProvider.GetSnapshotAt(blockEvent.Height) if err != nil { - return nil, err + return nil, nil, err } // create storage @@ -69,11 +73,11 @@ func (cr *Replayer) ReplayBlock( // get block snapshot bs, err := cr.blockProvider.GetSnapshotAt(blockEvent.Height) if err != nil { - return nil, err + return nil, nil, err } // replay transactions - err = ReplayBlockExecution( + results, err := ReplayBlockExecution( cr.chainID, cr.rootAddr, state, @@ -84,8 +88,8 @@ func (cr *Replayer) ReplayBlock( cr.validateResults, ) if err != nil { - return nil, err + return nil, nil, err } - return state, nil + return state, results, nil } diff --git a/fvm/evm/offchain/sync/replayer_test.go b/fvm/evm/offchain/sync/replayer_test.go index f7c05ab63b5..d193163283b 100644 --- a/fvm/evm/offchain/sync/replayer_test.go +++ b/fvm/evm/offchain/sync/replayer_test.go @@ -162,9 +162,11 @@ func TestChainReplay(t *testing.T) { sp := NewTestStorageProvider(snapshot, 1) cr := sync.NewReplayer(chainID, rootAddr, sp, bp, zerolog.Logger{}, nil, true) - res, err := cr.ReplayBlock(txEventPayloads, blockEventPayload) + res, results, err := cr.ReplayBlock(txEventPayloads, blockEventPayload) require.NoError(t, err) + require.Len(t, results, totalTxCount) + err = bp.OnBlockExecuted(blockEventPayload.Height, res) require.NoError(t, err) diff --git a/fvm/evm/offchain/utils/collection_test.go b/fvm/evm/offchain/utils/collection_test.go index e5b3059661b..ae8b10a0e59 100644 --- a/fvm/evm/offchain/utils/collection_test.go +++ b/fvm/evm/offchain/utils/collection_test.go @@ -147,7 +147,7 @@ func replayEvents( sp := NewTestStorageProvider(store, blockEventPayload.Height) cr := sync.NewReplayer(chainID, rootAddr, sp, bp, zerolog.Logger{}, nil, true) - res, err := cr.ReplayBlock(txEvents, blockEventPayload) + res, _, err := cr.ReplayBlock(txEvents, blockEventPayload) require.NoError(t, err) // commit all changes diff --git a/fvm/evm/offchain/utils/verify.go b/fvm/evm/offchain/utils/verify.go index 2045de36f22..a3f3e871f13 100644 --- a/fvm/evm/offchain/utils/verify.go +++ b/fvm/evm/offchain/utils/verify.go @@ -18,6 +18,7 @@ import ( evmStorage "github.com/onflow/flow-go/fvm/evm/offchain/storage" "github.com/onflow/flow-go/fvm/evm/offchain/sync" "github.com/onflow/flow-go/fvm/evm/testutils" + "github.com/onflow/flow-go/fvm/evm/types" "github.com/onflow/flow-go/ledger" "github.com/onflow/flow-go/ledger/common/convert" "github.com/onflow/flow-go/model/flow" @@ -100,7 +101,7 @@ func OffchainReplayBackwardCompatibilityTest( sp := testutils.NewTestStorageProvider(store, evmBlockEvent.Height) cr := sync.NewReplayer(chainID, rootAddr, sp, bp, log, nil, true) - res, err := cr.ReplayBlock(evmTxEvents, evmBlockEvent) + res, results, err := cr.ReplayBlock(evmTxEvents, evmBlockEvent) if err != nil { return err } @@ -113,7 +114,9 @@ func OffchainReplayBackwardCompatibilityTest( } } - err = bp.OnBlockExecuted(evmBlockEvent.Height, res) + blockProposal := reconstructProposal(evmBlockEvent, evmTxEvents, results) + + err = bp.OnBlockExecuted(evmBlockEvent.Height, res, blockProposal) if err != nil { return err } @@ -185,3 +188,35 @@ func parseEVMEvents(evts flow.EventsList) (*events.BlockEventPayload, []events.T return blockEvent, txEvents, nil } + +func reconstructProposal( + blockEvent *events.BlockEventPayload, + txEvents []events.TransactionEventPayload, + results []*types.Result, +) *types.BlockProposal { + receipts := make([]types.LightReceipt, 0, len(results)) + + for _, result := range results { + receipts = append(receipts, *result.LightReceipt()) + } + + txHashes := make(types.TransactionHashes, 0, len(txEvents)) + for _, tx := range txEvents { + txHashes = append(txHashes, tx.Hash) + } + + return &types.BlockProposal{ + Block: types.Block{ + ParentBlockHash: blockEvent.ParentBlockHash, + Height: blockEvent.Height, + Timestamp: blockEvent.Timestamp, + TotalSupply: blockEvent.TotalSupply.Big(), + ReceiptRoot: blockEvent.ReceiptRoot, + TransactionHashRoot: blockEvent.TransactionHashRoot, + TotalGasUsed: blockEvent.TotalGasUsed, + PrevRandao: blockEvent.PrevRandao, + }, + Receipts: receipts, + TxHashes: txHashes, + } +} From b96010e15f1e7d7d0664861214611ed516c03600 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Fri, 22 Nov 2024 11:18:53 -0800 Subject: [PATCH 07/45] fix tests --- fvm/evm/offchain/blocks/block_proposal.go | 38 +++++++++++++++++++++++ fvm/evm/offchain/sync/replayer_test.go | 14 ++++----- fvm/evm/offchain/utils/collection_test.go | 6 ++-- fvm/evm/offchain/utils/verify.go | 35 +-------------------- 4 files changed, 50 insertions(+), 43 deletions(-) create mode 100644 fvm/evm/offchain/blocks/block_proposal.go diff --git a/fvm/evm/offchain/blocks/block_proposal.go b/fvm/evm/offchain/blocks/block_proposal.go new file mode 100644 index 00000000000..877ba3303fe --- /dev/null +++ b/fvm/evm/offchain/blocks/block_proposal.go @@ -0,0 +1,38 @@ +package blocks + +import ( + "github.com/onflow/flow-go/fvm/evm/events" + "github.com/onflow/flow-go/fvm/evm/types" +) + +func ReconstructProposal( + blockEvent *events.BlockEventPayload, + txEvents []events.TransactionEventPayload, + results []*types.Result, +) *types.BlockProposal { + receipts := make([]types.LightReceipt, 0, len(results)) + + for _, result := range results { + receipts = append(receipts, *result.LightReceipt()) + } + + txHashes := make(types.TransactionHashes, 0, len(txEvents)) + for _, tx := range txEvents { + txHashes = append(txHashes, tx.Hash) + } + + return &types.BlockProposal{ + Block: types.Block{ + ParentBlockHash: blockEvent.ParentBlockHash, + Height: blockEvent.Height, + Timestamp: blockEvent.Timestamp, + TotalSupply: blockEvent.TotalSupply.Big(), + ReceiptRoot: blockEvent.ReceiptRoot, + TransactionHashRoot: blockEvent.TransactionHashRoot, + TotalGasUsed: blockEvent.TotalGasUsed, + PrevRandao: blockEvent.PrevRandao, + }, + Receipts: receipts, + TxHashes: txHashes, + } +} diff --git a/fvm/evm/offchain/sync/replayer_test.go b/fvm/evm/offchain/sync/replayer_test.go index d193163283b..2da1a5ba76b 100644 --- a/fvm/evm/offchain/sync/replayer_test.go +++ b/fvm/evm/offchain/sync/replayer_test.go @@ -12,6 +12,7 @@ import ( "github.com/onflow/flow-go/fvm/evm" "github.com/onflow/flow-go/fvm/evm/events" "github.com/onflow/flow-go/fvm/evm/offchain/blocks" + "github.com/onflow/flow-go/fvm/evm/offchain/storage" "github.com/onflow/flow-go/fvm/evm/offchain/sync" . "github.com/onflow/flow-go/fvm/evm/testutils" "github.com/onflow/flow-go/fvm/evm/types" @@ -154,7 +155,8 @@ func TestChainReplay(t *testing.T) { // check replay - bp, err := blocks.NewBasicProvider(chainID, snapshot, rootAddr) + bpStorage := storage.NewEphemeralStorage(snapshot) + bp, err := blocks.NewBasicProvider(chainID, bpStorage, rootAddr) require.NoError(t, err) err = bp.OnBlockReceived(blockEventPayload) @@ -167,14 +169,12 @@ func TestChainReplay(t *testing.T) { require.Len(t, results, totalTxCount) - err = bp.OnBlockExecuted(blockEventPayload.Height, res) + proposal := blocks.ReconstructProposal(blockEventPayload, txEventPayloads, results) + + err = bp.OnBlockExecuted(blockEventPayload.Height, res, proposal) require.NoError(t, err) - // TODO: verify the state delta - // currently the backend storage doesn't work well with this - // changes needed to make this work, which is left for future PRs - // - // for k, v := range result.StorageRegisterUpdates() { + // for k, v := range bpStorage.StorageRegisterUpdates() { // ret, err := backend.GetValue([]byte(k.Owner), []byte(k.Key)) // require.NoError(t, err) // require.Equal(t, ret[:], v[:]) diff --git a/fvm/evm/offchain/utils/collection_test.go b/fvm/evm/offchain/utils/collection_test.go index ae8b10a0e59..a4385c7f664 100644 --- a/fvm/evm/offchain/utils/collection_test.go +++ b/fvm/evm/offchain/utils/collection_test.go @@ -147,7 +147,7 @@ func replayEvents( sp := NewTestStorageProvider(store, blockEventPayload.Height) cr := sync.NewReplayer(chainID, rootAddr, sp, bp, zerolog.Logger{}, nil, true) - res, _, err := cr.ReplayBlock(txEvents, blockEventPayload) + res, results, err := cr.ReplayBlock(txEvents, blockEventPayload) require.NoError(t, err) // commit all changes @@ -156,7 +156,9 @@ func replayEvents( require.NoError(t, err) } - err = bp.OnBlockExecuted(blockEventPayload.Height, res) + proposal := blocks.ReconstructProposal(blockEventPayload, txEvents, results) + + err = bp.OnBlockExecuted(blockEventPayload.Height, res, proposal) require.NoError(t, err) // commit all block hash list changes diff --git a/fvm/evm/offchain/utils/verify.go b/fvm/evm/offchain/utils/verify.go index a3f3e871f13..3a3d9d9b9ce 100644 --- a/fvm/evm/offchain/utils/verify.go +++ b/fvm/evm/offchain/utils/verify.go @@ -18,7 +18,6 @@ import ( evmStorage "github.com/onflow/flow-go/fvm/evm/offchain/storage" "github.com/onflow/flow-go/fvm/evm/offchain/sync" "github.com/onflow/flow-go/fvm/evm/testutils" - "github.com/onflow/flow-go/fvm/evm/types" "github.com/onflow/flow-go/ledger" "github.com/onflow/flow-go/ledger/common/convert" "github.com/onflow/flow-go/model/flow" @@ -114,7 +113,7 @@ func OffchainReplayBackwardCompatibilityTest( } } - blockProposal := reconstructProposal(evmBlockEvent, evmTxEvents, results) + blockProposal := blocks.ReconstructProposal(evmBlockEvent, evmTxEvents, results) err = bp.OnBlockExecuted(evmBlockEvent.Height, res, blockProposal) if err != nil { @@ -188,35 +187,3 @@ func parseEVMEvents(evts flow.EventsList) (*events.BlockEventPayload, []events.T return blockEvent, txEvents, nil } - -func reconstructProposal( - blockEvent *events.BlockEventPayload, - txEvents []events.TransactionEventPayload, - results []*types.Result, -) *types.BlockProposal { - receipts := make([]types.LightReceipt, 0, len(results)) - - for _, result := range results { - receipts = append(receipts, *result.LightReceipt()) - } - - txHashes := make(types.TransactionHashes, 0, len(txEvents)) - for _, tx := range txEvents { - txHashes = append(txHashes, tx.Hash) - } - - return &types.BlockProposal{ - Block: types.Block{ - ParentBlockHash: blockEvent.ParentBlockHash, - Height: blockEvent.Height, - Timestamp: blockEvent.Timestamp, - TotalSupply: blockEvent.TotalSupply.Big(), - ReceiptRoot: blockEvent.ReceiptRoot, - TransactionHashRoot: blockEvent.TransactionHashRoot, - TotalGasUsed: blockEvent.TotalGasUsed, - PrevRandao: blockEvent.PrevRandao, - }, - Receipts: receipts, - TxHashes: txHashes, - } -} From faccfb684a370575f3d18694d6ff93b1d75ffd57 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Fri, 22 Nov 2024 11:43:45 -0800 Subject: [PATCH 08/45] update error message --- fvm/evm/offchain/utils/verify.go | 76 +++++++++++++++++++++++++------- 1 file changed, 61 insertions(+), 15 deletions(-) diff --git a/fvm/evm/offchain/utils/verify.go b/fvm/evm/offchain/utils/verify.go index 3a3d9d9b9ce..64c50fce3b7 100644 --- a/fvm/evm/offchain/utils/verify.go +++ b/fvm/evm/offchain/utils/verify.go @@ -68,7 +68,7 @@ func OffchainReplayBackwardCompatibilityTest( payloads = append(payloads, chunkData.TrieUpdate.Payloads...) } - updates := make(map[flow.RegisterID]flow.RegisterValue, len(payloads)) + expectedUpdates := make(map[flow.RegisterID]flow.RegisterValue, len(payloads)) for i := len(payloads) - 1; i >= 0; i-- { regID, regVal, err := convert.PayloadToRegister(payloads[i]) if err != nil { @@ -82,8 +82,8 @@ func OffchainReplayBackwardCompatibilityTest( // when iterating backwards, duplicated register updates are stale updates, // so skipping them - if _, ok := updates[regID]; !ok { - updates[regID] = regVal + if _, ok := expectedUpdates[regID]; !ok { + expectedUpdates[regID] = regVal } } @@ -105,12 +105,16 @@ func OffchainReplayBackwardCompatibilityTest( return err } + actualUpdates := make(map[flow.RegisterID]flow.RegisterValue, len(expectedUpdates)) + // commit all changes for k, v := range res.StorageRegisterUpdates() { err = store.SetValue([]byte(k.Owner), []byte(k.Key), v) if err != nil { return err } + + actualUpdates[k] = v } blockProposal := blocks.ReconstructProposal(evmBlockEvent, evmTxEvents, results) @@ -129,20 +133,12 @@ func OffchainReplayBackwardCompatibilityTest( return err } - expectedUpdate, ok := updates[k] - if !ok { - return fmt.Errorf("missing update for register %v, %v", k, expectedUpdate) - } - - if !bytes.Equal(expectedUpdate, v) { - return fmt.Errorf("unexpected update for register %v, expected %v, got %v", k, expectedUpdate, v) - } - - delete(updates, k) + actualUpdates[k] = v } - if len(updates) > 0 { - return fmt.Errorf("missing updates for registers %v", updates) + err = verifyRegisterUpdates(expectedUpdates, actualUpdates) + if err != nil { + return err } log.Info().Msgf("verified block %d", height) @@ -187,3 +183,53 @@ func parseEVMEvents(evts flow.EventsList) (*events.BlockEventPayload, []events.T return blockEvent, txEvents, nil } + +func verifyRegisterUpdates(expectedUpdates map[flow.RegisterID]flow.RegisterValue, actualUpdates map[flow.RegisterID]flow.RegisterValue) error { + missingUpdates := make(map[flow.RegisterID]flow.RegisterValue) + additionalUpdates := make(map[flow.RegisterID]flow.RegisterValue) + mismatchingUpdates := make(map[flow.RegisterID][2]flow.RegisterValue) + + for k, v := range expectedUpdates { + if actualVal, ok := actualUpdates[k]; !ok { + missingUpdates[k] = v + } else if !bytes.Equal(v, actualVal) { + mismatchingUpdates[k] = [2]flow.RegisterValue{v, actualVal} + } + + delete(actualUpdates, k) + } + + for k, v := range actualUpdates { + additionalUpdates[k] = v + } + + // Build a combined error message + var errorMessage strings.Builder + + if len(missingUpdates) > 0 { + errorMessage.WriteString("Missing register updates:\n") + for id, value := range missingUpdates { + errorMessage.WriteString(fmt.Sprintf(" RegisterID: %v, ExpectedValue: %v\n", id, value)) + } + } + + if len(additionalUpdates) > 0 { + errorMessage.WriteString("Additional register updates:\n") + for id, value := range additionalUpdates { + errorMessage.WriteString(fmt.Sprintf(" RegisterID: %v, ActualValue: %v\n", id, value)) + } + } + + if len(mismatchingUpdates) > 0 { + errorMessage.WriteString("Mismatching register updates:\n") + for id, values := range mismatchingUpdates { + errorMessage.WriteString(fmt.Sprintf(" RegisterID: %v, ExpectedValue: %v, ActualValue: %v\n", id, values[0], values[1])) + } + } + + if errorMessage.Len() > 0 { + return errors.New(errorMessage.String()) + } + + return nil +} From 6d3de03b03413d3a72bb2f7fa81ae2eff58ba535 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Fri, 22 Nov 2024 11:45:59 -0800 Subject: [PATCH 09/45] update error message --- fvm/evm/offchain/utils/verify.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fvm/evm/offchain/utils/verify.go b/fvm/evm/offchain/utils/verify.go index 64c50fce3b7..9afb272acec 100644 --- a/fvm/evm/offchain/utils/verify.go +++ b/fvm/evm/offchain/utils/verify.go @@ -209,21 +209,21 @@ func verifyRegisterUpdates(expectedUpdates map[flow.RegisterID]flow.RegisterValu if len(missingUpdates) > 0 { errorMessage.WriteString("Missing register updates:\n") for id, value := range missingUpdates { - errorMessage.WriteString(fmt.Sprintf(" RegisterID: %v, ExpectedValue: %v\n", id, value)) + errorMessage.WriteString(fmt.Sprintf(" RegisterKey: %v, ExpectedValue: %x\n", id.Key, value)) } } if len(additionalUpdates) > 0 { errorMessage.WriteString("Additional register updates:\n") for id, value := range additionalUpdates { - errorMessage.WriteString(fmt.Sprintf(" RegisterID: %v, ActualValue: %v\n", id, value)) + errorMessage.WriteString(fmt.Sprintf(" RegisterKey: %v, ActualValue: %x\n", id.Key, value)) } } if len(mismatchingUpdates) > 0 { errorMessage.WriteString("Mismatching register updates:\n") for id, values := range mismatchingUpdates { - errorMessage.WriteString(fmt.Sprintf(" RegisterID: %v, ExpectedValue: %v, ActualValue: %v\n", id, values[0], values[1])) + errorMessage.WriteString(fmt.Sprintf(" RegisterKey: %v, ExpectedValue: %x, ActualValue: %x\n", id.Key, values[0], values[1])) } } From 26b4481bc246cfb00348fc196d3344f8fdc2f80a Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Fri, 22 Nov 2024 12:06:47 -0800 Subject: [PATCH 10/45] add account status updates --- fvm/evm/offchain/blocks/block_proposal.go | 8 ++----- fvm/evm/offchain/sync/replayer_test.go | 2 +- fvm/evm/offchain/utils/collection_test.go | 2 +- fvm/evm/offchain/utils/verify.go | 27 +++++++++++++++++++---- 4 files changed, 27 insertions(+), 12 deletions(-) diff --git a/fvm/evm/offchain/blocks/block_proposal.go b/fvm/evm/offchain/blocks/block_proposal.go index 877ba3303fe..cd1d68ed517 100644 --- a/fvm/evm/offchain/blocks/block_proposal.go +++ b/fvm/evm/offchain/blocks/block_proposal.go @@ -7,18 +7,14 @@ import ( func ReconstructProposal( blockEvent *events.BlockEventPayload, - txEvents []events.TransactionEventPayload, results []*types.Result, ) *types.BlockProposal { receipts := make([]types.LightReceipt, 0, len(results)) + txHashes := make(types.TransactionHashes, 0, len(results)) for _, result := range results { receipts = append(receipts, *result.LightReceipt()) - } - - txHashes := make(types.TransactionHashes, 0, len(txEvents)) - for _, tx := range txEvents { - txHashes = append(txHashes, tx.Hash) + txHashes = append(txHashes, result.TxHash) } return &types.BlockProposal{ diff --git a/fvm/evm/offchain/sync/replayer_test.go b/fvm/evm/offchain/sync/replayer_test.go index 2da1a5ba76b..3668e445c84 100644 --- a/fvm/evm/offchain/sync/replayer_test.go +++ b/fvm/evm/offchain/sync/replayer_test.go @@ -169,7 +169,7 @@ func TestChainReplay(t *testing.T) { require.Len(t, results, totalTxCount) - proposal := blocks.ReconstructProposal(blockEventPayload, txEventPayloads, results) + proposal := blocks.ReconstructProposal(blockEventPayload, results) err = bp.OnBlockExecuted(blockEventPayload.Height, res, proposal) require.NoError(t, err) diff --git a/fvm/evm/offchain/utils/collection_test.go b/fvm/evm/offchain/utils/collection_test.go index a4385c7f664..8e292530534 100644 --- a/fvm/evm/offchain/utils/collection_test.go +++ b/fvm/evm/offchain/utils/collection_test.go @@ -156,7 +156,7 @@ func replayEvents( require.NoError(t, err) } - proposal := blocks.ReconstructProposal(blockEventPayload, txEvents, results) + proposal := blocks.ReconstructProposal(blockEventPayload, results) err = bp.OnBlockExecuted(blockEventPayload.Height, res, proposal) require.NoError(t, err) diff --git a/fvm/evm/offchain/utils/verify.go b/fvm/evm/offchain/utils/verify.go index 9afb272acec..740989eac23 100644 --- a/fvm/evm/offchain/utils/verify.go +++ b/fvm/evm/offchain/utils/verify.go @@ -25,6 +25,16 @@ import ( "github.com/onflow/flow-go/storage" ) +// EVM Root Height is the first block that has EVM Block Event where the EVM block height is 1 +func isEVMRootHeight(chainID flow.ChainID, flowHeight uint64) bool { + if chainID == flow.Testnet { + return flowHeight == 211176671 + } else if chainID == flow.Mainnet { + return flowHeight == 85981136 + } + return flowHeight == 1 +} + func OffchainReplayBackwardCompatibilityTest( log zerolog.Logger, chainID flow.ChainID, @@ -44,6 +54,14 @@ func OffchainReplayBackwardCompatibilityTest( return err } + // setup account status at EVM root block + if isEVMRootHeight(chainID, flowStartHeight) { + err = bpStorage.SetValue(rootAddr[:], []byte(flow.AccountStatusKey), environment.NewAccountStatus().ToBytes()) + if err != nil { + return err + } + } + for height := flowStartHeight; height <= flowEndHeight; height++ { blockID, err := headers.BlockIDByHeight(height) if err != nil { @@ -87,7 +105,7 @@ func OffchainReplayBackwardCompatibilityTest( } } - // parse events + // parse EVM events evmBlockEvent, evmTxEvents, err := parseEVMEvents(events) if err != nil { return err @@ -107,7 +125,7 @@ func OffchainReplayBackwardCompatibilityTest( actualUpdates := make(map[flow.RegisterID]flow.RegisterValue, len(expectedUpdates)) - // commit all changes + // commit all register changes from the EVM state transition for k, v := range res.StorageRegisterUpdates() { err = store.SetValue([]byte(k.Owner), []byte(k.Key), v) if err != nil { @@ -117,14 +135,15 @@ func OffchainReplayBackwardCompatibilityTest( actualUpdates[k] = v } - blockProposal := blocks.ReconstructProposal(evmBlockEvent, evmTxEvents, results) + blockProposal := blocks.ReconstructProposal(evmBlockEvent, results) err = bp.OnBlockExecuted(evmBlockEvent.Height, res, blockProposal) if err != nil { return err } - // verify and commit all block hash list changes + // commit all register changes from non-EVM state transition, such + // as block hash list changes for k, v := range bpStorage.StorageRegisterUpdates() { // verify the block hash list changes are included in the trie update From 7fc20ce7306984382282d0fcdda39eabc6dfbf9c Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Fri, 22 Nov 2024 16:32:42 -0800 Subject: [PATCH 11/45] update provider --- fvm/evm/offchain/utils/verify.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/fvm/evm/offchain/utils/verify.go b/fvm/evm/offchain/utils/verify.go index 740989eac23..f059874bbb9 100644 --- a/fvm/evm/offchain/utils/verify.go +++ b/fvm/evm/offchain/utils/verify.go @@ -48,21 +48,21 @@ func OffchainReplayBackwardCompatibilityTest( rootAddr := evm.StorageAccountAddress(chainID) rootAddrStr := string(rootAddr.Bytes()) - bpStorage := evmStorage.NewEphemeralStorage(store) - bp, err := blocks.NewBasicProvider(chainID, bpStorage, rootAddr) - if err != nil { - return err - } - - // setup account status at EVM root block - if isEVMRootHeight(chainID, flowStartHeight) { - err = bpStorage.SetValue(rootAddr[:], []byte(flow.AccountStatusKey), environment.NewAccountStatus().ToBytes()) + for height := flowStartHeight; height <= flowEndHeight; height++ { + bpStorage := evmStorage.NewEphemeralStorage(store) + bp, err := blocks.NewBasicProvider(chainID, bpStorage, rootAddr) if err != nil { return err } - } - for height := flowStartHeight; height <= flowEndHeight; height++ { + // setup account status at EVM root block + if isEVMRootHeight(chainID, flowStartHeight) { + err = bpStorage.SetValue(rootAddr[:], []byte(flow.AccountStatusKey), environment.NewAccountStatus().ToBytes()) + if err != nil { + return err + } + } + blockID, err := headers.BlockIDByHeight(height) if err != nil { return err From 2774fc38b65afd0c8fb918133575905f2279683e Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Fri, 22 Nov 2024 16:39:28 -0800 Subject: [PATCH 12/45] update verifable keys --- fvm/evm/handler/blockHashList.go | 9 +++++++++ fvm/evm/offchain/utils/verify.go | 14 ++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/fvm/evm/handler/blockHashList.go b/fvm/evm/handler/blockHashList.go index 91eefded24e..0db2aff73f9 100644 --- a/fvm/evm/handler/blockHashList.go +++ b/fvm/evm/handler/blockHashList.go @@ -3,6 +3,7 @@ package handler import ( "encoding/binary" "fmt" + "strings" gethCommon "github.com/onflow/go-ethereum/common" @@ -26,6 +27,14 @@ const ( heightEncodingSize ) +func IsBlockHashListBucketKeyFormat(id flow.RegisterID) bool { + return strings.HasPrefix(id.Key, "BlockHashListBucket") +} + +func IsBlockHashListMetaKey(id flow.RegisterID) bool { + return id.Key == blockHashListMetaKey +} + // BlockHashList stores the last `capacity` number of block hashes // // Under the hood it breaks the list of hashes into diff --git a/fvm/evm/offchain/utils/verify.go b/fvm/evm/offchain/utils/verify.go index f059874bbb9..cfa48d39f9d 100644 --- a/fvm/evm/offchain/utils/verify.go +++ b/fvm/evm/offchain/utils/verify.go @@ -14,6 +14,7 @@ import ( "github.com/onflow/flow-go/fvm/environment" "github.com/onflow/flow-go/fvm/evm" "github.com/onflow/flow-go/fvm/evm/events" + "github.com/onflow/flow-go/fvm/evm/handler" "github.com/onflow/flow-go/fvm/evm/offchain/blocks" evmStorage "github.com/onflow/flow-go/fvm/evm/offchain/storage" "github.com/onflow/flow-go/fvm/evm/offchain/sync" @@ -98,6 +99,10 @@ func OffchainReplayBackwardCompatibilityTest( continue } + if !verifiableKeys(regID) { + continue + } + // when iterating backwards, duplicated register updates are stale updates, // so skipping them if _, ok := expectedUpdates[regID]; !ok { @@ -152,7 +157,12 @@ func OffchainReplayBackwardCompatibilityTest( return err } + if !verifiableKeys(k) { + continue + } + actualUpdates[k] = v + } err = verifyRegisterUpdates(expectedUpdates, actualUpdates) @@ -166,6 +176,10 @@ func OffchainReplayBackwardCompatibilityTest( return nil } +func verifiableKeys(key flow.RegisterID) bool { + return handler.IsBlockHashListBucketKeyFormat(key) || handler.IsBlockHashListMetaKey(key) +} + func parseEVMEvents(evts flow.EventsList) (*events.BlockEventPayload, []events.TransactionEventPayload, error) { var blockEvent *events.BlockEventPayload txEvents := make([]events.TransactionEventPayload, 0) From 3086e38449a9bd74b676e06ed64cbef993c6cb88 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Fri, 22 Nov 2024 16:42:00 -0800 Subject: [PATCH 13/45] update verifable keys --- fvm/evm/offchain/utils/verify.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fvm/evm/offchain/utils/verify.go b/fvm/evm/offchain/utils/verify.go index cfa48d39f9d..a269e81ec1b 100644 --- a/fvm/evm/offchain/utils/verify.go +++ b/fvm/evm/offchain/utils/verify.go @@ -137,6 +137,10 @@ func OffchainReplayBackwardCompatibilityTest( return err } + if !verifiableKeys(k) { + continue + } + actualUpdates[k] = v } From be8c93ce1bf563f95db4f5d42a6dd58d5c156b10 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Fri, 22 Nov 2024 20:37:34 -0800 Subject: [PATCH 14/45] skip register verification --- fvm/evm/offchain/utils/verify.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fvm/evm/offchain/utils/verify.go b/fvm/evm/offchain/utils/verify.go index a269e81ec1b..9a6f6a45d87 100644 --- a/fvm/evm/offchain/utils/verify.go +++ b/fvm/evm/offchain/utils/verify.go @@ -14,7 +14,6 @@ import ( "github.com/onflow/flow-go/fvm/environment" "github.com/onflow/flow-go/fvm/evm" "github.com/onflow/flow-go/fvm/evm/events" - "github.com/onflow/flow-go/fvm/evm/handler" "github.com/onflow/flow-go/fvm/evm/offchain/blocks" evmStorage "github.com/onflow/flow-go/fvm/evm/offchain/storage" "github.com/onflow/flow-go/fvm/evm/offchain/sync" @@ -58,7 +57,8 @@ func OffchainReplayBackwardCompatibilityTest( // setup account status at EVM root block if isEVMRootHeight(chainID, flowStartHeight) { - err = bpStorage.SetValue(rootAddr[:], []byte(flow.AccountStatusKey), environment.NewAccountStatus().ToBytes()) + err = bpStorage.SetValue(rootAddr[:], []byte(flow.AccountStatusKey), + environment.NewAccountStatus().ToBytes()) if err != nil { return err } @@ -166,7 +166,6 @@ func OffchainReplayBackwardCompatibilityTest( } actualUpdates[k] = v - } err = verifyRegisterUpdates(expectedUpdates, actualUpdates) @@ -181,7 +180,8 @@ func OffchainReplayBackwardCompatibilityTest( } func verifiableKeys(key flow.RegisterID) bool { - return handler.IsBlockHashListBucketKeyFormat(key) || handler.IsBlockHashListMetaKey(key) + return false + // return handler.IsBlockHashListBucketKeyFormat(key) || handler.IsBlockHashListMetaKey(key) } func parseEVMEvents(evts flow.EventsList) (*events.BlockEventPayload, []events.TransactionEventPayload, error) { From 5f3fd1fbbb920507a4f012c1ea42a824b7cf3858 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Tue, 26 Nov 2024 09:42:21 -0800 Subject: [PATCH 15/45] refactor verify.go --- fvm/evm/offchain/utils/verify.go | 61 +++++++++++++++++++++++--------- 1 file changed, 45 insertions(+), 16 deletions(-) diff --git a/fvm/evm/offchain/utils/verify.go b/fvm/evm/offchain/utils/verify.go index 9a6f6a45d87..f09f392b6dd 100644 --- a/fvm/evm/offchain/utils/verify.go +++ b/fvm/evm/offchain/utils/verify.go @@ -26,7 +26,7 @@ import ( ) // EVM Root Height is the first block that has EVM Block Event where the EVM block height is 1 -func isEVMRootHeight(chainID flow.ChainID, flowHeight uint64) bool { +func IsEVMRootHeight(chainID flow.ChainID, flowHeight uint64) bool { if chainID == flow.Testnet { return flowHeight == 211176671 } else if chainID == flow.Mainnet { @@ -44,10 +44,27 @@ func OffchainReplayBackwardCompatibilityTest( results storage.ExecutionResults, executionDataStore execution_data.ExecutionDataGetter, store environment.ValueStore, + onHeightReplayed func(uint64) error, ) error { rootAddr := evm.StorageAccountAddress(chainID) rootAddrStr := string(rootAddr.Bytes()) + if IsEVMRootHeight(chainID, flowStartHeight) { + log.Info().Msgf("initializing EVM state for root height %d", flowStartHeight) + + as := environment.NewAccountStatus() + rootAddr := evm.StorageAccountAddress(chainID) + err := store.SetValue(rootAddr[:], []byte(flow.AccountStatusKey), as.ToBytes()) + if err != nil { + return err + } + } + + // pendingEVMTxEvents are tx events that are executed block included in a flow block that + // didn't emit EVM block event, which is caused when the system tx to emit EVM block fails. + // we accumulate these pending txs, and replay them when we encounter a block with EVM block event. + pendingEVMTxEvents := make([]events.TransactionEventPayload, 0) + for height := flowStartHeight; height <= flowEndHeight; height++ { bpStorage := evmStorage.NewEphemeralStorage(store) bp, err := blocks.NewBasicProvider(chainID, bpStorage, rootAddr) @@ -55,15 +72,6 @@ func OffchainReplayBackwardCompatibilityTest( return err } - // setup account status at EVM root block - if isEVMRootHeight(chainID, flowStartHeight) { - err = bpStorage.SetValue(rootAddr[:], []byte(flow.AccountStatusKey), - environment.NewAccountStatus().ToBytes()) - if err != nil { - return err - } - } - blockID, err := headers.BlockIDByHeight(height) if err != nil { return err @@ -76,14 +84,14 @@ func OffchainReplayBackwardCompatibilityTest( executionData, err := executionDataStore.Get(context.Background(), result.ExecutionDataID) if err != nil { - return err + return fmt.Errorf("could not get execution data %v for block %d: %w", result.ExecutionDataID, height, err) } - events := flow.EventsList{} + evts := flow.EventsList{} payloads := []*ledger.Payload{} for _, chunkData := range executionData.ChunkExecutionDatas { - events = append(events, chunkData.Events...) + evts = append(evts, chunkData.Events...) payloads = append(payloads, chunkData.TrieUpdate.Payloads...) } @@ -111,11 +119,29 @@ func OffchainReplayBackwardCompatibilityTest( } // parse EVM events - evmBlockEvent, evmTxEvents, err := parseEVMEvents(events) + evmBlockEvent, evmTxEvents, err := parseEVMEvents(evts) if err != nil { return err } + pendingEVMTxEvents = append(pendingEVMTxEvents, evmTxEvents...) + + if evmBlockEvent == nil { + log.Info().Msgf("block has no EVM block, height :%v, txEvents: %v", height, len(evmTxEvents)) + + err = onHeightReplayed(height) + if err != nil { + return err + } + continue + } + + // when we encounter a block with EVM block event, we replay the pending txs accumulated + // from previous blocks that had no EVM block event. + evmTxEventsIncludedInBlock := pendingEVMTxEvents + // reset pendingEVMTxEvents + pendingEVMTxEvents = make([]events.TransactionEventPayload, 0) + err = bp.OnBlockReceived(evmBlockEvent) if err != nil { return err @@ -123,7 +149,7 @@ func OffchainReplayBackwardCompatibilityTest( sp := testutils.NewTestStorageProvider(store, evmBlockEvent.Height) cr := sync.NewReplayer(chainID, rootAddr, sp, bp, log, nil, true) - res, results, err := cr.ReplayBlock(evmTxEvents, evmBlockEvent) + res, results, err := cr.ReplayBlock(evmTxEventsIncludedInBlock, evmBlockEvent) if err != nil { return err } @@ -173,7 +199,10 @@ func OffchainReplayBackwardCompatibilityTest( return err } - log.Info().Msgf("verified block %d", height) + err = onHeightReplayed(height) + if err != nil { + return err + } } return nil From 4fb8f6e62fc1c4140d4e33009f2344dcab8c6504 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Tue, 26 Nov 2024 11:24:31 -0800 Subject: [PATCH 16/45] refactor block replay verification --- .../cmd/verify-evm-offchain-replay/verify.go | 1 + fvm/evm/offchain/utils/replay.go | 101 +++++++ fvm/evm/offchain/utils/verify.go | 246 +++++++++--------- 3 files changed, 220 insertions(+), 128 deletions(-) create mode 100644 fvm/evm/offchain/utils/replay.go diff --git a/cmd/util/cmd/verify-evm-offchain-replay/verify.go b/cmd/util/cmd/verify-evm-offchain-replay/verify.go index f75cd8278b6..b7bb0ab0e87 100644 --- a/cmd/util/cmd/verify-evm-offchain-replay/verify.go +++ b/cmd/util/cmd/verify-evm-offchain-replay/verify.go @@ -77,6 +77,7 @@ func Verify(log zerolog.Logger, from uint64, to uint64, chainID flow.ChainID, da storages.Results, executionDataStore, store, + func(uint64) error { return nil }, ) if err != nil { diff --git a/fvm/evm/offchain/utils/replay.go b/fvm/evm/offchain/utils/replay.go new file mode 100644 index 00000000000..1c556f82d19 --- /dev/null +++ b/fvm/evm/offchain/utils/replay.go @@ -0,0 +1,101 @@ +package utils + +import ( + "github.com/rs/zerolog" + + "github.com/onflow/flow-go/fvm/environment" + "github.com/onflow/flow-go/fvm/evm/events" + "github.com/onflow/flow-go/fvm/evm/offchain/blocks" + evmStorage "github.com/onflow/flow-go/fvm/evm/offchain/storage" + "github.com/onflow/flow-go/fvm/evm/offchain/sync" + "github.com/onflow/flow-go/fvm/evm/testutils" + "github.com/onflow/flow-go/model/flow" +) + +func ReplayEVMEventsToStore( + log zerolog.Logger, + store environment.ValueStore, + chainID flow.ChainID, + rootAddr flow.Address, + evmBlockEvent *events.BlockEventPayload, // EVM block event + evmTxEvents []events.TransactionEventPayload, // EVM transaction event +) ( + map[flow.RegisterID]flow.RegisterValue, // EVM state transition updates + map[flow.RegisterID]flow.RegisterValue, // block provider updates + error, +) { + + bpStorage := evmStorage.NewEphemeralStorage(store) + bp, err := blocks.NewBasicProvider(chainID, bpStorage, rootAddr) + if err != nil { + return nil, nil, err + } + + err = bp.OnBlockReceived(evmBlockEvent) + if err != nil { + return nil, nil, err + } + + sp := testutils.NewTestStorageProvider(store, evmBlockEvent.Height) + cr := sync.NewReplayer(chainID, rootAddr, sp, bp, log, nil, true) + res, results, err := cr.ReplayBlock(evmTxEvents, evmBlockEvent) + if err != nil { + return nil, nil, err + } + + // commit all register changes from the EVM state transition + for k, v := range res.StorageRegisterUpdates() { + err = store.SetValue([]byte(k.Owner), []byte(k.Key), v) + if err != nil { + return nil, nil, err + } + } + + blockProposal := blocks.ReconstructProposal(evmBlockEvent, results) + + err = bp.OnBlockExecuted(evmBlockEvent.Height, res, blockProposal) + if err != nil { + return nil, nil, err + } + + // commit all register changes from non-EVM state transition, such + // as block hash list changes + for k, v := range bpStorage.StorageRegisterUpdates() { + // verify the block hash list changes are included in the trie update + + err = store.SetValue([]byte(k.Owner), []byte(k.Key), v) + if err != nil { + return nil, nil, err + } + } + + return res.StorageRegisterUpdates(), bpStorage.StorageRegisterUpdates(), nil +} + +type EVMEventsAccumulator struct { + pendingEVMTxEvents []events.TransactionEventPayload +} + +func NewEVMEventsAccumulator() *EVMEventsAccumulator { + return &EVMEventsAccumulator{ + pendingEVMTxEvents: make([]events.TransactionEventPayload, 0), + } +} + +func (a *EVMEventsAccumulator) HasBlockEvent( + evmBlockEvent *events.BlockEventPayload, + evmTxEvents []events.TransactionEventPayload) ( + *events.BlockEventPayload, + []events.TransactionEventPayload, + bool, // true if there is an EVM block event +) { + a.pendingEVMTxEvents = append(a.pendingEVMTxEvents, evmTxEvents...) + + // if there is no EVM block event, we will accumulate the pending txs + if evmBlockEvent == nil { + return evmBlockEvent, a.pendingEVMTxEvents, false + } + + // if there is an EVM block event, we return the EVM block and the accumulated tx events + return evmBlockEvent, a.pendingEVMTxEvents, true +} diff --git a/fvm/evm/offchain/utils/verify.go b/fvm/evm/offchain/utils/verify.go index f09f392b6dd..c007e4976bc 100644 --- a/fvm/evm/offchain/utils/verify.go +++ b/fvm/evm/offchain/utils/verify.go @@ -14,10 +14,6 @@ import ( "github.com/onflow/flow-go/fvm/environment" "github.com/onflow/flow-go/fvm/evm" "github.com/onflow/flow-go/fvm/evm/events" - "github.com/onflow/flow-go/fvm/evm/offchain/blocks" - evmStorage "github.com/onflow/flow-go/fvm/evm/offchain/storage" - "github.com/onflow/flow-go/fvm/evm/offchain/sync" - "github.com/onflow/flow-go/fvm/evm/testutils" "github.com/onflow/flow-go/ledger" "github.com/onflow/flow-go/ledger/common/convert" "github.com/onflow/flow-go/model/flow" @@ -35,6 +31,24 @@ func IsEVMRootHeight(chainID flow.ChainID, flowHeight uint64) bool { return flowHeight == 1 } +// IsSporkHeight returns true if the given flow height is a spork height for the given chainID +// At spork height, there is no EVM events +func IsSporkHeight(chainID flow.ChainID, flowHeight uint64) bool { + if IsEVMRootHeight(chainID, flowHeight) { + return true + } + + if chainID == flow.Testnet { + return flowHeight == 218215349 // Testnet 52 + } else if chainID == flow.Mainnet { + return flowHeight == 88226267 // Mainnet 26 + } + return false +} + +// OffchainReplayBackwardCompatibilityTest replays the offchain EVM state transition for a given range of flow blocks, +// the replay will also verify the StateUpdateChecksum of the EVM state transition from each transaction execution. +// the updated register values will be saved to the given value store. func OffchainReplayBackwardCompatibilityTest( log zerolog.Logger, chainID flow.ChainID, @@ -49,84 +63,40 @@ func OffchainReplayBackwardCompatibilityTest( rootAddr := evm.StorageAccountAddress(chainID) rootAddrStr := string(rootAddr.Bytes()) - if IsEVMRootHeight(chainID, flowStartHeight) { - log.Info().Msgf("initializing EVM state for root height %d", flowStartHeight) - - as := environment.NewAccountStatus() - rootAddr := evm.StorageAccountAddress(chainID) - err := store.SetValue(rootAddr[:], []byte(flow.AccountStatusKey), as.ToBytes()) - if err != nil { - return err - } - } - // pendingEVMTxEvents are tx events that are executed block included in a flow block that // didn't emit EVM block event, which is caused when the system tx to emit EVM block fails. // we accumulate these pending txs, and replay them when we encounter a block with EVM block event. - pendingEVMTxEvents := make([]events.TransactionEventPayload, 0) + pendingEVMEvents := NewEVMEventsAccumulator() for height := flowStartHeight; height <= flowEndHeight; height++ { - bpStorage := evmStorage.NewEphemeralStorage(store) - bp, err := blocks.NewBasicProvider(chainID, bpStorage, rootAddr) - if err != nil { - return err - } - - blockID, err := headers.BlockIDByHeight(height) - if err != nil { - return err - } - - result, err := results.ByBlockID(blockID) - if err != nil { - return err - } - - executionData, err := executionDataStore.Get(context.Background(), result.ExecutionDataID) - if err != nil { - return fmt.Errorf("could not get execution data %v for block %d: %w", result.ExecutionDataID, height, err) - } - - evts := flow.EventsList{} - payloads := []*ledger.Payload{} - - for _, chunkData := range executionData.ChunkExecutionDatas { - evts = append(evts, chunkData.Events...) - payloads = append(payloads, chunkData.TrieUpdate.Payloads...) - } + // account status initialization for the root account at the EVM root height + if IsEVMRootHeight(chainID, height) { + log.Info().Msgf("initializing EVM state for root height %d", flowStartHeight) - expectedUpdates := make(map[flow.RegisterID]flow.RegisterValue, len(payloads)) - for i := len(payloads) - 1; i >= 0; i-- { - regID, regVal, err := convert.PayloadToRegister(payloads[i]) + as := environment.NewAccountStatus() + rootAddr := evm.StorageAccountAddress(chainID) + err := store.SetValue(rootAddr[:], []byte(flow.AccountStatusKey), as.ToBytes()) if err != nil { return err } + } - // skip non-evm-account registers - if regID.Owner != rootAddrStr { - continue - } - - if !verifiableKeys(regID) { - continue - } - - // when iterating backwards, duplicated register updates are stale updates, - // so skipping them - if _, ok := expectedUpdates[regID]; !ok { - expectedUpdates[regID] = regVal - } + if IsSporkHeight(chainID, height) { + // spork root block has no EVM events + continue } - // parse EVM events - evmBlockEvent, evmTxEvents, err := parseEVMEvents(evts) + // get EVM events and register updates at the flow height + evmBlockEvent, evmTxEvents, registerUpdates, err := evmEventsAndRegisterUpdatesAtFlowHeight( + height, + headers, results, executionDataStore, rootAddrStr) if err != nil { - return err + return fmt.Errorf("failed to get EVM events and register updates at height %d: %w", height, err) } - pendingEVMTxEvents = append(pendingEVMTxEvents, evmTxEvents...) + blockEvent, txEvents, hasBlockEvent := pendingEVMEvents.HasBlockEvent(evmBlockEvent, evmTxEvents) - if evmBlockEvent == nil { + if !hasBlockEvent { log.Info().Msgf("block has no EVM block, height :%v, txEvents: %v", height, len(evmTxEvents)) err = onHeightReplayed(height) @@ -136,65 +106,19 @@ func OffchainReplayBackwardCompatibilityTest( continue } - // when we encounter a block with EVM block event, we replay the pending txs accumulated - // from previous blocks that had no EVM block event. - evmTxEventsIncludedInBlock := pendingEVMTxEvents - // reset pendingEVMTxEvents - pendingEVMTxEvents = make([]events.TransactionEventPayload, 0) - - err = bp.OnBlockReceived(evmBlockEvent) - if err != nil { - return err - } - - sp := testutils.NewTestStorageProvider(store, evmBlockEvent.Height) - cr := sync.NewReplayer(chainID, rootAddr, sp, bp, log, nil, true) - res, results, err := cr.ReplayBlock(evmTxEventsIncludedInBlock, evmBlockEvent) - if err != nil { - return err - } - - actualUpdates := make(map[flow.RegisterID]flow.RegisterValue, len(expectedUpdates)) - - // commit all register changes from the EVM state transition - for k, v := range res.StorageRegisterUpdates() { - err = store.SetValue([]byte(k.Owner), []byte(k.Key), v) - if err != nil { - return err - } - - if !verifiableKeys(k) { - continue - } - - actualUpdates[k] = v - } - - blockProposal := blocks.ReconstructProposal(evmBlockEvent, results) - - err = bp.OnBlockExecuted(evmBlockEvent.Height, res, blockProposal) + evmUpdates, blockProviderUpdates, err := ReplayEVMEventsToStore( + log, + store, + chainID, + rootAddr, + blockEvent, + txEvents, + ) if err != nil { - return err - } - - // commit all register changes from non-EVM state transition, such - // as block hash list changes - for k, v := range bpStorage.StorageRegisterUpdates() { - // verify the block hash list changes are included in the trie update - - err = store.SetValue([]byte(k.Owner), []byte(k.Key), v) - if err != nil { - return err - } - - if !verifiableKeys(k) { - continue - } - - actualUpdates[k] = v + return fmt.Errorf("fail to replay events: %w", err) } - err = verifyRegisterUpdates(expectedUpdates, actualUpdates) + err = verifyEVMRegisterUpdates(registerUpdates, evmUpdates, blockProviderUpdates) if err != nil { return err } @@ -208,11 +132,6 @@ func OffchainReplayBackwardCompatibilityTest( return nil } -func verifiableKeys(key flow.RegisterID) bool { - return false - // return handler.IsBlockHashListBucketKeyFormat(key) || handler.IsBlockHashListMetaKey(key) -} - func parseEVMEvents(evts flow.EventsList) (*events.BlockEventPayload, []events.TransactionEventPayload, error) { var blockEvent *events.BlockEventPayload txEvents := make([]events.TransactionEventPayload, 0) @@ -250,7 +169,78 @@ func parseEVMEvents(evts flow.EventsList) (*events.BlockEventPayload, []events.T return blockEvent, txEvents, nil } -func verifyRegisterUpdates(expectedUpdates map[flow.RegisterID]flow.RegisterValue, actualUpdates map[flow.RegisterID]flow.RegisterValue) error { +func evmEventsAndRegisterUpdatesAtFlowHeight( + flowHeight uint64, + headers storage.Headers, + results storage.ExecutionResults, + executionDataStore execution_data.ExecutionDataGetter, + rootAddr string, +) ( + *events.BlockEventPayload, // EVM block event, might be nil if there is no block Event at this height + []events.TransactionEventPayload, // EVM transaction event + map[flow.RegisterID]flow.RegisterValue, // update registers + error, +) { + + blockID, err := headers.BlockIDByHeight(flowHeight) + if err != nil { + return nil, nil, nil, err + } + + result, err := results.ByBlockID(blockID) + if err != nil { + return nil, nil, nil, err + } + + executionData, err := executionDataStore.Get(context.Background(), result.ExecutionDataID) + if err != nil { + return nil, nil, nil, + fmt.Errorf("could not get execution data %v for block %d: %w", + result.ExecutionDataID, flowHeight, err) + } + + evts := flow.EventsList{} + payloads := []*ledger.Payload{} + + for _, chunkData := range executionData.ChunkExecutionDatas { + evts = append(evts, chunkData.Events...) + payloads = append(payloads, chunkData.TrieUpdate.Payloads...) + } + + updates := make(map[flow.RegisterID]flow.RegisterValue, len(payloads)) + for i := len(payloads) - 1; i >= 0; i-- { + regID, regVal, err := convert.PayloadToRegister(payloads[i]) + if err != nil { + return nil, nil, nil, err + } + + // find the register updates for the root account + if regID.Owner == rootAddr { + updates[regID] = regVal + } + } + + // parse EVM events + evmBlockEvent, evmTxEvents, err := parseEVMEvents(evts) + if err != nil { + return nil, nil, nil, err + } + return evmBlockEvent, evmTxEvents, updates, nil +} + +func verifyEVMRegisterUpdates( + registerUpdates map[flow.RegisterID]flow.RegisterValue, + evmUpdates map[flow.RegisterID]flow.RegisterValue, + blockProviderUpdates map[flow.RegisterID]flow.RegisterValue, +) error { + // skip the register level validation + // since the register is not stored at the same slab id as the on-chain EVM + // instead, we will compare by exporting the logic EVM state, which contains + // accounts, codes and slots. + return nil +} + +func VerifyRegisterUpdates(expectedUpdates map[flow.RegisterID]flow.RegisterValue, actualUpdates map[flow.RegisterID]flow.RegisterValue) error { missingUpdates := make(map[flow.RegisterID]flow.RegisterValue) additionalUpdates := make(map[flow.RegisterID]flow.RegisterValue) mismatchingUpdates := make(map[flow.RegisterID][2]flow.RegisterValue) From a69b60134bfe92a59b4e05951f74f51c20021ee9 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Tue, 26 Nov 2024 11:34:55 -0800 Subject: [PATCH 17/45] refactor util tests --- fvm/evm/offchain/utils/collection_test.go | 42 ++++++----------------- 1 file changed, 10 insertions(+), 32 deletions(-) diff --git a/fvm/evm/offchain/utils/collection_test.go b/fvm/evm/offchain/utils/collection_test.go index 8e292530534..5dad9b86658 100644 --- a/fvm/evm/offchain/utils/collection_test.go +++ b/fvm/evm/offchain/utils/collection_test.go @@ -10,7 +10,6 @@ import ( "strings" "testing" - "github.com/rs/zerolog" "github.com/rs/zerolog/log" "github.com/stretchr/testify/require" @@ -20,9 +19,6 @@ import ( "github.com/onflow/flow-go/fvm/environment" "github.com/onflow/flow-go/fvm/evm" "github.com/onflow/flow-go/fvm/evm/events" - "github.com/onflow/flow-go/fvm/evm/offchain/blocks" - "github.com/onflow/flow-go/fvm/evm/offchain/storage" - "github.com/onflow/flow-go/fvm/evm/offchain/sync" "github.com/onflow/flow-go/fvm/evm/offchain/utils" . "github.com/onflow/flow-go/fvm/evm/testutils" "github.com/onflow/flow-go/model/flow" @@ -128,10 +124,6 @@ func replayEvents( rootAddr := evm.StorageAccountAddress(chainID) - bpStorage := storage.NewEphemeralStorage(store) - bp, err := blocks.NewBasicProvider(chainID, bpStorage, rootAddr) - require.NoError(t, err) - nextHeight := initialNextHeight scanEventFilesAndRun(t, eventsFilePath, @@ -142,31 +134,17 @@ func replayEvents( nextHeight, blockEventPayload.Height) } - err = bp.OnBlockReceived(blockEventPayload) - require.NoError(t, err) - - sp := NewTestStorageProvider(store, blockEventPayload.Height) - cr := sync.NewReplayer(chainID, rootAddr, sp, bp, zerolog.Logger{}, nil, true) - res, results, err := cr.ReplayBlock(txEvents, blockEventPayload) - require.NoError(t, err) - - // commit all changes - for k, v := range res.StorageRegisterUpdates() { - err = store.SetValue([]byte(k.Owner), []byte(k.Key), v) - require.NoError(t, err) - } - - proposal := blocks.ReconstructProposal(blockEventPayload, results) - - err = bp.OnBlockExecuted(blockEventPayload.Height, res, proposal) - require.NoError(t, err) - - // commit all block hash list changes - for k, v := range bpStorage.StorageRegisterUpdates() { - err = store.SetValue([]byte(k.Owner), []byte(k.Key), v) - require.NoError(t, err) + _, _, err := utils.ReplayEVMEventsToStore( + log.Logger, + store, + chainID, + rootAddr, + blockEventPayload, + txEvents, + ) + if err != nil { + return fmt.Errorf("fail to replay events: %w", err) } - // verify the block height is sequential without gap nextHeight++ From a41e107138ff30e013895d097493192ba04ee642 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Tue, 26 Nov 2024 15:30:43 -0800 Subject: [PATCH 18/45] remove unused chainID arg --- cmd/util/cmd/verify-evm-offchain-replay/verify.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/util/cmd/verify-evm-offchain-replay/verify.go b/cmd/util/cmd/verify-evm-offchain-replay/verify.go index b7bb0ab0e87..d3e0b5c6d1d 100644 --- a/cmd/util/cmd/verify-evm-offchain-replay/verify.go +++ b/cmd/util/cmd/verify-evm-offchain-replay/verify.go @@ -31,7 +31,7 @@ func Verify(log zerolog.Logger, from uint64, to uint64, chainID flow.ChainID, da Str("evmStateGobDir", evmStateGobDir). Msgf("verifying range from %d to %d", from, to) - db, storages, executionDataStore, dsStore, err := initStorages(chainID, dataDir, executionDataDir) + db, storages, executionDataStore, dsStore, err := initStorages(dataDir, executionDataDir) if err != nil { return fmt.Errorf("could not initialize storages: %w", err) } @@ -100,7 +100,7 @@ func Verify(log zerolog.Logger, from uint64, to uint64, chainID flow.ChainID, da return nil } -func initStorages(chainID flow.ChainID, dataDir string, executionDataDir string) ( +func initStorages(dataDir string, executionDataDir string) ( *badger.DB, *storage.All, execution_data.ExecutionDataGetter, From 4677d9702c0ebe7963ff356ef9f320d4161b705b Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Tue, 26 Nov 2024 16:53:13 -0800 Subject: [PATCH 19/45] refactor verify --- .../cmd/verify-evm-offchain-replay/main.go | 13 +-- .../cmd/verify-evm-offchain-replay/verify.go | 100 +++++++++++------- 2 files changed, 71 insertions(+), 42 deletions(-) diff --git a/cmd/util/cmd/verify-evm-offchain-replay/main.go b/cmd/util/cmd/verify-evm-offchain-replay/main.go index 0bc6eef8187..d1027fb8a74 100644 --- a/cmd/util/cmd/verify-evm-offchain-replay/main.go +++ b/cmd/util/cmd/verify-evm-offchain-replay/main.go @@ -17,11 +17,12 @@ var ( flagEVMStateGobDir string flagChain string flagFromTo string + flagSaveEveryNBlocks uint64 ) // usage example // -// ./util verify-evm-offchain-replay --chain flow-testnet --from-to 211176671-211177000 +// ./util verify-evm-offchain-replay --chain flow-testnet --from_to 211176671-211177000 // --datadir /var/flow/data/protocol --execution_data_dir /var/flow/data/execution_data var Cmd = &cobra.Command{ Use: "verify-evm-offchain-replay", @@ -44,23 +45,23 @@ func init() { Cmd.Flags().StringVar(&flagEVMStateGobDir, "evm_state_gob_dir", "/var/flow/data/evm_state_gob", "directory that stores the evm state gob files as checkpoint") + + Cmd.Flags().Uint64Var(&flagSaveEveryNBlocks, "save_every", uint64(1_000_000), + "save the evm state gob files every N blocks") } func run(*cobra.Command, []string) { - _ = flow.ChainID(flagChain).Chain() + chainID := flow.ChainID(flagChain) from, to, err := parseFromTo(flagFromTo) if err != nil { log.Fatal().Err(err).Msg("could not parse from_to") } - log.Info().Msgf("verifying range from %d to %d", from, to) - err = Verify(log.Logger, from, to, flow.Testnet, flagDatadir, flagExecutionDataDir, flagEVMStateGobDir) + err = Verify(log.Logger, from, to, chainID, flagDatadir, flagExecutionDataDir, flagEVMStateGobDir, flagSaveEveryNBlocks) if err != nil { log.Fatal().Err(err).Msg("could not verify height") } - log.Info().Msgf("successfully verified range from %d to %d", from, to) - } func parseFromTo(fromTo string) (from, to uint64, err error) { diff --git a/cmd/util/cmd/verify-evm-offchain-replay/verify.go b/cmd/util/cmd/verify-evm-offchain-replay/verify.go index d3e0b5c6d1d..47b34c72afa 100644 --- a/cmd/util/cmd/verify-evm-offchain-replay/verify.go +++ b/cmd/util/cmd/verify-evm-offchain-replay/verify.go @@ -9,10 +9,9 @@ import ( "github.com/dgraph-io/badger/v2" badgerds "github.com/ipfs/go-ds-badger2" "github.com/rs/zerolog" + "github.com/rs/zerolog/log" "github.com/onflow/flow-go/cmd/util/cmd/common" - "github.com/onflow/flow-go/fvm/environment" - "github.com/onflow/flow-go/fvm/evm" "github.com/onflow/flow-go/fvm/evm/offchain/utils" "github.com/onflow/flow-go/fvm/evm/testutils" "github.com/onflow/flow-go/model/flow" @@ -23,13 +22,26 @@ import ( // Verify verifies the offchain replay of EVM blocks from the given height range // and updates the EVM state gob files with the latest state -func Verify(log zerolog.Logger, from uint64, to uint64, chainID flow.ChainID, dataDir string, executionDataDir string, evmStateGobDir string) error { - log.Info(). +func Verify( + log zerolog.Logger, + from uint64, + to uint64, + chainID flow.ChainID, + dataDir string, + executionDataDir string, + evmStateGobDir string, + saveEveryNBlocks uint64, +) error { + lg := log.With(). + Uint64("from", from).Uint64("to", to). Str("chain", chainID.String()). Str("dataDir", dataDir). Str("executionDataDir", executionDataDir). Str("evmStateGobDir", evmStateGobDir). - Msgf("verifying range from %d to %d", from, to) + Uint64("saveEveryNBlocks", saveEveryNBlocks). + Logger() + + lg.Info().Msgf("verifying range from %d to %d", from, to) db, storages, executionDataStore, dsStore, err := initStorages(dataDir, executionDataDir) if err != nil { @@ -40,34 +52,32 @@ func Verify(log zerolog.Logger, from uint64, to uint64, chainID flow.ChainID, da defer dsStore.Close() var store *testutils.TestValueStore - isRoot := isEVMRootHeight(chainID, from) - if isRoot { - log.Info().Msgf("initializing EVM state for root height %d", from) + // root block require the account status registers to be saved + isRoot := utils.IsEVMRootHeight(chainID, from) + if isRoot { store = testutils.GetSimpleValueStore() - as := environment.NewAccountStatus() - rootAddr := evm.StorageAccountAddress(chainID) - err = store.SetValue(rootAddr[:], []byte(flow.AccountStatusKey), as.ToBytes()) - if err != nil { - return err - } } else { prev := from - 1 - log.Info().Msgf("loading EVM state from previous height %d", prev) - - valueFileName, allocatorFileName := evmStateGobFileNamesByEndHeight(evmStateGobDir, prev) - values, err := testutils.DeserializeState(valueFileName) + store, err = loadState(prev, evmStateGobDir) if err != nil { - return fmt.Errorf("could not deserialize state %v: %w", valueFileName, err) + return fmt.Errorf("could not load EVM state from previous height %d: %w", prev, err) } + } - allocators, err := testutils.DeserializeAllocator(allocatorFileName) - if err != nil { - return fmt.Errorf("could not deserialize allocator %v: %w", allocatorFileName, err) + // save state every N blocks + onHeightReplayed := func(height uint64) error { + log.Info().Msgf("replayed height %d", height) + if height%saveEveryNBlocks == 0 { + err := saveState(store, height, evmStateGobDir) + if err != nil { + return err + } } - store = testutils.GetSimpleValueStorePopulated(values, allocators) + return nil } + // replay blocks err = utils.OffchainReplayBackwardCompatibilityTest( log, chainID, @@ -77,16 +87,27 @@ func Verify(log zerolog.Logger, from uint64, to uint64, chainID flow.ChainID, da storages.Results, executionDataStore, store, - func(uint64) error { return nil }, + onHeightReplayed, ) if err != nil { return err } - valueFileName, allocatorFileName := evmStateGobFileNamesByEndHeight(evmStateGobDir, to) + err = saveState(store, to, evmStateGobDir) + if err != nil { + return err + } + + lg.Info().Msgf("successfully verified range from %d to %d", from, to) + + return nil +} + +func saveState(store *testutils.TestValueStore, height uint64, gobDir string) error { + valueFileName, allocatorFileName := evmStateGobFileNamesByEndHeight(gobDir, height) values, allocators := store.Dump() - err = testutils.SerializeState(valueFileName, values) + err := testutils.SerializeState(valueFileName, values) if err != nil { return err } @@ -100,6 +121,23 @@ func Verify(log zerolog.Logger, from uint64, to uint64, chainID flow.ChainID, da return nil } +func loadState(height uint64, gobDir string) (*testutils.TestValueStore, error) { + valueFileName, allocatorFileName := evmStateGobFileNamesByEndHeight(gobDir, height) + values, err := testutils.DeserializeState(valueFileName) + if err != nil { + return nil, fmt.Errorf("could not deserialize state %v: %w", valueFileName, err) + } + + allocators, err := testutils.DeserializeAllocator(allocatorFileName) + if err != nil { + return nil, fmt.Errorf("could not deserialize allocator %v: %w", allocatorFileName, err) + } + store := testutils.GetSimpleValueStorePopulated(values, allocators) + + log.Info().Msgf("loaded EVM state for height %d from gob file %v", height, valueFileName) + return store, nil +} + func initStorages(dataDir string, executionDataDir string) ( *badger.DB, *storage.All, @@ -128,16 +166,6 @@ func initStorages(dataDir string, executionDataDir string) ( return db, storages, executionDataStore, ds, nil } -// EVM Root Height is the first block that has EVM Block Event where the EVM block height is 1 -func isEVMRootHeight(chainID flow.ChainID, flowHeight uint64) bool { - if chainID == flow.Testnet { - return flowHeight == 211176671 - } else if chainID == flow.Mainnet { - return flowHeight == 85981136 - } - return flowHeight == 1 -} - func evmStateGobFileNamesByEndHeight(evmStateGobDir string, endHeight uint64) (string, string) { valueFileName := filepath.Join(evmStateGobDir, fmt.Sprintf("values-%d.gob", endHeight)) allocatorFileName := filepath.Join(evmStateGobDir, fmt.Sprintf("allocators-%d.gob", endHeight)) From bf4d4294d8b484a7d8124295f65c20c47dc798e9 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Tue, 26 Nov 2024 22:50:17 -0800 Subject: [PATCH 20/45] fix spork heights --- cmd/util/cmd/verify-evm-offchain-replay/main.go | 2 +- fvm/evm/offchain/utils/verify.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/util/cmd/verify-evm-offchain-replay/main.go b/cmd/util/cmd/verify-evm-offchain-replay/main.go index d1027fb8a74..d42c9841435 100644 --- a/cmd/util/cmd/verify-evm-offchain-replay/main.go +++ b/cmd/util/cmd/verify-evm-offchain-replay/main.go @@ -22,7 +22,7 @@ var ( // usage example // -// ./util verify-evm-offchain-replay --chain flow-testnet --from_to 211176671-211177000 +// ./util verify-evm-offchain-replay --chain flow-testnet --from_to 211176670-211177000 // --datadir /var/flow/data/protocol --execution_data_dir /var/flow/data/execution_data var Cmd = &cobra.Command{ Use: "verify-evm-offchain-replay", diff --git a/fvm/evm/offchain/utils/verify.go b/fvm/evm/offchain/utils/verify.go index c007e4976bc..3cfe410f315 100644 --- a/fvm/evm/offchain/utils/verify.go +++ b/fvm/evm/offchain/utils/verify.go @@ -24,9 +24,9 @@ import ( // EVM Root Height is the first block that has EVM Block Event where the EVM block height is 1 func IsEVMRootHeight(chainID flow.ChainID, flowHeight uint64) bool { if chainID == flow.Testnet { - return flowHeight == 211176671 + return flowHeight == 211176670 } else if chainID == flow.Mainnet { - return flowHeight == 85981136 + return flowHeight == 85981135 } return flowHeight == 1 } From b82fc5e0d4b0e513b00e46352f2695b9adf7d019 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Tue, 26 Nov 2024 23:27:33 -0800 Subject: [PATCH 21/45] fix pendingEVMTxEvents --- fvm/evm/offchain/utils/replay.go | 5 ++++- fvm/evm/offchain/utils/verify.go | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/fvm/evm/offchain/utils/replay.go b/fvm/evm/offchain/utils/replay.go index 1c556f82d19..d6cb222fa73 100644 --- a/fvm/evm/offchain/utils/replay.go +++ b/fvm/evm/offchain/utils/replay.go @@ -96,6 +96,9 @@ func (a *EVMEventsAccumulator) HasBlockEvent( return evmBlockEvent, a.pendingEVMTxEvents, false } + pendingEVMTxEvents := a.pendingEVMTxEvents + // reset pending events + a.pendingEVMTxEvents = make([]events.TransactionEventPayload, 0) // if there is an EVM block event, we return the EVM block and the accumulated tx events - return evmBlockEvent, a.pendingEVMTxEvents, true + return evmBlockEvent, pendingEVMTxEvents, true } diff --git a/fvm/evm/offchain/utils/verify.go b/fvm/evm/offchain/utils/verify.go index 3cfe410f315..9335beb6230 100644 --- a/fvm/evm/offchain/utils/verify.go +++ b/fvm/evm/offchain/utils/verify.go @@ -79,6 +79,8 @@ func OffchainReplayBackwardCompatibilityTest( if err != nil { return err } + + continue } if IsSporkHeight(chainID, height) { From 556d9036b2540911d1f5578f461329a82e3f8f2b Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Wed, 27 Nov 2024 06:43:35 -0800 Subject: [PATCH 22/45] keep ReplayBlock unchanged --- fvm/evm/offchain/sync/replayer.go | 11 ++++++++++- fvm/evm/offchain/sync/replayer_test.go | 8 +------- fvm/evm/offchain/utils/replay.go | 2 +- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/fvm/evm/offchain/sync/replayer.go b/fvm/evm/offchain/sync/replayer.go index 33411b7c133..96df01d58a0 100644 --- a/fvm/evm/offchain/sync/replayer.go +++ b/fvm/evm/offchain/sync/replayer.go @@ -45,6 +45,15 @@ func NewReplayer( } // ReplayBlock replays the execution of the transactions of an EVM block +func (cr *Replayer) ReplayBlock( + transactionEvents []events.TransactionEventPayload, + blockEvent *events.BlockEventPayload, +) (types.ReplayResultCollector, error) { + res, _, err := cr.ReplayBlockEvents(transactionEvents, blockEvent) + return res, err +} + +// ReplayBlockEvents replays the execution of the transactions of an EVM block // using the provided transactionEvents and blockEvents, // which include all the context data for re-executing the transactions, and returns // the replay result and the result of each transaction. @@ -57,7 +66,7 @@ func NewReplayer( // Warning! the list of transaction events has to be sorted based on their // execution, sometimes the access node might return events out of order // it needs to be sorted by txIndex and eventIndex respectively. -func (cr *Replayer) ReplayBlock( +func (cr *Replayer) ReplayBlockEvents( transactionEvents []events.TransactionEventPayload, blockEvent *events.BlockEventPayload, ) (types.ReplayResultCollector, []*types.Result, error) { diff --git a/fvm/evm/offchain/sync/replayer_test.go b/fvm/evm/offchain/sync/replayer_test.go index 3668e445c84..06262b5811e 100644 --- a/fvm/evm/offchain/sync/replayer_test.go +++ b/fvm/evm/offchain/sync/replayer_test.go @@ -164,7 +164,7 @@ func TestChainReplay(t *testing.T) { sp := NewTestStorageProvider(snapshot, 1) cr := sync.NewReplayer(chainID, rootAddr, sp, bp, zerolog.Logger{}, nil, true) - res, results, err := cr.ReplayBlock(txEventPayloads, blockEventPayload) + res, results, err := cr.ReplayBlockEvents(txEventPayloads, blockEventPayload) require.NoError(t, err) require.Len(t, results, totalTxCount) @@ -173,12 +173,6 @@ func TestChainReplay(t *testing.T) { err = bp.OnBlockExecuted(blockEventPayload.Height, res, proposal) require.NoError(t, err) - - // for k, v := range bpStorage.StorageRegisterUpdates() { - // ret, err := backend.GetValue([]byte(k.Owner), []byte(k.Key)) - // require.NoError(t, err) - // require.Equal(t, ret[:], v[:]) - // } }) }) }) diff --git a/fvm/evm/offchain/utils/replay.go b/fvm/evm/offchain/utils/replay.go index d6cb222fa73..5aba8affcd1 100644 --- a/fvm/evm/offchain/utils/replay.go +++ b/fvm/evm/offchain/utils/replay.go @@ -38,7 +38,7 @@ func ReplayEVMEventsToStore( sp := testutils.NewTestStorageProvider(store, evmBlockEvent.Height) cr := sync.NewReplayer(chainID, rootAddr, sp, bp, log, nil, true) - res, results, err := cr.ReplayBlock(evmTxEvents, evmBlockEvent) + res, results, err := cr.ReplayBlockEvents(evmTxEvents, evmBlockEvent) if err != nil { return nil, nil, err } From 2af3c9eb8760d7ace3f9378d860c0ddf9ca56c4a Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Fri, 22 Nov 2024 23:12:55 -0800 Subject: [PATCH 23/45] from gobs --- cmd/util/cmd/export-evm-state/cmd.go | 67 +++++++++++++++++++++++++--- 1 file changed, 62 insertions(+), 5 deletions(-) diff --git a/cmd/util/cmd/export-evm-state/cmd.go b/cmd/util/cmd/export-evm-state/cmd.go index 2927b9a313a..985c7d1b644 100644 --- a/cmd/util/cmd/export-evm-state/cmd.go +++ b/cmd/util/cmd/export-evm-state/cmd.go @@ -3,13 +3,17 @@ package evm_exporter import ( "fmt" "os" + "path/filepath" "github.com/rs/zerolog/log" "github.com/spf13/cobra" + "github.com/onflow/atree" + "github.com/onflow/flow-go/cmd/util/ledger/util" "github.com/onflow/flow-go/fvm/evm" "github.com/onflow/flow-go/fvm/evm/emulator/state" + "github.com/onflow/flow-go/fvm/evm/testutils" "github.com/onflow/flow-go/ledger" "github.com/onflow/flow-go/ledger/common/convert" "github.com/onflow/flow-go/model/flow" @@ -20,6 +24,8 @@ var ( flagExecutionStateDir string flagOutputDir string flagStateCommitment string + flagEVMStateGobDir string + flagEVMStateGobHeight uint64 ) var Cmd = &cobra.Command{ @@ -34,7 +40,6 @@ func init() { Cmd.Flags().StringVar(&flagExecutionStateDir, "execution-state-dir", "", "Execution Node state dir (where WAL logs are written") - _ = Cmd.MarkFlagRequired("execution-state-dir") Cmd.Flags().StringVar(&flagOutputDir, "output-dir", "", "Directory to write new Execution State to") @@ -42,13 +47,26 @@ func init() { Cmd.Flags().StringVar(&flagStateCommitment, "state-commitment", "", "State commitment (hex-encoded, 64 characters)") + + Cmd.Flags().StringVar(&flagEVMStateGobDir, "evm_state_gob_dir", "/var/flow/data/evm_state_gob", + "directory that stores the evm state gob files as checkpoint") + + Cmd.Flags().Uint64Var(&flagEVMStateGobHeight, "evm_state_gob_height", 0, + "the flow height of the evm state gob files") } func run(*cobra.Command, []string) { log.Info().Msg("start exporting evm state") - err := ExportEVMState(flagChain, flagExecutionStateDir, flagStateCommitment, flagOutputDir) - if err != nil { - log.Fatal().Err(err).Msg("cannot get export evm state") + if flagExecutionStateDir != "" { + err := ExportEVMState(flagChain, flagExecutionStateDir, flagStateCommitment, flagOutputDir) + if err != nil { + log.Fatal().Err(err).Msg("cannot get export evm state") + } + } else if flagEVMStateGobDir != "" { + err := ExportEVMStateFromGob(flagChain, flagEVMStateGobDir, flagEVMStateGobHeight, flagOutputDir) + if err != nil { + log.Fatal().Err(err).Msg("cannot get export evm state from gob files") + } } } @@ -83,7 +101,40 @@ func ExportEVMState( payloadsLedger := util.NewPayloadsLedger(filteredPayloads) - exporter, err := state.NewExporter(payloadsLedger, storageRoot) + return ExportEVMStateFromPayloads(payloadsLedger, storageRoot, outputPath) +} + +func ExportEVMStateFromGob( + chainName string, + evmStateGobDir string, + flowHeight uint64, + outputPath string) error { + + valueFileName, allocatorFileName := evmStateGobFileNamesByEndHeight(evmStateGobDir, flowHeight) + chainID := flow.ChainID(chainName) + + storageRoot := evm.StorageAccountAddress(chainID) + valuesGob, err := testutils.DeserializeState(valueFileName) + if err != nil { + return err + } + + allocatorGobs, err := testutils.DeserializeAllocator(allocatorFileName) + if err != nil { + return err + } + + store := testutils.GetSimpleValueStorePopulated(valuesGob, allocatorGobs) + + return ExportEVMStateFromPayloads(store, storageRoot, outputPath) +} + +func ExportEVMStateFromPayloads( + ledger atree.Ledger, + storageRoot flow.Address, + outputPath string, +) error { + exporter, err := state.NewExporter(ledger, storageRoot) if err != nil { return fmt.Errorf("failed to create exporter: %w", err) } @@ -107,3 +158,9 @@ func ExportEVMState( } return nil } + +func evmStateGobFileNamesByEndHeight(evmStateGobDir string, endHeight uint64) (string, string) { + valueFileName := filepath.Join(evmStateGobDir, fmt.Sprintf("values-%d.gob", endHeight)) + allocatorFileName := filepath.Join(evmStateGobDir, fmt.Sprintf("allocators-%d.gob", endHeight)) + return valueFileName, allocatorFileName +} From d8751bd7d5a84c703a423d1190e31a081c398f49 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Fri, 22 Nov 2024 23:16:34 -0800 Subject: [PATCH 24/45] fix folder exists issue --- cmd/util/cmd/export-evm-state/cmd.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/cmd/util/cmd/export-evm-state/cmd.go b/cmd/util/cmd/export-evm-state/cmd.go index 985c7d1b644..6cb05da37ba 100644 --- a/cmd/util/cmd/export-evm-state/cmd.go +++ b/cmd/util/cmd/export-evm-state/cmd.go @@ -146,12 +146,6 @@ func ExportEVMStateFromPayloads( } } - fi, err := os.Create(outputPath) - if err != nil { - return err - } - defer fi.Close() - err = exporter.Export(outputPath) if err != nil { return fmt.Errorf("failed to export: %w", err) From 066c4e2c1bdb40afda5fc35cb0156e466c4db01f Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Fri, 22 Nov 2024 23:17:54 -0800 Subject: [PATCH 25/45] fix creating files --- fvm/evm/emulator/state/exporter.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fvm/evm/emulator/state/exporter.go b/fvm/evm/emulator/state/exporter.go index 49f3a0fdbd8..595c451b672 100644 --- a/fvm/evm/emulator/state/exporter.go +++ b/fvm/evm/emulator/state/exporter.go @@ -37,7 +37,7 @@ func NewExporter(ledger atree.Ledger, root flow.Address) (*Exporter, error) { } func (e *Exporter) Export(path string) error { - af, err := os.OpenFile(filepath.Join(path, ExportedAccountsFileName), os.O_RDWR, 0644) + af, err := os.Create(filepath.Join(path, ExportedAccountsFileName)) if err != nil { return err } @@ -48,7 +48,7 @@ func (e *Exporter) Export(path string) error { return err } - cf, err := os.OpenFile(filepath.Join(path, ExportedCodesFileName), os.O_RDWR, 0644) + cf, err := os.Create(filepath.Join(path, ExportedCodesFileName)) if err != nil { return err } @@ -59,7 +59,7 @@ func (e *Exporter) Export(path string) error { return err } - sf, err := os.OpenFile(filepath.Join(path, ExportedSlotsFileName), os.O_RDWR, 0644) + sf, err := os.Create(filepath.Join(path, ExportedSlotsFileName)) if err != nil { return err } From 6dbedf15c6125da2a970e433067f5293913a5a0e Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Sat, 23 Nov 2024 16:42:14 -0800 Subject: [PATCH 26/45] add state diff --- fvm/evm/emulator/state/diff.go | 91 +++++++++++++++++++++ fvm/evm/emulator/state/diff_test.go | 19 +++++ fvm/evm/emulator/state/importer.go | 121 ++++++++++++++++++++++++++++ 3 files changed, 231 insertions(+) create mode 100644 fvm/evm/emulator/state/diff.go create mode 100644 fvm/evm/emulator/state/diff_test.go create mode 100644 fvm/evm/emulator/state/importer.go diff --git a/fvm/evm/emulator/state/diff.go b/fvm/evm/emulator/state/diff.go new file mode 100644 index 00000000000..5f79d4102b3 --- /dev/null +++ b/fvm/evm/emulator/state/diff.go @@ -0,0 +1,91 @@ +package state + +import ( + "bytes" + "fmt" +) + +func AccountEqual(a, b *Account) bool { + if a.Address != b.Address { + return false + } + if !bytes.Equal(a.Balance.Bytes(), b.Balance.Bytes()) { + return false + } + if a.Nonce != b.Nonce { + return false + } + if a.CodeHash != b.CodeHash { + return false + } + + // CollectionID could be different + return true +} + +// find the difference and return as error +func Diff(a *EVMState, b *EVMState) []error { + var differences []error + + // Compare Accounts + for addr, accA := range a.Accounts { + if accB, exists := b.Accounts[addr]; exists { + if !AccountEqual(accA, accB) { + differences = append(differences, fmt.Errorf("account %s differs", addr.Hex())) + } + } else { + differences = append(differences, fmt.Errorf("account %s exists in a but not in b", addr.Hex())) + } + } + for addr := range b.Accounts { + if _, exists := a.Accounts[addr]; !exists { + differences = append(differences, fmt.Errorf("account %s exists in b but not in a", addr.Hex())) + } + } + + // Compare Slots + for addr, slotsA := range a.Slots { + slotsB, exists := b.Slots[addr] + if !exists { + differences = append(differences, fmt.Errorf("slots for address %s exist in a but not in b", addr.Hex())) + continue + } + for key, valueA := range slotsA { + if valueB, exists := slotsB[key]; exists { + if valueA.Value != valueB.Value { + differences = append(differences, fmt.Errorf("slot value for address %s and key %s differs", addr.Hex(), key.Hex())) + } + } else { + differences = append(differences, fmt.Errorf("slot with key %s for address %s exists in a but not in b", key.Hex(), addr.Hex())) + } + } + for key := range slotsB { + if _, exists := slotsA[key]; !exists { + differences = append(differences, fmt.Errorf("slot with key %s for address %s exists in b but not in a", key.Hex(), addr.Hex())) + } + } + } + for addr := range b.Slots { + if _, exists := a.Slots[addr]; !exists { + differences = append(differences, fmt.Errorf("slots for address %s exist in b but not in a", addr.Hex())) + } + } + + // Compare Codes + for hash, codeA := range a.Codes { + if codeB, exists := b.Codes[hash]; exists { + if !bytes.Equal(codeA.Code, codeB.Code) { + differences = append(differences, fmt.Errorf("code for hash %s differs", hash.Hex())) + } + } else { + differences = append(differences, fmt.Errorf("code with hash %s exists in a but not in b", hash.Hex())) + } + } + for hash := range b.Codes { + if _, exists := a.Codes[hash]; !exists { + differences = append(differences, fmt.Errorf("code with hash %s exists in b but not in a", hash.Hex())) + } + } + + return differences +} diff --git a/fvm/evm/emulator/state/diff_test.go b/fvm/evm/emulator/state/diff_test.go new file mode 100644 index 00000000000..598a35ca7f5 --- /dev/null +++ b/fvm/evm/emulator/state/diff_test.go @@ -0,0 +1,19 @@ +package state + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestStateDiff(t *testing.T) { + enState, err := ImportEVMState("~/Downloads/compare-state/evm-state-from-checkpoint/") + require.NoError(t, err) + + offchainState, err := ImportEVMState("~/Downloads/compare-state/evm-state-from-offchain/") + require.NoError(t, err) + + differences := Diff(enState, offchainState) + + require.Len(t, differences, 0) +} diff --git a/fvm/evm/emulator/state/importer.go b/fvm/evm/emulator/state/importer.go new file mode 100644 index 00000000000..52a81f1d481 --- /dev/null +++ b/fvm/evm/emulator/state/importer.go @@ -0,0 +1,121 @@ +package state + +import ( + "bufio" + "fmt" + "os" + "path/filepath" + + gethCommon "github.com/onflow/go-ethereum/common" + + "github.com/onflow/flow-go/fvm/evm/types" +) + +type EVMState struct { + Accounts map[gethCommon.Address]*Account + Codes map[gethCommon.Hash]*CodeInContext + // account address -> key -> value + Slots map[gethCommon.Address]map[gethCommon.Hash]*types.SlotEntry +} + +func ToEVMState( + accounts map[gethCommon.Address]*Account, + codes []*CodeInContext, + slots []*types.SlotEntry, +) (*EVMState, error) { + state := &EVMState{ + Accounts: accounts, + Codes: make(map[gethCommon.Hash]*CodeInContext), + Slots: make(map[gethCommon.Address]map[gethCommon.Hash]*types.SlotEntry), + } + + // Process codes + for _, code := range codes { + if _, ok := state.Codes[code.Hash]; ok { + return nil, fmt.Errorf("duplicate code hash: %s", code.Hash) + } + state.Codes[code.Hash] = code + } + + // Process slots + for _, slot := range slots { + if _, ok := state.Slots[slot.Address]; !ok { + state.Slots[slot.Address] = make(map[gethCommon.Hash]*types.SlotEntry) + } + + if _, ok := state.Slots[slot.Address][slot.Key]; ok { + return nil, fmt.Errorf("duplicate slot key: %s", slot.Key) + } + + state.Slots[slot.Address][slot.Key] = slot + } + + return state, nil +} + +func ImportEVMState(path string) (*EVMState, error) { + accounts := make(map[gethCommon.Address]*Account) + var codes []*CodeInContext + var slots []*types.SlotEntry + + // Import accounts + accountsFile, err := os.Open(filepath.Join(path, ExportedAccountsFileName)) + if err != nil { + return nil, fmt.Errorf("error opening accounts file: %w", err) + } + defer accountsFile.Close() + + scanner := bufio.NewScanner(accountsFile) + for scanner.Scan() { + acc, err := DecodeAccount(scanner.Bytes()) + if err != nil { + return nil, fmt.Errorf("error decoding account: %w", err) + } + accounts[acc.Address] = acc + } + + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("error reading accounts file: %w", err) + } + + // Import codes + codesFile, err := os.Open(filepath.Join(path, ExportedCodesFileName)) + if err != nil { + return nil, fmt.Errorf("error opening codes file: %w", err) + } + defer codesFile.Close() + + scanner = bufio.NewScanner(codesFile) + for scanner.Scan() { + code, err := CodeInContextFromEncoded(scanner.Bytes()) + if err != nil { + return nil, fmt.Errorf("error decoding code in context: %w", err) + } + codes = append(codes, code) + } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("error reading codes file: %w", err) + } + + // Import slots + slotsFile, err := os.Open(filepath.Join(path, ExportedSlotsFileName)) + if err != nil { + return nil, fmt.Errorf("error opening slots file: %w", err) + } + defer slotsFile.Close() + + scanner = bufio.NewScanner(slotsFile) + for scanner.Scan() { + slot, err := types.SlotEntryFromEncoded(scanner.Bytes()) + if err != nil { + return nil, fmt.Errorf("error decoding slot entry: %w", err) + } + slots = append(slots, slot) + } + + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("error reading slots file: %w", err) + } + + return ToEVMState(accounts, codes, slots) +} From d1ea26159d7d66b7e22559b1652cd3035eecb13d Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Sat, 23 Nov 2024 19:20:31 -0800 Subject: [PATCH 27/45] debug --- fvm/evm/emulator/state/diff_test.go | 4 +-- fvm/evm/emulator/state/exporter.go | 20 +++++++++++++ fvm/evm/emulator/state/importer.go | 44 +++++++++++++++-------------- 3 files changed, 45 insertions(+), 23 deletions(-) diff --git a/fvm/evm/emulator/state/diff_test.go b/fvm/evm/emulator/state/diff_test.go index 598a35ca7f5..1a1a3161af4 100644 --- a/fvm/evm/emulator/state/diff_test.go +++ b/fvm/evm/emulator/state/diff_test.go @@ -7,10 +7,10 @@ import ( ) func TestStateDiff(t *testing.T) { - enState, err := ImportEVMState("~/Downloads/compare-state/evm-state-from-checkpoint/") + offchainState, err := ImportEVMState("/Users/leozhang/Downloads/compare-state/evm-state-from-gobs/") require.NoError(t, err) - offchainState, err := ImportEVMState("~/Downloads/compare-state/evm-state-from-offchain/") + enState, err := ImportEVMState("/Users/leozhang/Downloads/compare-state/evm-state-from-checkpoint/") require.NoError(t, err) differences := Diff(enState, offchainState) diff --git a/fvm/evm/emulator/state/exporter.go b/fvm/evm/emulator/state/exporter.go index 595c451b672..09c71af277b 100644 --- a/fvm/evm/emulator/state/exporter.go +++ b/fvm/evm/emulator/state/exporter.go @@ -1,6 +1,7 @@ package state import ( + "fmt" "io" "os" "path/filepath" @@ -8,6 +9,7 @@ import ( "github.com/onflow/atree" gethCommon "github.com/onflow/go-ethereum/common" + "github.com/onflow/flow-go/fvm/evm/types" "github.com/onflow/flow-go/model/flow" ) @@ -96,6 +98,12 @@ func (e *Exporter) exportAccounts(writer io.Writer) ([]gethCommon.Address, error if err != nil { return nil, err } + + _, err = DecodeAccount(encoded) + if err != nil { + return nil, fmt.Errorf("account can not be decoded: %w", err) + } + // write every account on a new line _, err = writer.Write(append(encoded, byte('\n'))) if err != nil { @@ -123,6 +131,12 @@ func (e *Exporter) exportCodes(writer io.Writer) error { if err != nil { return err } + + _, err = CodeInContextFromEncoded(encoded) + if err != nil { + return fmt.Errorf("error decoding code in context: %w", err) + } + // write every codes on a new line _, err = writer.Write(append(encoded, byte('\n'))) if err != nil { @@ -151,6 +165,12 @@ func (e *Exporter) exportSlots(addresses []gethCommon.Address, writer io.Writer) if err != nil { return err } + + _, err = types.SlotEntryFromEncoded(encoded) + if err != nil { + return fmt.Errorf("error decoding slot entry: %w", err) + } + // write every codes on a new line _, err = writer.Write(append(encoded, byte('\n'))) if err != nil { diff --git a/fvm/evm/emulator/state/importer.go b/fvm/evm/emulator/state/importer.go index 52a81f1d481..2149b716616 100644 --- a/fvm/evm/emulator/state/importer.go +++ b/fvm/evm/emulator/state/importer.go @@ -58,26 +58,6 @@ func ImportEVMState(path string) (*EVMState, error) { var codes []*CodeInContext var slots []*types.SlotEntry - // Import accounts - accountsFile, err := os.Open(filepath.Join(path, ExportedAccountsFileName)) - if err != nil { - return nil, fmt.Errorf("error opening accounts file: %w", err) - } - defer accountsFile.Close() - - scanner := bufio.NewScanner(accountsFile) - for scanner.Scan() { - acc, err := DecodeAccount(scanner.Bytes()) - if err != nil { - return nil, fmt.Errorf("error decoding account: %w", err) - } - accounts[acc.Address] = acc - } - - if err := scanner.Err(); err != nil { - return nil, fmt.Errorf("error reading accounts file: %w", err) - } - // Import codes codesFile, err := os.Open(filepath.Join(path, ExportedCodesFileName)) if err != nil { @@ -85,7 +65,7 @@ func ImportEVMState(path string) (*EVMState, error) { } defer codesFile.Close() - scanner = bufio.NewScanner(codesFile) + scanner := bufio.NewScanner(codesFile) for scanner.Scan() { code, err := CodeInContextFromEncoded(scanner.Bytes()) if err != nil { @@ -113,6 +93,28 @@ func ImportEVMState(path string) (*EVMState, error) { slots = append(slots, slot) } + // Import accounts + accountsFile, err := os.Open(filepath.Join(path, ExportedAccountsFileName)) + if err != nil { + return nil, fmt.Errorf("error opening accounts file: %w", err) + } + defer accountsFile.Close() + + scanner = bufio.NewScanner(accountsFile) + for scanner.Scan() { + acc, err := DecodeAccount(scanner.Bytes()) + if err != nil { + fmt.Println("error decoding account: ", err, scanner.Bytes()) + } else { + fmt.Println("decoded account", acc.Address) + } + accounts[acc.Address] = acc + } + + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("error reading accounts file: %w", err) + } + if err := scanner.Err(); err != nil { return nil, fmt.Errorf("error reading slots file: %w", err) } From b202ebc85945230c85536c6937ec8b9b5bce05cb Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Sat, 23 Nov 2024 20:12:41 -0800 Subject: [PATCH 28/45] refactor importer --- fvm/evm/emulator/state/diff_test.go | 4 +- fvm/evm/emulator/state/importer.go | 60 ++++++++++++----------------- 2 files changed, 27 insertions(+), 37 deletions(-) diff --git a/fvm/evm/emulator/state/diff_test.go b/fvm/evm/emulator/state/diff_test.go index 1a1a3161af4..60d5a699a88 100644 --- a/fvm/evm/emulator/state/diff_test.go +++ b/fvm/evm/emulator/state/diff_test.go @@ -7,10 +7,10 @@ import ( ) func TestStateDiff(t *testing.T) { - offchainState, err := ImportEVMState("/Users/leozhang/Downloads/compare-state/evm-state-from-gobs/") + offchainState, err := ImportEVMState("/var/flow2/evm-state-from-gobs-218215348/") require.NoError(t, err) - enState, err := ImportEVMState("/Users/leozhang/Downloads/compare-state/evm-state-from-checkpoint/") + enState, err := ImportEVMState("/var/flow2/evm-state-from-gobs-218215348/") require.NoError(t, err) differences := Diff(enState, offchainState) diff --git a/fvm/evm/emulator/state/importer.go b/fvm/evm/emulator/state/importer.go index 2149b716616..c210787a7fe 100644 --- a/fvm/evm/emulator/state/importer.go +++ b/fvm/evm/emulator/state/importer.go @@ -1,10 +1,10 @@ package state import ( - "bufio" "fmt" - "os" + "io/ioutil" "path/filepath" + "strings" gethCommon "github.com/onflow/go-ethereum/common" @@ -57,36 +57,34 @@ func ImportEVMState(path string) (*EVMState, error) { accounts := make(map[gethCommon.Address]*Account) var codes []*CodeInContext var slots []*types.SlotEntry - // Import codes - codesFile, err := os.Open(filepath.Join(path, ExportedCodesFileName)) + codesData, err := ioutil.ReadFile(filepath.Join(path, ExportedCodesFileName)) if err != nil { return nil, fmt.Errorf("error opening codes file: %w", err) } - defer codesFile.Close() - - scanner := bufio.NewScanner(codesFile) - for scanner.Scan() { - code, err := CodeInContextFromEncoded(scanner.Bytes()) + codesLines := strings.Split(string(codesData), "\n") + for _, line := range codesLines { + if line == "" { + continue + } + code, err := CodeInContextFromEncoded([]byte(line)) if err != nil { return nil, fmt.Errorf("error decoding code in context: %w", err) } codes = append(codes, code) } - if err := scanner.Err(); err != nil { - return nil, fmt.Errorf("error reading codes file: %w", err) - } // Import slots - slotsFile, err := os.Open(filepath.Join(path, ExportedSlotsFileName)) + slotsData, err := ioutil.ReadFile(filepath.Join(path, ExportedSlotsFileName)) if err != nil { return nil, fmt.Errorf("error opening slots file: %w", err) } - defer slotsFile.Close() - - scanner = bufio.NewScanner(slotsFile) - for scanner.Scan() { - slot, err := types.SlotEntryFromEncoded(scanner.Bytes()) + slotsLines := strings.Split(string(slotsData), "\n") + for _, line := range slotsLines { + if line == "" { + continue + } + slot, err := types.SlotEntryFromEncoded([]byte(line)) if err != nil { return nil, fmt.Errorf("error decoding slot entry: %w", err) } @@ -94,30 +92,22 @@ func ImportEVMState(path string) (*EVMState, error) { } // Import accounts - accountsFile, err := os.Open(filepath.Join(path, ExportedAccountsFileName)) + accountsData, err := ioutil.ReadFile(filepath.Join(path, ExportedAccountsFileName)) if err != nil { return nil, fmt.Errorf("error opening accounts file: %w", err) } - defer accountsFile.Close() - - scanner = bufio.NewScanner(accountsFile) - for scanner.Scan() { - acc, err := DecodeAccount(scanner.Bytes()) + accountsLines := strings.Split(string(accountsData), "\n") + for _, line := range accountsLines { + if line == "" { + continue + } + acc, err := DecodeAccount([]byte(line)) if err != nil { - fmt.Println("error decoding account: ", err, scanner.Bytes()) + fmt.Println("error decoding account: ", err, line) } else { fmt.Println("decoded account", acc.Address) + accounts[acc.Address] = acc } - accounts[acc.Address] = acc - } - - if err := scanner.Err(); err != nil { - return nil, fmt.Errorf("error reading accounts file: %w", err) } - - if err := scanner.Err(); err != nil { - return nil, fmt.Errorf("error reading slots file: %w", err) - } - return ToEVMState(accounts, codes, slots) } From 2564f0352a839f3945b786e0e42fda72233ffe51 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Sat, 23 Nov 2024 20:53:49 -0800 Subject: [PATCH 29/45] export evm state as gobs --- fvm/evm/emulator/state/diff_test.go | 4 +- fvm/evm/emulator/state/exporter.go | 26 +++++++++ fvm/evm/emulator/state/extract.go | 84 +++++++++++++++++++++++++++++ fvm/evm/emulator/state/importer.go | 24 +++++++++ 4 files changed, 136 insertions(+), 2 deletions(-) create mode 100644 fvm/evm/emulator/state/extract.go diff --git a/fvm/evm/emulator/state/diff_test.go b/fvm/evm/emulator/state/diff_test.go index 60d5a699a88..913b6e3bd7f 100644 --- a/fvm/evm/emulator/state/diff_test.go +++ b/fvm/evm/emulator/state/diff_test.go @@ -7,10 +7,10 @@ import ( ) func TestStateDiff(t *testing.T) { - offchainState, err := ImportEVMState("/var/flow2/evm-state-from-gobs-218215348/") + offchainState, err := ImportEVMStateFromGob("/var/flow2/evm-state-from-gobs-218215348/") require.NoError(t, err) - enState, err := ImportEVMState("/var/flow2/evm-state-from-gobs-218215348/") + enState, err := ImportEVMStateFromGob("/var/flow2/evm-state-from-gobs-218215348/") require.NoError(t, err) differences := Diff(enState, offchainState) diff --git a/fvm/evm/emulator/state/exporter.go b/fvm/evm/emulator/state/exporter.go index 09c71af277b..2dd3028b287 100644 --- a/fvm/evm/emulator/state/exporter.go +++ b/fvm/evm/emulator/state/exporter.go @@ -1,6 +1,7 @@ package state import ( + "encoding/gob" "fmt" "io" "os" @@ -17,6 +18,7 @@ const ( ExportedAccountsFileName = "accounts.bin" ExportedCodesFileName = "codes.bin" ExportedSlotsFileName = "slots.bin" + ExportedStateGobFileName = "state.gob" ) type Exporter struct { @@ -38,6 +40,30 @@ func NewExporter(ledger atree.Ledger, root flow.Address) (*Exporter, error) { }, nil } +func (e *Exporter) ExportGob(path string) error { + fileName := filepath.Join(path, ExportedStateGobFileName) + // Open the file for reading + file, err := os.Open(fileName) + if err != nil { + return err + } + defer file.Close() + + state, err := Extract(e.ledger, e.root, e.baseView) + if err != nil { + return err + } + + // Use gob to encode data + encoder := gob.NewEncoder(file) + err = encoder.Encode(state) + if err != nil { + return err + } + + return nil +} + func (e *Exporter) Export(path string) error { af, err := os.Create(filepath.Join(path, ExportedAccountsFileName)) if err != nil { diff --git a/fvm/evm/emulator/state/extract.go b/fvm/evm/emulator/state/extract.go new file mode 100644 index 00000000000..47d78d41d99 --- /dev/null +++ b/fvm/evm/emulator/state/extract.go @@ -0,0 +1,84 @@ +package state + +import ( + "github.com/onflow/atree" + gethCommon "github.com/onflow/go-ethereum/common" + + "github.com/onflow/flow-go/fvm/evm/types" + "github.com/onflow/flow-go/model/flow" +) + +func Extract( + ledger atree.Ledger, + root flow.Address, + baseView *BaseView, +) (*EVMState, error) { + + accounts := make(map[gethCommon.Address]*Account, 0) + + itr, err := baseView.AccountIterator() + + if err != nil { + return nil, err + } + // make a list of accounts with storage + addrWithSlots := make([]gethCommon.Address, 0) + for { + // TODO: we can optimize by returning the encoded value + acc, err := itr.Next() + if err != nil { + return nil, err + } + if acc == nil { + break + } + if acc.HasStoredValues() { + addrWithSlots = append(addrWithSlots, acc.Address) + } + accounts[acc.Address] = acc + } + + codes := make(map[gethCommon.Hash]*CodeInContext, 0) + codeItr, err := baseView.CodeIterator() + if err != nil { + return nil, err + } + for { + cic, err := codeItr.Next() + if err != nil { + return nil, err + } + if cic == nil { + break + } + codes[cic.Hash] = cic + } + + // account address -> key -> value + slots := make(map[gethCommon.Address]map[gethCommon.Hash]*types.SlotEntry) + + for _, addr := range addrWithSlots { + slots[addr] = make(map[gethCommon.Hash]*types.SlotEntry) + slotItr, err := baseView.AccountStorageIterator(addr) + if err != nil { + return nil, err + } + for { + slot, err := slotItr.Next() + if err != nil { + return nil, err + } + if slot == nil { + break + } + + slots[addr][slot.Key] = slot + } + } + + return &EVMState{ + Accounts: accounts, + Codes: codes, + Slots: slots, + }, nil +} diff --git a/fvm/evm/emulator/state/importer.go b/fvm/evm/emulator/state/importer.go index c210787a7fe..132846512f4 100644 --- a/fvm/evm/emulator/state/importer.go +++ b/fvm/evm/emulator/state/importer.go @@ -1,8 +1,10 @@ package state import ( + "encoding/gob" "fmt" "io/ioutil" + "os" "path/filepath" "strings" @@ -53,6 +55,28 @@ func ToEVMState( return state, nil } +func ImportEVMStateFromGob(path string) (*EVMState, error) { + fileName := filepath.Join(path, ExportedStateGobFileName) + // Open the file for reading + file, err := os.Open(fileName) + if err != nil { + return nil, err + } + defer file.Close() + + // Prepare the map to store decoded data + var data EVMState + + // Use gob to decode data + decoder := gob.NewDecoder(file) + err = decoder.Decode(&data) + if err != nil { + return nil, err + } + + return &data, nil +} + func ImportEVMState(path string) (*EVMState, error) { accounts := make(map[gethCommon.Address]*Account) var codes []*CodeInContext From 86fc3be20b150c7006ec84258b28c7e4b2d35613 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Sat, 23 Nov 2024 20:55:34 -0800 Subject: [PATCH 30/45] as gob --- cmd/util/cmd/export-evm-state/cmd.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/util/cmd/export-evm-state/cmd.go b/cmd/util/cmd/export-evm-state/cmd.go index 6cb05da37ba..c29ac6c1436 100644 --- a/cmd/util/cmd/export-evm-state/cmd.go +++ b/cmd/util/cmd/export-evm-state/cmd.go @@ -146,7 +146,7 @@ func ExportEVMStateFromPayloads( } } - err = exporter.Export(outputPath) + err = exporter.ExportGob(outputPath) if err != nil { return fmt.Errorf("failed to export: %w", err) } From eb8d42804b629b5ac44df7ecc0e2d556bdab3be0 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Sat, 23 Nov 2024 20:57:19 -0800 Subject: [PATCH 31/45] open file --- fvm/evm/emulator/state/exporter.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fvm/evm/emulator/state/exporter.go b/fvm/evm/emulator/state/exporter.go index 2dd3028b287..f3c8072e070 100644 --- a/fvm/evm/emulator/state/exporter.go +++ b/fvm/evm/emulator/state/exporter.go @@ -43,7 +43,7 @@ func NewExporter(ledger atree.Ledger, root flow.Address) (*Exporter, error) { func (e *Exporter) ExportGob(path string) error { fileName := filepath.Join(path, ExportedStateGobFileName) // Open the file for reading - file, err := os.Open(fileName) + file, err := os.Create(fileName) if err != nil { return err } From 24e36fd47a92c62f4e425375bf9d2ac5986e8d8b Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Sat, 23 Nov 2024 21:32:19 -0800 Subject: [PATCH 32/45] show account diff --- fvm/evm/emulator/state/diff.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fvm/evm/emulator/state/diff.go b/fvm/evm/emulator/state/diff.go index 5f79d4102b3..bae539bd5db 100644 --- a/fvm/evm/emulator/state/diff.go +++ b/fvm/evm/emulator/state/diff.go @@ -31,7 +31,7 @@ func Diff(a *EVMState, b *EVMState) []error { for addr, accA := range a.Accounts { if accB, exists := b.Accounts[addr]; exists { if !AccountEqual(accA, accB) { - differences = append(differences, fmt.Errorf("account %s differs", addr.Hex())) + differences = append(differences, fmt.Errorf("account %s differs, accA %v, accB %v", addr.Hex(), accA, accB)) } } else { differences = append(differences, fmt.Errorf("account %s exists in a but not in b", addr.Hex())) From 5fe1eea336ad0bf0db5b1ad59a2ddfbcac0acef2 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Mon, 25 Nov 2024 10:24:38 -0800 Subject: [PATCH 33/45] updat export gob method --- fvm/evm/emulator/state/exporter.go | 2 +- fvm/evm/emulator/state/extract.go | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/fvm/evm/emulator/state/exporter.go b/fvm/evm/emulator/state/exporter.go index f3c8072e070..f1cb9bcfa10 100644 --- a/fvm/evm/emulator/state/exporter.go +++ b/fvm/evm/emulator/state/exporter.go @@ -49,7 +49,7 @@ func (e *Exporter) ExportGob(path string) error { } defer file.Close() - state, err := Extract(e.ledger, e.root, e.baseView) + state, err := Extract(e.root, e.baseView) if err != nil { return err } diff --git a/fvm/evm/emulator/state/extract.go b/fvm/evm/emulator/state/extract.go index 47d78d41d99..e0bb30d82aa 100644 --- a/fvm/evm/emulator/state/extract.go +++ b/fvm/evm/emulator/state/extract.go @@ -1,7 +1,6 @@ package state import ( - "github.com/onflow/atree" gethCommon "github.com/onflow/go-ethereum/common" "github.com/onflow/flow-go/fvm/evm/types" @@ -9,7 +8,6 @@ import ( ) func Extract( - ledger atree.Ledger, root flow.Address, baseView *BaseView, ) (*EVMState, error) { From fb7b4bc84d6fffde3a594bb478e0a9b4a059139c Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Mon, 25 Nov 2024 10:33:59 -0800 Subject: [PATCH 34/45] fix lint --- fvm/evm/offchain/blocks/block_context.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/fvm/evm/offchain/blocks/block_context.go b/fvm/evm/offchain/blocks/block_context.go index 2da73f9cb32..5b5095a9d2f 100644 --- a/fvm/evm/offchain/blocks/block_context.go +++ b/fvm/evm/offchain/blocks/block_context.go @@ -83,6 +83,11 @@ const blockHashListFixHCUEVMHeightMainnet = 8357079 // PR: https://github.com/onflow/flow-go/pull/6734 const blockHashListFixHCUEVMHeightTestnet = 16848829 +// Testnet52 - Spork +// Flow Block: 218215350 cc7188f0bdac4c442cc3ee072557d7f7c8ca4462537da945b148d5d0efa7a1ff +// PR: https://github.com/onflow/flow-go/pull/6377 +// const blockHashListBugIntroducedHCUEVMHeightTestnet = 7038679 + // Testnet51 - Height Coordinated Upgrade 1 // Flow Block: 212562161 1a520608c5457f228405c4c30fc39c8a0af7cf915fb2ede7ec5ccffc2a000f57 // PR: https://github.com/onflow/flow-go/pull/6380 From 888bddbac9dc6cda48640d78b243878f15067997 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Tue, 26 Nov 2024 07:05:08 -0800 Subject: [PATCH 35/45] fix coinbaseAddressChangeEVMHeightTestnet --- fvm/evm/offchain/blocks/block_context.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fvm/evm/offchain/blocks/block_context.go b/fvm/evm/offchain/blocks/block_context.go index 5b5095a9d2f..669f2dda998 100644 --- a/fvm/evm/offchain/blocks/block_context.go +++ b/fvm/evm/offchain/blocks/block_context.go @@ -91,7 +91,7 @@ const blockHashListFixHCUEVMHeightTestnet = 16848829 // Testnet51 - Height Coordinated Upgrade 1 // Flow Block: 212562161 1a520608c5457f228405c4c30fc39c8a0af7cf915fb2ede7ec5ccffc2a000f57 // PR: https://github.com/onflow/flow-go/pull/6380 -const coinbaseAddressChangeEVMHeightTestnet = 1385491 +const coinbaseAddressChangeEVMHeightTestnet = 1385490 var genesisCoinbaseAddressTestnet = types.Address(gethCommon.HexToAddress("0000000000000000000000021169100eecb7c1a6")) From cf326885405e6e9d9426659924697bfc5f0826f2 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Tue, 26 Nov 2024 19:31:19 -0800 Subject: [PATCH 36/45] add diff test cases --- fvm/evm/emulator/state/diff_test.go | 62 +++++++++++++++++++++++++++-- 1 file changed, 58 insertions(+), 4 deletions(-) diff --git a/fvm/evm/emulator/state/diff_test.go b/fvm/evm/emulator/state/diff_test.go index 913b6e3bd7f..68f798e05c6 100644 --- a/fvm/evm/emulator/state/diff_test.go +++ b/fvm/evm/emulator/state/diff_test.go @@ -1,19 +1,73 @@ -package state +package state_test import ( + "fmt" + "path/filepath" "testing" + "github.com/onflow/flow-go/fvm/evm" + "github.com/onflow/flow-go/fvm/evm/emulator/state" + "github.com/onflow/flow-go/fvm/evm/testutils" + "github.com/onflow/flow-go/model/flow" "github.com/stretchr/testify/require" ) func TestStateDiff(t *testing.T) { - offchainState, err := ImportEVMStateFromGob("/var/flow2/evm-state-from-gobs-218215348/") + offchainState, err := state.ImportEVMStateFromGob("/var/flow2/evm-state-from-gobs-218215348/") require.NoError(t, err) - enState, err := ImportEVMStateFromGob("/var/flow2/evm-state-from-gobs-218215348/") + enState, err := state.ImportEVMStateFromGob("/var/flow2/evm-state-from-gobs-218215348/") require.NoError(t, err) - differences := Diff(enState, offchainState) + differences := state.Diff(enState, offchainState) require.Len(t, differences, 0) } + +func TestEVMStateDiff(t *testing.T) { + + state1 := EVMStateFromReplayGobDir(t, "/var/flow2/evm-state-from-gobs-218215348/", uint64(218215348)) + // state2 := EVMStateFromReplayGobDir(t, "/var/flow2/evm-state-from-gobs-218215348/", uint64(218215348)) + state2 := EVMStateFromCheckpointExtract(t, "/var/flow2/evm-state-from-checkpoint-218215348/") + + differences := state.Diff(state1, state2) + + for i, diff := range differences { + fmt.Printf("Difference %d: %v\n", i, diff) + } + + require.Len(t, differences, 0) +} + +func EVMStateFromCheckpointExtract(t *testing.T, dir string) *state.EVMState { + enState, err := state.ImportEVMStateFromGob("/var/flow2/evm-state-from-gobs-218215348/") + require.NoError(t, err) + return enState +} + +func EVMStateFromReplayGobDir(t *testing.T, gobDir string, flowHeight uint64) *state.EVMState { + valueFileName, allocatorFileName := evmStateGobFileNamesByEndHeight(gobDir, flowHeight) + chainID := flow.Testnet + + allocatorGobs, err := testutils.DeserializeAllocator(allocatorFileName) + require.NoError(t, err) + + storageRoot := evm.StorageAccountAddress(chainID) + valuesGob, err := testutils.DeserializeState(valueFileName) + require.NoError(t, err) + + store := testutils.GetSimpleValueStorePopulated(valuesGob, allocatorGobs) + + bv, err := state.NewBaseView(store, storageRoot) + require.NoError(t, err) + + evmState, err := state.Extract(storageRoot, bv) + require.NoError(t, err) + return evmState +} + +func evmStateGobFileNamesByEndHeight(evmStateGobDir string, endHeight uint64) (string, string) { + valueFileName := filepath.Join(evmStateGobDir, fmt.Sprintf("values-%d.gob", endHeight)) + allocatorFileName := filepath.Join(evmStateGobDir, fmt.Sprintf("allocators-%d.gob", endHeight)) + return valueFileName, allocatorFileName +} From d50ddb78c18718d7d7ef0f0b55af7f1eea42ca01 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Wed, 27 Nov 2024 06:46:15 -0800 Subject: [PATCH 37/45] fix lint --- fvm/evm/emulator/state/diff_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fvm/evm/emulator/state/diff_test.go b/fvm/evm/emulator/state/diff_test.go index 68f798e05c6..057c4314fa4 100644 --- a/fvm/evm/emulator/state/diff_test.go +++ b/fvm/evm/emulator/state/diff_test.go @@ -5,11 +5,12 @@ import ( "path/filepath" "testing" + "github.com/stretchr/testify/require" + "github.com/onflow/flow-go/fvm/evm" "github.com/onflow/flow-go/fvm/evm/emulator/state" "github.com/onflow/flow-go/fvm/evm/testutils" "github.com/onflow/flow-go/model/flow" - "github.com/stretchr/testify/require" ) func TestStateDiff(t *testing.T) { From 10d4b9db7f3275280d04f65055f3b0aa1c0919fd Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Wed, 27 Nov 2024 09:38:11 -0800 Subject: [PATCH 38/45] update error message --- fvm/evm/offchain/storage/readonly.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fvm/evm/offchain/storage/readonly.go b/fvm/evm/offchain/storage/readonly.go index 4ed33a6fe44..6c66e7c1e43 100644 --- a/fvm/evm/offchain/storage/readonly.go +++ b/fvm/evm/offchain/storage/readonly.go @@ -1,7 +1,7 @@ package storage import ( - "errors" + "fmt" "github.com/onflow/atree" @@ -29,7 +29,7 @@ func (s *ReadOnlyStorage) GetValue(owner []byte, key []byte) ([]byte, error) { // SetValue returns an error if called func (s *ReadOnlyStorage) SetValue(owner, key, value []byte) error { - return errors.New("unexpected call received") + return fmt.Errorf("unexpected call received for SetValue with owner: %x, key: %v, value: %x", owner, key, value) } // ValueExists checks if a register exists @@ -40,5 +40,5 @@ func (s *ReadOnlyStorage) ValueExists(owner []byte, key []byte) (bool, error) { // AllocateSlabIndex returns an error if called func (s *ReadOnlyStorage) AllocateSlabIndex(owner []byte) (atree.SlabIndex, error) { - return atree.SlabIndex{}, errors.New("unexpected call received") + return atree.SlabIndex{}, fmt.Errorf("unexpected call received for AllocateSlabIndex with owner: %x", owner) } From 59a1d05f56115848b71d882637fa033bd1276db2 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Wed, 27 Nov 2024 09:41:43 -0800 Subject: [PATCH 39/45] use panic --- fvm/evm/offchain/storage/readonly.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fvm/evm/offchain/storage/readonly.go b/fvm/evm/offchain/storage/readonly.go index 6c66e7c1e43..730c8670dcb 100644 --- a/fvm/evm/offchain/storage/readonly.go +++ b/fvm/evm/offchain/storage/readonly.go @@ -29,7 +29,7 @@ func (s *ReadOnlyStorage) GetValue(owner []byte, key []byte) ([]byte, error) { // SetValue returns an error if called func (s *ReadOnlyStorage) SetValue(owner, key, value []byte) error { - return fmt.Errorf("unexpected call received for SetValue with owner: %x, key: %v, value: %x", owner, key, value) + panic(fmt.Sprintf("unexpected call received for SetValue with owner: %x, key: %v, value: %x", owner, key, value)) } // ValueExists checks if a register exists @@ -40,5 +40,5 @@ func (s *ReadOnlyStorage) ValueExists(owner []byte, key []byte) (bool, error) { // AllocateSlabIndex returns an error if called func (s *ReadOnlyStorage) AllocateSlabIndex(owner []byte) (atree.SlabIndex, error) { - return atree.SlabIndex{}, fmt.Errorf("unexpected call received for AllocateSlabIndex with owner: %x", owner) + panic(fmt.Errorf("unexpected call received for AllocateSlabIndex with owner: %x", owner)) } From 2ff3bee18d812931bab6044de1f3d9e051b72793 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Wed, 27 Nov 2024 09:55:17 -0800 Subject: [PATCH 40/45] include missing key in error message --- fvm/evm/emulator/state/base.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fvm/evm/emulator/state/base.go b/fvm/evm/emulator/state/base.go index 9f11ce6e3f0..9295bf6ac9f 100644 --- a/fvm/evm/emulator/state/base.go +++ b/fvm/evm/emulator/state/base.go @@ -74,13 +74,13 @@ func NewBaseView(ledger atree.Ledger, rootAddress flow.Address) (*BaseView, erro // fetch the account collection, if not exist, create one view.accounts, view.accountSetupOnCommit, err = view.fetchOrCreateCollection(AccountsStorageIDKey) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to fetch or create account collection with key %v: %w", AccountsStorageIDKey, err) } // fetch the code collection, if not exist, create one view.codes, view.codeSetupOnCommit, err = view.fetchOrCreateCollection(CodesStorageIDKey) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to fetch or create code collection with key %v: %w", CodesStorageIDKey, err) } return view, nil From b73baac082e79e38449d487329f65265240d3e09 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Wed, 27 Nov 2024 10:00:13 -0800 Subject: [PATCH 41/45] update error message --- fvm/evm/emulator/state/base.go | 2 +- fvm/evm/offchain/storage/readonly.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/fvm/evm/emulator/state/base.go b/fvm/evm/emulator/state/base.go index 9295bf6ac9f..97df2076957 100644 --- a/fvm/evm/emulator/state/base.go +++ b/fvm/evm/emulator/state/base.go @@ -485,7 +485,7 @@ func (v *BaseView) fetchOrCreateCollection(path string) (collection *Collection, } if len(collectionID) == 0 { collection, err = v.collectionProvider.NewCollection() - return collection, true, err + return collection, true, fmt.Errorf("fail to create collection with key %v: %w", path, err) } collection, err = v.collectionProvider.CollectionByID(collectionID) return collection, false, err diff --git a/fvm/evm/offchain/storage/readonly.go b/fvm/evm/offchain/storage/readonly.go index 730c8670dcb..6c66e7c1e43 100644 --- a/fvm/evm/offchain/storage/readonly.go +++ b/fvm/evm/offchain/storage/readonly.go @@ -29,7 +29,7 @@ func (s *ReadOnlyStorage) GetValue(owner []byte, key []byte) ([]byte, error) { // SetValue returns an error if called func (s *ReadOnlyStorage) SetValue(owner, key, value []byte) error { - panic(fmt.Sprintf("unexpected call received for SetValue with owner: %x, key: %v, value: %x", owner, key, value)) + return fmt.Errorf("unexpected call received for SetValue with owner: %x, key: %v, value: %x", owner, key, value) } // ValueExists checks if a register exists @@ -40,5 +40,5 @@ func (s *ReadOnlyStorage) ValueExists(owner []byte, key []byte) (bool, error) { // AllocateSlabIndex returns an error if called func (s *ReadOnlyStorage) AllocateSlabIndex(owner []byte) (atree.SlabIndex, error) { - panic(fmt.Errorf("unexpected call received for AllocateSlabIndex with owner: %x", owner)) + return atree.SlabIndex{}, fmt.Errorf("unexpected call received for AllocateSlabIndex with owner: %x", owner) } From e7f94c538f0a9cf2ebdc6d4b473a973ae49fd6ed Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Mon, 2 Dec 2024 14:44:06 -0800 Subject: [PATCH 42/45] apply review comments --- fvm/evm/offchain/blocks/block_context.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/fvm/evm/offchain/blocks/block_context.go b/fvm/evm/offchain/blocks/block_context.go index 669f2dda998..ecbc8813c76 100644 --- a/fvm/evm/offchain/blocks/block_context.go +++ b/fvm/evm/offchain/blocks/block_context.go @@ -83,11 +83,6 @@ const blockHashListFixHCUEVMHeightMainnet = 8357079 // PR: https://github.com/onflow/flow-go/pull/6734 const blockHashListFixHCUEVMHeightTestnet = 16848829 -// Testnet52 - Spork -// Flow Block: 218215350 cc7188f0bdac4c442cc3ee072557d7f7c8ca4462537da945b148d5d0efa7a1ff -// PR: https://github.com/onflow/flow-go/pull/6377 -// const blockHashListBugIntroducedHCUEVMHeightTestnet = 7038679 - // Testnet51 - Height Coordinated Upgrade 1 // Flow Block: 212562161 1a520608c5457f228405c4c30fc39c8a0af7cf915fb2ede7ec5ccffc2a000f57 // PR: https://github.com/onflow/flow-go/pull/6380 From 4d419c4ab515af55e4f0c33899dca934a62afc38 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Thu, 5 Dec 2024 08:01:00 -0800 Subject: [PATCH 43/45] disable tests --- fvm/evm/emulator/state/diff_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fvm/evm/emulator/state/diff_test.go b/fvm/evm/emulator/state/diff_test.go index 057c4314fa4..4abb6868795 100644 --- a/fvm/evm/emulator/state/diff_test.go +++ b/fvm/evm/emulator/state/diff_test.go @@ -13,7 +13,7 @@ import ( "github.com/onflow/flow-go/model/flow" ) -func TestStateDiff(t *testing.T) { +func StateDiff(t *testing.T) { offchainState, err := state.ImportEVMStateFromGob("/var/flow2/evm-state-from-gobs-218215348/") require.NoError(t, err) @@ -25,7 +25,7 @@ func TestStateDiff(t *testing.T) { require.Len(t, differences, 0) } -func TestEVMStateDiff(t *testing.T) { +func EVMStateDiff(t *testing.T) { state1 := EVMStateFromReplayGobDir(t, "/var/flow2/evm-state-from-gobs-218215348/", uint64(218215348)) // state2 := EVMStateFromReplayGobDir(t, "/var/flow2/evm-state-from-gobs-218215348/", uint64(218215348)) From 2d4bb1065751a7629e037d389a4d988368eec1bb Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Thu, 5 Dec 2024 08:11:14 -0800 Subject: [PATCH 44/45] fix error handling --- fvm/evm/emulator/state/base.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/fvm/evm/emulator/state/base.go b/fvm/evm/emulator/state/base.go index 97df2076957..0f690b7367a 100644 --- a/fvm/evm/emulator/state/base.go +++ b/fvm/evm/emulator/state/base.go @@ -485,7 +485,10 @@ func (v *BaseView) fetchOrCreateCollection(path string) (collection *Collection, } if len(collectionID) == 0 { collection, err = v.collectionProvider.NewCollection() - return collection, true, fmt.Errorf("fail to create collection with key %v: %w", path, err) + if err != nil { + return collection, true, fmt.Errorf("fail to create collection with key %v: %w", path, err) + } + return collection, true, nil } collection, err = v.collectionProvider.CollectionByID(collectionID) return collection, false, err From 0c778a7c1d69b441d4a711b7ba14a6f61e978212 Mon Sep 17 00:00:00 2001 From: sjonpaulbrown Date: Fri, 6 Dec 2024 14:39:31 -0700 Subject: [PATCH 45/45] Update Makefile to support private util builds --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 36075edd094..32feb2a5184 100644 --- a/Makefile +++ b/Makefile @@ -859,6 +859,7 @@ docker-all-tools: tool-util tool-remove-execution-fork PHONY: docker-build-util docker-build-util: docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/util --build-arg GOARCH=$(GOARCH) --build-arg VERSION=$(IMAGE_TAG) --build-arg CGO_FLAG=$(DISABLE_ADX) --target production \ + --secret id=cadence_deploy_key,env=CADENCE_DEPLOY_KEY --build-arg GOPRIVATE=$(GOPRIVATE) \ -t "$(CONTAINER_REGISTRY)/util:latest" \ -t "$(CONTAINER_REGISTRY)/util:$(IMAGE_TAG)" .