diff --git a/cmd/util/cmd/export-evm-state/cmd.go b/cmd/util/cmd/export-evm-state/cmd.go index 2927b9a313a..c29ac6c1436 100644 --- a/cmd/util/cmd/export-evm-state/cmd.go +++ b/cmd/util/cmd/export-evm-state/cmd.go @@ -3,13 +3,17 @@ package evm_exporter import ( "fmt" "os" + "path/filepath" "github.com/rs/zerolog/log" "github.com/spf13/cobra" + "github.com/onflow/atree" + "github.com/onflow/flow-go/cmd/util/ledger/util" "github.com/onflow/flow-go/fvm/evm" "github.com/onflow/flow-go/fvm/evm/emulator/state" + "github.com/onflow/flow-go/fvm/evm/testutils" "github.com/onflow/flow-go/ledger" "github.com/onflow/flow-go/ledger/common/convert" "github.com/onflow/flow-go/model/flow" @@ -20,6 +24,8 @@ var ( flagExecutionStateDir string flagOutputDir string flagStateCommitment string + flagEVMStateGobDir string + flagEVMStateGobHeight uint64 ) var Cmd = &cobra.Command{ @@ -34,7 +40,6 @@ func init() { Cmd.Flags().StringVar(&flagExecutionStateDir, "execution-state-dir", "", "Execution Node state dir (where WAL logs are written") - _ = Cmd.MarkFlagRequired("execution-state-dir") Cmd.Flags().StringVar(&flagOutputDir, "output-dir", "", "Directory to write new Execution State to") @@ -42,13 +47,26 @@ func init() { Cmd.Flags().StringVar(&flagStateCommitment, "state-commitment", "", "State commitment (hex-encoded, 64 characters)") + + Cmd.Flags().StringVar(&flagEVMStateGobDir, "evm_state_gob_dir", "/var/flow/data/evm_state_gob", + "directory that stores the evm state gob files as checkpoint") + + Cmd.Flags().Uint64Var(&flagEVMStateGobHeight, "evm_state_gob_height", 0, + "the flow height of the evm state gob files") } func run(*cobra.Command, []string) { log.Info().Msg("start exporting evm state") - err := ExportEVMState(flagChain, flagExecutionStateDir, flagStateCommitment, flagOutputDir) - if err != nil { - log.Fatal().Err(err).Msg("cannot get export evm state") + if flagExecutionStateDir != "" { + err := ExportEVMState(flagChain, flagExecutionStateDir, flagStateCommitment, flagOutputDir) + if err != nil { + log.Fatal().Err(err).Msg("cannot get export evm state") + } + } else if flagEVMStateGobDir != "" { + err := ExportEVMStateFromGob(flagChain, flagEVMStateGobDir, flagEVMStateGobHeight, flagOutputDir) + if err != nil { + log.Fatal().Err(err).Msg("cannot get export evm state from gob files") + } } } @@ -83,7 +101,40 @@ func ExportEVMState( payloadsLedger := util.NewPayloadsLedger(filteredPayloads) - exporter, err := state.NewExporter(payloadsLedger, storageRoot) + return ExportEVMStateFromPayloads(payloadsLedger, storageRoot, outputPath) +} + +func ExportEVMStateFromGob( + chainName string, + evmStateGobDir string, + flowHeight uint64, + outputPath string) error { + + valueFileName, allocatorFileName := evmStateGobFileNamesByEndHeight(evmStateGobDir, flowHeight) + chainID := flow.ChainID(chainName) + + storageRoot := evm.StorageAccountAddress(chainID) + valuesGob, err := testutils.DeserializeState(valueFileName) + if err != nil { + return err + } + + allocatorGobs, err := testutils.DeserializeAllocator(allocatorFileName) + if err != nil { + return err + } + + store := testutils.GetSimpleValueStorePopulated(valuesGob, allocatorGobs) + + return ExportEVMStateFromPayloads(store, storageRoot, outputPath) +} + +func ExportEVMStateFromPayloads( + ledger atree.Ledger, + storageRoot flow.Address, + outputPath string, +) error { + exporter, err := state.NewExporter(ledger, storageRoot) if err != nil { return fmt.Errorf("failed to create exporter: %w", err) } @@ -95,15 +146,15 @@ func ExportEVMState( } } - fi, err := os.Create(outputPath) - if err != nil { - return err - } - defer fi.Close() - - err = exporter.Export(outputPath) + err = exporter.ExportGob(outputPath) if err != nil { return fmt.Errorf("failed to export: %w", err) } return nil } + +func evmStateGobFileNamesByEndHeight(evmStateGobDir string, endHeight uint64) (string, string) { + valueFileName := filepath.Join(evmStateGobDir, fmt.Sprintf("values-%d.gob", endHeight)) + allocatorFileName := filepath.Join(evmStateGobDir, fmt.Sprintf("allocators-%d.gob", endHeight)) + return valueFileName, allocatorFileName +} diff --git a/fvm/evm/emulator/state/base.go b/fvm/evm/emulator/state/base.go index 9f11ce6e3f0..0f690b7367a 100644 --- a/fvm/evm/emulator/state/base.go +++ b/fvm/evm/emulator/state/base.go @@ -74,13 +74,13 @@ func NewBaseView(ledger atree.Ledger, rootAddress flow.Address) (*BaseView, erro // fetch the account collection, if not exist, create one view.accounts, view.accountSetupOnCommit, err = view.fetchOrCreateCollection(AccountsStorageIDKey) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to fetch or create account collection with key %v: %w", AccountsStorageIDKey, err) } // fetch the code collection, if not exist, create one view.codes, view.codeSetupOnCommit, err = view.fetchOrCreateCollection(CodesStorageIDKey) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to fetch or create code collection with key %v: %w", CodesStorageIDKey, err) } return view, nil @@ -485,7 +485,10 @@ func (v *BaseView) fetchOrCreateCollection(path string) (collection *Collection, } if len(collectionID) == 0 { collection, err = v.collectionProvider.NewCollection() - return collection, true, err + if err != nil { + return collection, true, fmt.Errorf("fail to create collection with key %v: %w", path, err) + } + return collection, true, nil } collection, err = v.collectionProvider.CollectionByID(collectionID) return collection, false, err diff --git a/fvm/evm/emulator/state/diff.go b/fvm/evm/emulator/state/diff.go new file mode 100644 index 00000000000..bae539bd5db --- /dev/null +++ b/fvm/evm/emulator/state/diff.go @@ -0,0 +1,91 @@ +package state + +import ( + "bytes" + "fmt" +) + +func AccountEqual(a, b *Account) bool { + if a.Address != b.Address { + return false + } + if !bytes.Equal(a.Balance.Bytes(), b.Balance.Bytes()) { + return false + } + if a.Nonce != b.Nonce { + return false + } + if a.CodeHash != b.CodeHash { + return false + } + + // CollectionID could be different + return true +} + +// find the difference and return as error +func Diff(a *EVMState, b *EVMState) []error { + var differences []error + + // Compare Accounts + for addr, accA := range a.Accounts { + if accB, exists := b.Accounts[addr]; exists { + if !AccountEqual(accA, accB) { + differences = append(differences, fmt.Errorf("account %s differs, accA %v, accB %v", addr.Hex(), accA, accB)) + } + } else { + differences = append(differences, fmt.Errorf("account %s exists in a but not in b", addr.Hex())) + } + } + for addr := range b.Accounts { + if _, exists := a.Accounts[addr]; !exists { + differences = append(differences, fmt.Errorf("account %s exists in b but not in a", addr.Hex())) + } + } + + // Compare Slots + for addr, slotsA := range a.Slots { + slotsB, exists := b.Slots[addr] + if !exists { + differences = append(differences, fmt.Errorf("slots for address %s exist in a but not in b", addr.Hex())) + continue + } + for key, valueA := range slotsA { + if valueB, exists := slotsB[key]; exists { + if valueA.Value != valueB.Value { + differences = append(differences, fmt.Errorf("slot value for address %s and key %s differs", addr.Hex(), key.Hex())) + } + } else { + differences = append(differences, fmt.Errorf("slot with key %s for address %s exists in a but not in b", key.Hex(), addr.Hex())) + } + } + for key := range slotsB { + if _, exists := slotsA[key]; !exists { + differences = append(differences, fmt.Errorf("slot with key %s for address %s exists in b but not in a", key.Hex(), addr.Hex())) + } + } + } + for addr := range b.Slots { + if _, exists := a.Slots[addr]; !exists { + differences = append(differences, fmt.Errorf("slots for address %s exist in b but not in a", addr.Hex())) + } + } + + // Compare Codes + for hash, codeA := range a.Codes { + if codeB, exists := b.Codes[hash]; exists { + if !bytes.Equal(codeA.Code, codeB.Code) { + differences = append(differences, fmt.Errorf("code for hash %s differs", hash.Hex())) + } + } else { + differences = append(differences, fmt.Errorf("code with hash %s exists in a but not in b", hash.Hex())) + } + } + for hash := range b.Codes { + if _, exists := a.Codes[hash]; !exists { + differences = append(differences, fmt.Errorf("code with hash %s exists in b but not in a", hash.Hex())) + } + } + + return differences +} diff --git a/fvm/evm/emulator/state/diff_test.go b/fvm/evm/emulator/state/diff_test.go new file mode 100644 index 00000000000..4abb6868795 --- /dev/null +++ b/fvm/evm/emulator/state/diff_test.go @@ -0,0 +1,74 @@ +package state_test + +import ( + "fmt" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/fvm/evm" + "github.com/onflow/flow-go/fvm/evm/emulator/state" + "github.com/onflow/flow-go/fvm/evm/testutils" + "github.com/onflow/flow-go/model/flow" +) + +func StateDiff(t *testing.T) { + offchainState, err := state.ImportEVMStateFromGob("/var/flow2/evm-state-from-gobs-218215348/") + require.NoError(t, err) + + enState, err := state.ImportEVMStateFromGob("/var/flow2/evm-state-from-gobs-218215348/") + require.NoError(t, err) + + differences := state.Diff(enState, offchainState) + + require.Len(t, differences, 0) +} + +func EVMStateDiff(t *testing.T) { + + state1 := EVMStateFromReplayGobDir(t, "/var/flow2/evm-state-from-gobs-218215348/", uint64(218215348)) + // state2 := EVMStateFromReplayGobDir(t, "/var/flow2/evm-state-from-gobs-218215348/", uint64(218215348)) + state2 := EVMStateFromCheckpointExtract(t, "/var/flow2/evm-state-from-checkpoint-218215348/") + + differences := state.Diff(state1, state2) + + for i, diff := range differences { + fmt.Printf("Difference %d: %v\n", i, diff) + } + + require.Len(t, differences, 0) +} + +func EVMStateFromCheckpointExtract(t *testing.T, dir string) *state.EVMState { + enState, err := state.ImportEVMStateFromGob("/var/flow2/evm-state-from-gobs-218215348/") + require.NoError(t, err) + return enState +} + +func EVMStateFromReplayGobDir(t *testing.T, gobDir string, flowHeight uint64) *state.EVMState { + valueFileName, allocatorFileName := evmStateGobFileNamesByEndHeight(gobDir, flowHeight) + chainID := flow.Testnet + + allocatorGobs, err := testutils.DeserializeAllocator(allocatorFileName) + require.NoError(t, err) + + storageRoot := evm.StorageAccountAddress(chainID) + valuesGob, err := testutils.DeserializeState(valueFileName) + require.NoError(t, err) + + store := testutils.GetSimpleValueStorePopulated(valuesGob, allocatorGobs) + + bv, err := state.NewBaseView(store, storageRoot) + require.NoError(t, err) + + evmState, err := state.Extract(storageRoot, bv) + require.NoError(t, err) + return evmState +} + +func evmStateGobFileNamesByEndHeight(evmStateGobDir string, endHeight uint64) (string, string) { + valueFileName := filepath.Join(evmStateGobDir, fmt.Sprintf("values-%d.gob", endHeight)) + allocatorFileName := filepath.Join(evmStateGobDir, fmt.Sprintf("allocators-%d.gob", endHeight)) + return valueFileName, allocatorFileName +} diff --git a/fvm/evm/emulator/state/exporter.go b/fvm/evm/emulator/state/exporter.go index 49f3a0fdbd8..f1cb9bcfa10 100644 --- a/fvm/evm/emulator/state/exporter.go +++ b/fvm/evm/emulator/state/exporter.go @@ -1,6 +1,8 @@ package state import ( + "encoding/gob" + "fmt" "io" "os" "path/filepath" @@ -8,6 +10,7 @@ import ( "github.com/onflow/atree" gethCommon "github.com/onflow/go-ethereum/common" + "github.com/onflow/flow-go/fvm/evm/types" "github.com/onflow/flow-go/model/flow" ) @@ -15,6 +18,7 @@ const ( ExportedAccountsFileName = "accounts.bin" ExportedCodesFileName = "codes.bin" ExportedSlotsFileName = "slots.bin" + ExportedStateGobFileName = "state.gob" ) type Exporter struct { @@ -36,8 +40,32 @@ func NewExporter(ledger atree.Ledger, root flow.Address) (*Exporter, error) { }, nil } +func (e *Exporter) ExportGob(path string) error { + fileName := filepath.Join(path, ExportedStateGobFileName) + // Open the file for reading + file, err := os.Create(fileName) + if err != nil { + return err + } + defer file.Close() + + state, err := Extract(e.root, e.baseView) + if err != nil { + return err + } + + // Use gob to encode data + encoder := gob.NewEncoder(file) + err = encoder.Encode(state) + if err != nil { + return err + } + + return nil +} + func (e *Exporter) Export(path string) error { - af, err := os.OpenFile(filepath.Join(path, ExportedAccountsFileName), os.O_RDWR, 0644) + af, err := os.Create(filepath.Join(path, ExportedAccountsFileName)) if err != nil { return err } @@ -48,7 +76,7 @@ func (e *Exporter) Export(path string) error { return err } - cf, err := os.OpenFile(filepath.Join(path, ExportedCodesFileName), os.O_RDWR, 0644) + cf, err := os.Create(filepath.Join(path, ExportedCodesFileName)) if err != nil { return err } @@ -59,7 +87,7 @@ func (e *Exporter) Export(path string) error { return err } - sf, err := os.OpenFile(filepath.Join(path, ExportedSlotsFileName), os.O_RDWR, 0644) + sf, err := os.Create(filepath.Join(path, ExportedSlotsFileName)) if err != nil { return err } @@ -96,6 +124,12 @@ func (e *Exporter) exportAccounts(writer io.Writer) ([]gethCommon.Address, error if err != nil { return nil, err } + + _, err = DecodeAccount(encoded) + if err != nil { + return nil, fmt.Errorf("account can not be decoded: %w", err) + } + // write every account on a new line _, err = writer.Write(append(encoded, byte('\n'))) if err != nil { @@ -123,6 +157,12 @@ func (e *Exporter) exportCodes(writer io.Writer) error { if err != nil { return err } + + _, err = CodeInContextFromEncoded(encoded) + if err != nil { + return fmt.Errorf("error decoding code in context: %w", err) + } + // write every codes on a new line _, err = writer.Write(append(encoded, byte('\n'))) if err != nil { @@ -151,6 +191,12 @@ func (e *Exporter) exportSlots(addresses []gethCommon.Address, writer io.Writer) if err != nil { return err } + + _, err = types.SlotEntryFromEncoded(encoded) + if err != nil { + return fmt.Errorf("error decoding slot entry: %w", err) + } + // write every codes on a new line _, err = writer.Write(append(encoded, byte('\n'))) if err != nil { diff --git a/fvm/evm/emulator/state/extract.go b/fvm/evm/emulator/state/extract.go new file mode 100644 index 00000000000..e0bb30d82aa --- /dev/null +++ b/fvm/evm/emulator/state/extract.go @@ -0,0 +1,82 @@ +package state + +import ( + gethCommon "github.com/onflow/go-ethereum/common" + + "github.com/onflow/flow-go/fvm/evm/types" + "github.com/onflow/flow-go/model/flow" +) + +func Extract( + root flow.Address, + baseView *BaseView, +) (*EVMState, error) { + + accounts := make(map[gethCommon.Address]*Account, 0) + + itr, err := baseView.AccountIterator() + + if err != nil { + return nil, err + } + // make a list of accounts with storage + addrWithSlots := make([]gethCommon.Address, 0) + for { + // TODO: we can optimize by returning the encoded value + acc, err := itr.Next() + if err != nil { + return nil, err + } + if acc == nil { + break + } + if acc.HasStoredValues() { + addrWithSlots = append(addrWithSlots, acc.Address) + } + accounts[acc.Address] = acc + } + + codes := make(map[gethCommon.Hash]*CodeInContext, 0) + codeItr, err := baseView.CodeIterator() + if err != nil { + return nil, err + } + for { + cic, err := codeItr.Next() + if err != nil { + return nil, err + } + if cic == nil { + break + } + codes[cic.Hash] = cic + } + + // account address -> key -> value + slots := make(map[gethCommon.Address]map[gethCommon.Hash]*types.SlotEntry) + + for _, addr := range addrWithSlots { + slots[addr] = make(map[gethCommon.Hash]*types.SlotEntry) + slotItr, err := baseView.AccountStorageIterator(addr) + if err != nil { + return nil, err + } + for { + slot, err := slotItr.Next() + if err != nil { + return nil, err + } + if slot == nil { + break + } + + slots[addr][slot.Key] = slot + } + } + + return &EVMState{ + Accounts: accounts, + Codes: codes, + Slots: slots, + }, nil +} diff --git a/fvm/evm/emulator/state/importer.go b/fvm/evm/emulator/state/importer.go new file mode 100644 index 00000000000..132846512f4 --- /dev/null +++ b/fvm/evm/emulator/state/importer.go @@ -0,0 +1,137 @@ +package state + +import ( + "encoding/gob" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + + gethCommon "github.com/onflow/go-ethereum/common" + + "github.com/onflow/flow-go/fvm/evm/types" +) + +type EVMState struct { + Accounts map[gethCommon.Address]*Account + Codes map[gethCommon.Hash]*CodeInContext + // account address -> key -> value + Slots map[gethCommon.Address]map[gethCommon.Hash]*types.SlotEntry +} + +func ToEVMState( + accounts map[gethCommon.Address]*Account, + codes []*CodeInContext, + slots []*types.SlotEntry, +) (*EVMState, error) { + state := &EVMState{ + Accounts: accounts, + Codes: make(map[gethCommon.Hash]*CodeInContext), + Slots: make(map[gethCommon.Address]map[gethCommon.Hash]*types.SlotEntry), + } + + // Process codes + for _, code := range codes { + if _, ok := state.Codes[code.Hash]; ok { + return nil, fmt.Errorf("duplicate code hash: %s", code.Hash) + } + state.Codes[code.Hash] = code + } + + // Process slots + for _, slot := range slots { + if _, ok := state.Slots[slot.Address]; !ok { + state.Slots[slot.Address] = make(map[gethCommon.Hash]*types.SlotEntry) + } + + if _, ok := state.Slots[slot.Address][slot.Key]; ok { + return nil, fmt.Errorf("duplicate slot key: %s", slot.Key) + } + + state.Slots[slot.Address][slot.Key] = slot + } + + return state, nil +} + +func ImportEVMStateFromGob(path string) (*EVMState, error) { + fileName := filepath.Join(path, ExportedStateGobFileName) + // Open the file for reading + file, err := os.Open(fileName) + if err != nil { + return nil, err + } + defer file.Close() + + // Prepare the map to store decoded data + var data EVMState + + // Use gob to decode data + decoder := gob.NewDecoder(file) + err = decoder.Decode(&data) + if err != nil { + return nil, err + } + + return &data, nil +} + +func ImportEVMState(path string) (*EVMState, error) { + accounts := make(map[gethCommon.Address]*Account) + var codes []*CodeInContext + var slots []*types.SlotEntry + // Import codes + codesData, err := ioutil.ReadFile(filepath.Join(path, ExportedCodesFileName)) + if err != nil { + return nil, fmt.Errorf("error opening codes file: %w", err) + } + codesLines := strings.Split(string(codesData), "\n") + for _, line := range codesLines { + if line == "" { + continue + } + code, err := CodeInContextFromEncoded([]byte(line)) + if err != nil { + return nil, fmt.Errorf("error decoding code in context: %w", err) + } + codes = append(codes, code) + } + + // Import slots + slotsData, err := ioutil.ReadFile(filepath.Join(path, ExportedSlotsFileName)) + if err != nil { + return nil, fmt.Errorf("error opening slots file: %w", err) + } + slotsLines := strings.Split(string(slotsData), "\n") + for _, line := range slotsLines { + if line == "" { + continue + } + slot, err := types.SlotEntryFromEncoded([]byte(line)) + if err != nil { + return nil, fmt.Errorf("error decoding slot entry: %w", err) + } + slots = append(slots, slot) + } + + // Import accounts + accountsData, err := ioutil.ReadFile(filepath.Join(path, ExportedAccountsFileName)) + if err != nil { + return nil, fmt.Errorf("error opening accounts file: %w", err) + } + accountsLines := strings.Split(string(accountsData), "\n") + for _, line := range accountsLines { + if line == "" { + continue + } + acc, err := DecodeAccount([]byte(line)) + if err != nil { + fmt.Println("error decoding account: ", err, line) + } else { + fmt.Println("decoded account", acc.Address) + accounts[acc.Address] = acc + } + } + return ToEVMState(accounts, codes, slots) +} diff --git a/fvm/evm/offchain/storage/readonly.go b/fvm/evm/offchain/storage/readonly.go index 4ed33a6fe44..6c66e7c1e43 100644 --- a/fvm/evm/offchain/storage/readonly.go +++ b/fvm/evm/offchain/storage/readonly.go @@ -1,7 +1,7 @@ package storage import ( - "errors" + "fmt" "github.com/onflow/atree" @@ -29,7 +29,7 @@ func (s *ReadOnlyStorage) GetValue(owner []byte, key []byte) ([]byte, error) { // SetValue returns an error if called func (s *ReadOnlyStorage) SetValue(owner, key, value []byte) error { - return errors.New("unexpected call received") + return fmt.Errorf("unexpected call received for SetValue with owner: %x, key: %v, value: %x", owner, key, value) } // ValueExists checks if a register exists @@ -40,5 +40,5 @@ func (s *ReadOnlyStorage) ValueExists(owner []byte, key []byte) (bool, error) { // AllocateSlabIndex returns an error if called func (s *ReadOnlyStorage) AllocateSlabIndex(owner []byte) (atree.SlabIndex, error) { - return atree.SlabIndex{}, errors.New("unexpected call received") + return atree.SlabIndex{}, fmt.Errorf("unexpected call received for AllocateSlabIndex with owner: %x", owner) }