From 74f41ac3e970a21643482e3ce3d3d794bb9e5048 Mon Sep 17 00:00:00 2001 From: Jared Wasinger Date: Mon, 16 Sep 2024 16:40:14 +0700 Subject: [PATCH 01/12] state test benchmarking --- cmd/evm/staterunner.go | 130 +++++++++++++++++++++++++++++++++-------- 1 file changed, 105 insertions(+), 25 deletions(-) diff --git a/cmd/evm/staterunner.go b/cmd/evm/staterunner.go index 4514367e8a45..d5cb1b6056d5 100644 --- a/cmd/evm/staterunner.go +++ b/cmd/evm/staterunner.go @@ -20,7 +20,10 @@ import ( "bufio" "encoding/json" "fmt" + "github.com/ethereum/go-ethereum/internal/flags" "os" + "testing" + "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" @@ -31,11 +34,33 @@ import ( "github.com/urfave/cli/v2" ) +var ( + ForkFlag = &cli.StringFlag{ + Name: "subtest.fork", + Usage: "The hard-fork to run the test against", + Category: flags.VMCategory, + } + IdxFlag = &cli.IntFlag{ + Name: "subtest.index", + Usage: "The index of the subtest to run", + Category: flags.VMCategory, + } + TestNameFlag = &cli.StringFlag{ + Name: "subtest.name", + Usage: "The hard-fork to run the test against", + Category: flags.VMCategory, + } +) var stateTestCommand = &cli.Command{ Action: stateTestCmd, Name: "statetest", Usage: "Executes the given state tests. Filenames can be fed via standard input (batch mode) or as an argument (one-off execution).", ArgsUsage: "", + Flags: []cli.Flag{ + ForkFlag, + IdxFlag, + TestNameFlag, + }, } // StatetestResult contains the execution status after running a state test, any @@ -67,7 +92,7 @@ func stateTestCmd(ctx *cli.Context) error { } // Load the test content from the input file if len(ctx.Args().First()) != 0 { - return runStateTest(ctx.Args().First(), cfg, ctx.Bool(DumpFlag.Name)) + return runStateTest(ctx, ctx.Args().First(), cfg, ctx.Bool(DumpFlag.Name), ctx.Bool(BenchFlag.Name)) } // Read filenames from stdin and execute back-to-back scanner := bufio.NewScanner(os.Stdin) @@ -76,15 +101,48 @@ func stateTestCmd(ctx *cli.Context) error { if len(fname) == 0 { return nil } - if err := runStateTest(fname, cfg, ctx.Bool(DumpFlag.Name)); err != nil { + if err := runStateTest(ctx, fname, cfg, ctx.Bool(DumpFlag.Name), ctx.Bool(BenchFlag.Name)); err != nil { return err } } return nil } +type stateTestCase struct { + name string + test tests.StateTest + st tests.StateSubtest +} + +func aggregateSubtests(ctx *cli.Context, testsByName map[string]tests.StateTest) []stateTestCase { + res := []stateTestCase{} + + subtestName := ctx.String(TestNameFlag.Name) + if subtestName != "" { + if subtest, ok := testsByName[subtestName]; ok { + testsByName := make(map[string]tests.StateTest) + testsByName[subtestName] = subtest + } + } + idx := ctx.Int(IdxFlag.Name) + fork := ctx.String(ForkFlag.Name) + + for key, test := range testsByName { + for _, st := range test.Subtests() { + if idx != -1 && st.Index != idx { + continue + } + if fork != "" && st.Fork != fork { + continue + } + res = append(res, stateTestCase{name: key, st: st, test: test}) + } + } + return res +} + // runStateTest loads the state-test given by fname, and executes the test. -func runStateTest(fname string, cfg vm.Config, dump bool) error { +func runStateTest(ctx *cli.Context, fname string, cfg vm.Config, dump bool, bench bool) error { src, err := os.ReadFile(fname) if err != nil { return err @@ -94,33 +152,55 @@ func runStateTest(fname string, cfg vm.Config, dump bool) error { return err } + matchingTests := aggregateSubtests(ctx, testsByName) + // Iterate over all the tests, run them and aggregate the results results := make([]StatetestResult, 0, len(testsByName)) - for key, test := range testsByName { - for _, st := range test.Subtests() { - // Run the test and aggregate the result - result := &StatetestResult{Name: key, Fork: st.Fork, Pass: true} - test.Run(st, cfg, false, rawdb.HashScheme, func(err error, tstate *tests.StateTestState) { - var root common.Hash - if tstate.StateDB != nil { - root = tstate.StateDB.IntermediateRoot(false) - result.Root = &root - fmt.Fprintf(os.Stderr, "{\"stateRoot\": \"%#x\"}\n", root) - if dump { // Dump any state to aid debugging - cpy, _ := state.New(root, tstate.StateDB.Database()) - dump := cpy.RawDump(nil) - result.State = &dump - } - } - if err != nil { - // Test failed, mark as so - result.Pass, result.Error = false, err.Error() + for _, test := range matchingTests { + // Run the test and aggregate the result + result := &StatetestResult{Name: test.name, Fork: test.st.Fork, Pass: true} + test.test.Run(test.st, cfg, false, rawdb.HashScheme, func(err error, tstate *tests.StateTestState) { + var root common.Hash + if tstate.StateDB != nil { + root = tstate.StateDB.IntermediateRoot(false) + result.Root = &root + fmt.Fprintf(os.Stderr, "{\"stateRoot\": \"%#x\"}\n", root) + if dump { // Dump any state to aid debugging + cpy, _ := state.New(root, tstate.StateDB.Database()) + dump := cpy.RawDump(nil) + result.State = &dump } - }) - results = append(results, *result) - } + } + if err != nil { + // Test failed, mark as so + result.Pass, result.Error = false, err.Error() + } + }) + results = append(results, *result) } out, _ := json.MarshalIndent(results, "", " ") fmt.Println(string(out)) + + if !bench { + return nil + } + result := testing.Benchmark(func(b *testing.B) { + for i := 0; i < b.N; i++ { + for _, test := range matchingTests { + test.test.Run(test.st, cfg, false, rawdb.HashScheme, func(err error, tstate *tests.StateTestState) {}) + } + } + }) + var stats execStats + // Get the average execution time from the benchmarking result. + // There are other useful stats here that could be reported. + stats.time = time.Duration(result.NsPerOp()) + stats.allocs = result.AllocsPerOp() + stats.bytesAllocated = result.AllocedBytesPerOp() + fmt.Fprintf(os.Stderr, `EVM gas used: %d +execution time: %v +allocations: %d +allocated bytes: %d +`, 0, stats.time, stats.allocs, stats.bytesAllocated) return nil } From 46040cc5444e4d0f7fb6bcda68b98882123d664d Mon Sep 17 00:00:00 2001 From: Jared Wasinger Date: Mon, 16 Sep 2024 17:24:08 +0700 Subject: [PATCH 02/12] add used gas --- cmd/evm/staterunner.go | 7 +++++-- tests/state_test_util.go | 16 ++++++++-------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/cmd/evm/staterunner.go b/cmd/evm/staterunner.go index d5cb1b6056d5..24ecf5574f9d 100644 --- a/cmd/evm/staterunner.go +++ b/cmd/evm/staterunner.go @@ -183,11 +183,14 @@ func runStateTest(ctx *cli.Context, fname string, cfg vm.Config, dump bool, benc if !bench { return nil + } else if len(matchingTests) != 1 { + return fmt.Errorf("can only benchmark single state test case (more than one matching params)") } + var gasUsed uint64 result := testing.Benchmark(func(b *testing.B) { for i := 0; i < b.N; i++ { for _, test := range matchingTests { - test.test.Run(test.st, cfg, false, rawdb.HashScheme, func(err error, tstate *tests.StateTestState) {}) + _, _, gasUsed, _ = test.test.RunNoVerify(test.st, cfg, false, rawdb.HashScheme) } } }) @@ -201,6 +204,6 @@ func runStateTest(ctx *cli.Context, fname string, cfg vm.Config, dump bool, benc execution time: %v allocations: %d allocated bytes: %d -`, 0, stats.time, stats.allocs, stats.bytesAllocated) +`, gasUsed, stats.time, stats.allocs, stats.bytesAllocated) return nil } diff --git a/tests/state_test_util.go b/tests/state_test_util.go index cf0ce9777f67..e6d180ab3b51 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -196,7 +196,7 @@ func (t *StateTest) checkError(subtest StateSubtest, err error) error { // Run executes a specific subtest and verifies the post-state and logs func (t *StateTest) Run(subtest StateSubtest, vmconfig vm.Config, snapshotter bool, scheme string, postCheck func(err error, st *StateTestState)) (result error) { - st, root, err := t.RunNoVerify(subtest, vmconfig, snapshotter, scheme) + st, root, _, err := t.RunNoVerify(subtest, vmconfig, snapshotter, scheme) // Invoke the callback at the end of function for further analysis. defer func() { postCheck(result, &st) @@ -228,10 +228,10 @@ func (t *StateTest) Run(subtest StateSubtest, vmconfig vm.Config, snapshotter bo // RunNoVerify runs a specific subtest and returns the statedb and post-state root. // Remember to call state.Close after verifying the test result! -func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapshotter bool, scheme string) (st StateTestState, root common.Hash, err error) { +func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapshotter bool, scheme string) (st StateTestState, root common.Hash, gasUsed uint64, err error) { config, eips, err := GetChainConfig(subtest.Fork) if err != nil { - return st, common.Hash{}, UnsupportedForkError{subtest.Fork} + return st, common.Hash{}, 0, UnsupportedForkError{subtest.Fork} } vmconfig.ExtraEips = eips @@ -250,7 +250,7 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh post := t.json.Post[subtest.Fork][subtest.Index] msg, err := t.json.Tx.toMessage(post, baseFee) if err != nil { - return st, common.Hash{}, err + return st, common.Hash{}, 0, err } { // Blob transactions may be present after the Cancun fork. @@ -260,7 +260,7 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh // Here, we just do this shortcut smaller fix, since state tests do not // utilize those codepaths if len(msg.BlobHashes)*params.BlobTxBlobGasPerBlob > params.MaxBlobGasPerBlock { - return st, common.Hash{}, errors.New("blob gas exceeds maximum") + return st, common.Hash{}, 0, errors.New("blob gas exceeds maximum") } } @@ -269,10 +269,10 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh var ttx types.Transaction err := ttx.UnmarshalBinary(post.TxBytes) if err != nil { - return st, common.Hash{}, err + return st, common.Hash{}, 0, err } if _, err := types.Sender(types.LatestSigner(config), &ttx); err != nil { - return st, common.Hash{}, err + return st, common.Hash{}, 0, err } } @@ -322,7 +322,7 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh receipt := &types.Receipt{GasUsed: vmRet.UsedGas} tracer.OnTxEnd(receipt, nil) } - return st, root, err + return st, root, vmRet.UsedGas, err } func (t *StateTest) gasLimit(subtest StateSubtest) uint64 { From 96fbf667c20198712f848c2eb2c849c53a0e704e Mon Sep 17 00:00:00 2001 From: Jared Wasinger Date: Mon, 16 Sep 2024 17:53:02 +0700 Subject: [PATCH 03/12] some docs, rename helper func, specify default value for indices (glob all) --- cmd/evm/staterunner.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/cmd/evm/staterunner.go b/cmd/evm/staterunner.go index 24ecf5574f9d..2f547992f1b4 100644 --- a/cmd/evm/staterunner.go +++ b/cmd/evm/staterunner.go @@ -44,6 +44,7 @@ var ( Name: "subtest.index", Usage: "The index of the subtest to run", Category: flags.VMCategory, + Value: -1, // default to select all subtest indices } TestNameFlag = &cli.StringFlag{ Name: "subtest.name", @@ -114,7 +115,8 @@ type stateTestCase struct { st tests.StateSubtest } -func aggregateSubtests(ctx *cli.Context, testsByName map[string]tests.StateTest) []stateTestCase { +// collectMatchedSubtests returns test cases which match against provided filtering CLI parameters +func collectMatchedSubtests(ctx *cli.Context, testsByName map[string]tests.StateTest) []stateTestCase { res := []stateTestCase{} subtestName := ctx.String(TestNameFlag.Name) @@ -152,7 +154,7 @@ func runStateTest(ctx *cli.Context, fname string, cfg vm.Config, dump bool, benc return err } - matchingTests := aggregateSubtests(ctx, testsByName) + matchingTests := collectMatchedSubtests(ctx, testsByName) // Iterate over all the tests, run them and aggregate the results results := make([]StatetestResult, 0, len(testsByName)) @@ -189,9 +191,8 @@ func runStateTest(ctx *cli.Context, fname string, cfg vm.Config, dump bool, benc var gasUsed uint64 result := testing.Benchmark(func(b *testing.B) { for i := 0; i < b.N; i++ { - for _, test := range matchingTests { - _, _, gasUsed, _ = test.test.RunNoVerify(test.st, cfg, false, rawdb.HashScheme) - } + test := matchingTests[0] + _, _, gasUsed, _ = test.test.RunNoVerify(test.st, cfg, false, rawdb.HashScheme) } }) var stats execStats From 3060ae1c2ffad65da8571aefd514456f306f30f9 Mon Sep 17 00:00:00 2001 From: Jared Wasinger Date: Tue, 24 Sep 2024 17:40:49 +0700 Subject: [PATCH 04/12] make requested changes --- cmd/evm/staterunner.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/cmd/evm/staterunner.go b/cmd/evm/staterunner.go index 2f547992f1b4..fb036222ff9d 100644 --- a/cmd/evm/staterunner.go +++ b/cmd/evm/staterunner.go @@ -20,11 +20,12 @@ import ( "bufio" "encoding/json" "fmt" - "github.com/ethereum/go-ethereum/internal/flags" "os" "testing" "time" + "github.com/ethereum/go-ethereum/internal/flags" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" @@ -36,19 +37,19 @@ import ( var ( ForkFlag = &cli.StringFlag{ - Name: "subtest.fork", + Name: "statetest.fork", Usage: "The hard-fork to run the test against", Category: flags.VMCategory, } IdxFlag = &cli.IntFlag{ - Name: "subtest.index", + Name: "statetest.index", Usage: "The index of the subtest to run", Category: flags.VMCategory, Value: -1, // default to select all subtest indices } TestNameFlag = &cli.StringFlag{ - Name: "subtest.name", - Usage: "The hard-fork to run the test against", + Name: "statetest.name", + Usage: "The name of the state test to run", Category: flags.VMCategory, } ) @@ -157,7 +158,7 @@ func runStateTest(ctx *cli.Context, fname string, cfg vm.Config, dump bool, benc matchingTests := collectMatchedSubtests(ctx, testsByName) // Iterate over all the tests, run them and aggregate the results - results := make([]StatetestResult, 0, len(testsByName)) + var results []StatetestResult for _, test := range matchingTests { // Run the test and aggregate the result result := &StatetestResult{Name: test.name, Fork: test.st.Fork, Pass: true} From 6b05f830834159ef4f9588f5ade486379fca38fe Mon Sep 17 00:00:00 2001 From: Jared Wasinger Date: Tue, 24 Sep 2024 17:43:18 +0700 Subject: [PATCH 05/12] small nit --- cmd/evm/staterunner.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cmd/evm/staterunner.go b/cmd/evm/staterunner.go index fb036222ff9d..29f31845fe23 100644 --- a/cmd/evm/staterunner.go +++ b/cmd/evm/staterunner.go @@ -118,8 +118,7 @@ type stateTestCase struct { // collectMatchedSubtests returns test cases which match against provided filtering CLI parameters func collectMatchedSubtests(ctx *cli.Context, testsByName map[string]tests.StateTest) []stateTestCase { - res := []stateTestCase{} - + var res []stateTestCase subtestName := ctx.String(TestNameFlag.Name) if subtestName != "" { if subtest, ok := testsByName[subtestName]; ok { From 64cbd00e86345600e6249e6e7642ff4c87c72044 Mon Sep 17 00:00:00 2001 From: Jared Wasinger Date: Tue, 24 Sep 2024 19:00:45 +0700 Subject: [PATCH 06/12] fix goimport --- cmd/evm/staterunner.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cmd/evm/staterunner.go b/cmd/evm/staterunner.go index 29f31845fe23..6d17b5f912bb 100644 --- a/cmd/evm/staterunner.go +++ b/cmd/evm/staterunner.go @@ -24,13 +24,12 @@ import ( "testing" "time" - "github.com/ethereum/go-ethereum/internal/flags" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth/tracers/logger" + "github.com/ethereum/go-ethereum/internal/flags" "github.com/ethereum/go-ethereum/tests" "github.com/urfave/cli/v2" ) From b8ed3deee77b6d68c6c9d8332a3861e8d2558351 Mon Sep 17 00:00:00 2001 From: Jared Wasinger Date: Mon, 30 Sep 2024 22:02:24 +0700 Subject: [PATCH 07/12] unexport evm state test bench command flags --- cmd/evm/staterunner.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/cmd/evm/staterunner.go b/cmd/evm/staterunner.go index 6d17b5f912bb..f1868e5b05f0 100644 --- a/cmd/evm/staterunner.go +++ b/cmd/evm/staterunner.go @@ -35,18 +35,18 @@ import ( ) var ( - ForkFlag = &cli.StringFlag{ + forkFlag = &cli.StringFlag{ Name: "statetest.fork", Usage: "The hard-fork to run the test against", Category: flags.VMCategory, } - IdxFlag = &cli.IntFlag{ + idxFlag = &cli.IntFlag{ Name: "statetest.index", Usage: "The index of the subtest to run", Category: flags.VMCategory, Value: -1, // default to select all subtest indices } - TestNameFlag = &cli.StringFlag{ + testNameFlag = &cli.StringFlag{ Name: "statetest.name", Usage: "The name of the state test to run", Category: flags.VMCategory, @@ -58,9 +58,9 @@ var stateTestCommand = &cli.Command{ Usage: "Executes the given state tests. Filenames can be fed via standard input (batch mode) or as an argument (one-off execution).", ArgsUsage: "", Flags: []cli.Flag{ - ForkFlag, - IdxFlag, - TestNameFlag, + forkFlag, + idxFlag, + testNameFlag, }, } @@ -118,15 +118,15 @@ type stateTestCase struct { // collectMatchedSubtests returns test cases which match against provided filtering CLI parameters func collectMatchedSubtests(ctx *cli.Context, testsByName map[string]tests.StateTest) []stateTestCase { var res []stateTestCase - subtestName := ctx.String(TestNameFlag.Name) + subtestName := ctx.String(testNameFlag.Name) if subtestName != "" { if subtest, ok := testsByName[subtestName]; ok { testsByName := make(map[string]tests.StateTest) testsByName[subtestName] = subtest } } - idx := ctx.Int(IdxFlag.Name) - fork := ctx.String(ForkFlag.Name) + idx := ctx.Int(idxFlag.Name) + fork := ctx.String(forkFlag.Name) for key, test := range testsByName { for _, st := range test.Subtests() { From 20256a147555b26e0504b75103fa4563765739fa Mon Sep 17 00:00:00 2001 From: Jared Wasinger Date: Tue, 1 Oct 2024 14:50:36 +0700 Subject: [PATCH 08/12] incorporate ExecStats into StateTestResult. add gas used to ExecStats --- cmd/evm/runner.go | 41 ++++++++++++++++++++++----------------- cmd/evm/staterunner.go | 44 +++++++++++++++--------------------------- 2 files changed, 39 insertions(+), 46 deletions(-) diff --git a/cmd/evm/runner.go b/cmd/evm/runner.go index 235fed66302a..98351dddbc4f 100644 --- a/cmd/evm/runner.go +++ b/cmd/evm/runner.go @@ -74,37 +74,41 @@ func readGenesis(genesisPath string) *core.Genesis { return genesis } -type execStats struct { - time time.Duration // The execution time. - allocs int64 // The number of heap allocations during execution. - bytesAllocated int64 // The cumulative number of bytes allocated during execution. +type ExecStats struct { + Time time.Duration `json:"time"` // The execution Time. + Allocs int64 `json:"allocs"` // The number of heap allocations during execution. + BytesAllocated int64 `json:"bytesAllocated"` // The cumulative number of bytes allocated during execution. + GasUsed uint64 `json:"gasUsed"` // the amount of gas used during execution } -func timedExec(bench bool, execFunc func() ([]byte, uint64, error)) (output []byte, gasLeft uint64, stats execStats, err error) { +func timedExec(bench bool, execFunc func() ([]byte, uint64, error)) (output []byte, stats ExecStats, err error) { + var gasUsed uint64 if bench { result := testing.Benchmark(func(b *testing.B) { for i := 0; i < b.N; i++ { - output, gasLeft, err = execFunc() + output, gasUsed, err = execFunc() } }) - // Get the average execution time from the benchmarking result. // There are other useful stats here that could be reported. - stats.time = time.Duration(result.NsPerOp()) - stats.allocs = result.AllocsPerOp() - stats.bytesAllocated = result.AllocedBytesPerOp() + stats.Time = time.Duration(result.NsPerOp()) + stats.Allocs = result.AllocsPerOp() + stats.BytesAllocated = result.AllocedBytesPerOp() + stats.GasUsed = gasUsed } else { + g var memStatsBefore, memStatsAfter goruntime.MemStats goruntime.ReadMemStats(&memStatsBefore) startTime := time.Now() - output, gasLeft, err = execFunc() - stats.time = time.Since(startTime) + output, gasUsed, err = execFunc() + stats.Time = time.Since(startTime) goruntime.ReadMemStats(&memStatsAfter) - stats.allocs = int64(memStatsAfter.Mallocs - memStatsBefore.Mallocs) - stats.bytesAllocated = int64(memStatsAfter.TotalAlloc - memStatsBefore.TotalAlloc) + stats.Allocs = int64(memStatsAfter.Mallocs - memStatsBefore.Mallocs) + stats.BytesAllocated = int64(memStatsAfter.TotalAlloc - memStatsBefore.TotalAlloc) + stats.GasUsed = gasUsed } - return output, gasLeft, stats, err + return output, stats, err } func runCmd(ctx *cli.Context) error { @@ -264,12 +268,13 @@ func runCmd(ctx *cli.Context) error { statedb.SetCode(receiver, code) } execFunc = func() ([]byte, uint64, error) { - return runtime.Call(receiver, input, &runtimeConfig) + output, gasLeft, err := runtime.Call(receiver, input, &runtimeConfig) + return output, initialGas - gasLeft, err } } bench := ctx.Bool(BenchFlag.Name) - output, leftOverGas, stats, err := timedExec(bench, execFunc) + output, stats, err := timedExec(bench, execFunc) if ctx.Bool(DumpFlag.Name) { root, err := statedb.Commit(genesisConfig.Number, true) @@ -299,7 +304,7 @@ func runCmd(ctx *cli.Context) error { execution time: %v allocations: %d allocated bytes: %d -`, initialGas-leftOverGas, stats.time, stats.allocs, stats.bytesAllocated) +`, stats.GasUsed, stats.Time, stats.Allocs, stats.BytesAllocated) } if tracer == nil { fmt.Printf("%#x\n", output) diff --git a/cmd/evm/staterunner.go b/cmd/evm/staterunner.go index f1868e5b05f0..76300c2e11e4 100644 --- a/cmd/evm/staterunner.go +++ b/cmd/evm/staterunner.go @@ -20,10 +20,6 @@ import ( "bufio" "encoding/json" "fmt" - "os" - "testing" - "time" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" @@ -32,6 +28,7 @@ import ( "github.com/ethereum/go-ethereum/internal/flags" "github.com/ethereum/go-ethereum/tests" "github.com/urfave/cli/v2" + "os" ) var ( @@ -67,12 +64,13 @@ var stateTestCommand = &cli.Command{ // StatetestResult contains the execution status after running a state test, any // error that might have occurred and a dump of the final state if requested. type StatetestResult struct { - Name string `json:"name"` - Pass bool `json:"pass"` - Root *common.Hash `json:"stateRoot,omitempty"` - Fork string `json:"fork"` - Error string `json:"error,omitempty"` - State *state.Dump `json:"state,omitempty"` + Name string `json:"name"` + Pass bool `json:"pass"` + Root *common.Hash `json:"stateRoot,omitempty"` + Fork string `json:"fork"` + Error string `json:"error,omitempty"` + State *state.Dump `json:"state,omitempty"` + BenchStats *ExecStats `json:"benchStats,omitempty"` } func stateTestCmd(ctx *cli.Context) error { @@ -177,6 +175,13 @@ func runStateTest(ctx *cli.Context, fname string, cfg vm.Config, dump bool, benc result.Pass, result.Error = false, err.Error() } }) + if bench { + _, stats, _ := timedExec(true, func() ([]byte, uint64, error) { + _, _, gasUsed, _ := test.test.RunNoVerify(test.st, cfg, false, rawdb.HashScheme) + return nil, gasUsed, nil + }) + result.BenchStats = &stats + } results = append(results, *result) } out, _ := json.MarshalIndent(results, "", " ") @@ -187,23 +192,6 @@ func runStateTest(ctx *cli.Context, fname string, cfg vm.Config, dump bool, benc } else if len(matchingTests) != 1 { return fmt.Errorf("can only benchmark single state test case (more than one matching params)") } - var gasUsed uint64 - result := testing.Benchmark(func(b *testing.B) { - for i := 0; i < b.N; i++ { - test := matchingTests[0] - _, _, gasUsed, _ = test.test.RunNoVerify(test.st, cfg, false, rawdb.HashScheme) - } - }) - var stats execStats - // Get the average execution time from the benchmarking result. - // There are other useful stats here that could be reported. - stats.time = time.Duration(result.NsPerOp()) - stats.allocs = result.AllocsPerOp() - stats.bytesAllocated = result.AllocedBytesPerOp() - fmt.Fprintf(os.Stderr, `EVM gas used: %d -execution time: %v -allocations: %d -allocated bytes: %d -`, gasUsed, stats.time, stats.allocs, stats.bytesAllocated) + return nil } From 7cf43bf793b477b73c4eae678d4dda280b8c4ac4 Mon Sep 17 00:00:00 2001 From: Jared Wasinger Date: Tue, 1 Oct 2024 14:52:33 +0700 Subject: [PATCH 09/12] remove typo --- cmd/evm/runner.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/evm/runner.go b/cmd/evm/runner.go index 98351dddbc4f..782a2ec78f0e 100644 --- a/cmd/evm/runner.go +++ b/cmd/evm/runner.go @@ -96,7 +96,6 @@ func timedExec(bench bool, execFunc func() ([]byte, uint64, error)) (output []by stats.BytesAllocated = result.AllocedBytesPerOp() stats.GasUsed = gasUsed } else { - g var memStatsBefore, memStatsAfter goruntime.MemStats goruntime.ReadMemStats(&memStatsBefore) startTime := time.Now() From cf774411a59035e1099d4312ebda0f161b7d5c82 Mon Sep 17 00:00:00 2001 From: Jared Wasinger Date: Tue, 1 Oct 2024 15:00:57 +0700 Subject: [PATCH 10/12] gofmt --- cmd/evm/staterunner.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/evm/staterunner.go b/cmd/evm/staterunner.go index 76300c2e11e4..828a58efd843 100644 --- a/cmd/evm/staterunner.go +++ b/cmd/evm/staterunner.go @@ -20,6 +20,8 @@ import ( "bufio" "encoding/json" "fmt" + "os" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" @@ -28,7 +30,6 @@ import ( "github.com/ethereum/go-ethereum/internal/flags" "github.com/ethereum/go-ethereum/tests" "github.com/urfave/cli/v2" - "os" ) var ( From fedbdee893f6b1f7ec1b27354ee9e29bd1e6b4d6 Mon Sep 17 00:00:00 2001 From: Jared Wasinger Date: Fri, 4 Oct 2024 20:05:33 +0700 Subject: [PATCH 11/12] remove unused code --- cmd/evm/staterunner.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/cmd/evm/staterunner.go b/cmd/evm/staterunner.go index 828a58efd843..34ca0f1f21aa 100644 --- a/cmd/evm/staterunner.go +++ b/cmd/evm/staterunner.go @@ -187,12 +187,5 @@ func runStateTest(ctx *cli.Context, fname string, cfg vm.Config, dump bool, benc } out, _ := json.MarshalIndent(results, "", " ") fmt.Println(string(out)) - - if !bench { - return nil - } else if len(matchingTests) != 1 { - return fmt.Errorf("can only benchmark single state test case (more than one matching params)") - } - return nil } From 1f84917da0cca3d8de3250cbf0219c0236332213 Mon Sep 17 00:00:00 2001 From: Jared Wasinger Date: Fri, 4 Oct 2024 20:13:24 +0700 Subject: [PATCH 12/12] make ExecStats non-public --- cmd/evm/runner.go | 4 ++-- cmd/evm/staterunner.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/evm/runner.go b/cmd/evm/runner.go index 782a2ec78f0e..2e61db266402 100644 --- a/cmd/evm/runner.go +++ b/cmd/evm/runner.go @@ -74,14 +74,14 @@ func readGenesis(genesisPath string) *core.Genesis { return genesis } -type ExecStats struct { +type execStats struct { Time time.Duration `json:"time"` // The execution Time. Allocs int64 `json:"allocs"` // The number of heap allocations during execution. BytesAllocated int64 `json:"bytesAllocated"` // The cumulative number of bytes allocated during execution. GasUsed uint64 `json:"gasUsed"` // the amount of gas used during execution } -func timedExec(bench bool, execFunc func() ([]byte, uint64, error)) (output []byte, stats ExecStats, err error) { +func timedExec(bench bool, execFunc func() ([]byte, uint64, error)) (output []byte, stats execStats, err error) { var gasUsed uint64 if bench { result := testing.Benchmark(func(b *testing.B) { diff --git a/cmd/evm/staterunner.go b/cmd/evm/staterunner.go index 34ca0f1f21aa..d0a0d3287cb5 100644 --- a/cmd/evm/staterunner.go +++ b/cmd/evm/staterunner.go @@ -71,7 +71,7 @@ type StatetestResult struct { Fork string `json:"fork"` Error string `json:"error,omitempty"` State *state.Dump `json:"state,omitempty"` - BenchStats *ExecStats `json:"benchStats,omitempty"` + BenchStats *execStats `json:"benchStats,omitempty"` } func stateTestCmd(ctx *cli.Context) error {