From f421ded331f76d81c07e3bbd0f2fef472d7e4e69 Mon Sep 17 00:00:00 2001 From: ramtinms Date: Wed, 7 Aug 2024 00:20:48 -0400 Subject: [PATCH 1/7] keep the check on nil for methods --- fvm/evm/debug/tracer.go | 238 +++++++++++++++++++++------------------- 1 file changed, 128 insertions(+), 110 deletions(-) diff --git a/fvm/evm/debug/tracer.go b/fvm/evm/debug/tracer.go index 5af0828166c..00a0231ae92 100644 --- a/fvm/evm/debug/tracer.go +++ b/fvm/evm/debug/tracer.go @@ -36,6 +36,7 @@ type CallTracer struct { tracer *tracers.Tracer uploader Uploader blockID flow.Identifier + txTracer *tracers.Tracer } func NewEVMCallTracer(uploader Uploader, logger zerolog.Logger) (*CallTracer, error) { @@ -52,7 +53,10 @@ func NewEVMCallTracer(uploader Uploader, logger zerolog.Logger) (*CallTracer, er } func (t *CallTracer) TxTracer() *tracers.Tracer { - return NewSafeTxTracer(t) + if t.txTracer == nil { + t.txTracer = NewSafeTxTracer(t) + } + return t.txTracer } func (t *CallTracer) WithBlockID(id flow.Identifier) { @@ -126,134 +130,148 @@ func NewSafeTxTracer(ct *CallTracer) *tracers.Tracer { Str("block-id", ct.blockID.String()). Logger() - wrapped.OnTxStart = func( - vm *tracing.VMContext, - tx *types.Transaction, - from gethCommon.Address, - ) { - defer func() { - if r := recover(); r != nil { - err, ok := r.(error) - if !ok { - err = fmt.Errorf("panic: %v", r) + if ct.tracer.OnTxStart != nil { + wrapped.OnTxStart = func( + vm *tracing.VMContext, + tx *types.Transaction, + from gethCommon.Address, + ) { + defer func() { + if r := recover(); r != nil { + err, ok := r.(error) + if !ok { + err = fmt.Errorf("panic: %v", r) + } + l.Err(err). + Stack(). + Msg("OnTxStart trace collection failed") } - l.Err(err). - Stack(). - Msg("OnTxStart trace collection failed") - } - }() - ct.tracer.OnTxStart(vm, tx, from) + }() + ct.tracer.OnTxStart(vm, tx, from) + } } - wrapped.OnTxEnd = func(receipt *types.Receipt, err error) { - defer func() { - if r := recover(); r != nil { - err, ok := r.(error) - if !ok { - err = fmt.Errorf("panic: %v", r) + if ct.tracer.OnTxEnd != nil { + wrapped.OnTxEnd = func(receipt *types.Receipt, err error) { + defer func() { + if r := recover(); r != nil { + err, ok := r.(error) + if !ok { + err = fmt.Errorf("panic: %v", r) + } + l.Err(err). + Stack(). + Msg("OnTxEnd trace collection failed") } - l.Err(err). - Stack(). - Msg("OnTxEnd trace collection failed") - } - }() - ct.tracer.OnTxEnd(receipt, err) + }() + ct.tracer.OnTxEnd(receipt, err) + } } - wrapped.OnEnter = func( - depth int, - typ byte, - from, to gethCommon.Address, - input []byte, - gas uint64, - value *big.Int, - ) { - defer func() { - if r := recover(); r != nil { - err, ok := r.(error) - if !ok { - err = fmt.Errorf("panic: %v", r) + if ct.tracer.OnEnter != nil { + wrapped.OnEnter = func( + depth int, + typ byte, + from, to gethCommon.Address, + input []byte, + gas uint64, + value *big.Int, + ) { + defer func() { + if r := recover(); r != nil { + err, ok := r.(error) + if !ok { + err = fmt.Errorf("panic: %v", r) + } + l.Err(err). + Stack(). + Msg("OnEnter trace collection failed") } - l.Err(err). - Stack(). - Msg("OnEnter trace collection failed") - } - }() - ct.tracer.OnEnter(depth, typ, from, to, input, gas, value) + }() + ct.tracer.OnEnter(depth, typ, from, to, input, gas, value) + } } - wrapped.OnExit = func(depth int, output []byte, gasUsed uint64, err error, reverted bool) { - defer func() { - if r := recover(); r != nil { - err, ok := r.(error) - if !ok { - err = fmt.Errorf("panic: %v", r) + if ct.tracer.OnExit != nil { + wrapped.OnExit = func(depth int, output []byte, gasUsed uint64, err error, reverted bool) { + defer func() { + if r := recover(); r != nil { + err, ok := r.(error) + if !ok { + err = fmt.Errorf("panic: %v", r) + } + l.Err(err). + Stack(). + Msg("OnExit trace collection failed") } - l.Err(err). - Stack(). - Msg("OnExit trace collection failed") - } - }() - ct.tracer.OnExit(depth, output, gasUsed, err, reverted) + }() + ct.tracer.OnExit(depth, output, gasUsed, err, reverted) + } } - wrapped.OnOpcode = func( - pc uint64, - op byte, - gas, cost uint64, - scope tracing.OpContext, - rData []byte, - depth int, - err error, - ) { - defer func() { - if r := recover(); r != nil { - err, ok := r.(error) - if !ok { - err = fmt.Errorf("panic: %v", r) + if ct.tracer.OnOpcode != nil { + wrapped.OnOpcode = func( + pc uint64, + op byte, + gas, cost uint64, + scope tracing.OpContext, + rData []byte, + depth int, + err error, + ) { + defer func() { + if r := recover(); r != nil { + err, ok := r.(error) + if !ok { + err = fmt.Errorf("panic: %v", r) + } + l.Err(err). + Stack(). + Msg("OnOpcode trace collection failed") } - l.Err(err). - Stack(). - Msg("OnOpcode trace collection failed") - } - }() - ct.tracer.OnOpcode(pc, op, gas, cost, scope, rData, depth, err) + }() + ct.tracer.OnOpcode(pc, op, gas, cost, scope, rData, depth, err) + } } - wrapped.OnFault = func( - pc uint64, - op byte, - gas, cost uint64, - scope tracing.OpContext, - depth int, - err error) { - defer func() { - if r := recover(); r != nil { - err, ok := r.(error) - if !ok { - err = fmt.Errorf("panic: %v", r) + if ct.tracer.OnFault != nil { + wrapped.OnFault = func( + pc uint64, + op byte, + gas, cost uint64, + scope tracing.OpContext, + depth int, + err error) { + defer func() { + if r := recover(); r != nil { + err, ok := r.(error) + if !ok { + err = fmt.Errorf("panic: %v", r) + } + l.Err(err). + Stack(). + Msg("OnFault trace collection failed") } - l.Err(err). - Stack(). - Msg("OnFault trace collection failed") - } - }() - ct.tracer.OnFault(pc, op, gas, cost, scope, depth, err) + }() + ct.tracer.OnFault(pc, op, gas, cost, scope, depth, err) + } } - wrapped.OnGasChange = func(old, new uint64, reason tracing.GasChangeReason) { - defer func() { - if r := recover(); r != nil { - err, ok := r.(error) - if !ok { - err = fmt.Errorf("panic: %v", r) + if ct.tracer.OnGasChange != nil { + wrapped.OnGasChange = func(old, new uint64, reason tracing.GasChangeReason) { + defer func() { + if r := recover(); r != nil { + err, ok := r.(error) + if !ok { + err = fmt.Errorf("panic: %v", r) + } + l.Err(err). + Stack(). + Msg("OnGasChange trace collection failed") } - l.Err(err). - Stack(). - Msg("OnGasChange trace collection failed") - } - }() - ct.tracer.OnGasChange(old, new, reason) + }() + ct.tracer.OnGasChange(old, new, reason) + } } wrapped.GetResult = ct.tracer.GetResult From 0f87b8d062bf99513636d2fd3b420607b9355f24 Mon Sep 17 00:00:00 2001 From: ramtinms Date: Wed, 7 Aug 2024 00:58:44 -0400 Subject: [PATCH 2/7] add test for proxy call tracing --- fvm/evm/emulator/emulator_test.go | 78 +++++++++++++++++++++ fvm/evm/testutils/contract.go | 7 ++ fvm/evm/testutils/contracts/contracts.go | 8 +++ fvm/evm/testutils/contracts/proxy.sol | 25 +++++++ fvm/evm/testutils/contracts/proxy_abi.json | 19 +++++ fvm/evm/testutils/contracts/proxy_bytes.hex | 1 + 6 files changed, 138 insertions(+) create mode 100644 fvm/evm/testutils/contracts/proxy.sol create mode 100644 fvm/evm/testutils/contracts/proxy_abi.json create mode 100644 fvm/evm/testutils/contracts/proxy_bytes.hex diff --git a/fvm/evm/emulator/emulator_test.go b/fvm/evm/emulator/emulator_test.go index 1698ce2c1ea..e1ff4b28e91 100644 --- a/fvm/evm/emulator/emulator_test.go +++ b/fvm/evm/emulator/emulator_test.go @@ -1063,6 +1063,84 @@ func TestTransactionTracing(t *testing.T) { }) + t.Run("contract interaction using run transaction with proxy", func(t *testing.T) { + runWithDeployedContract(t, func(testContract *testutils.TestContract, testAccount *testutils.EOATestAccount, emu *emulator.Emulator) { + + // deploy proxy + proxyContract := testutils.GetProxyContract(t) + RunWithNewBlockView(t, emu, func(blk types.BlockView) { + call := types.NewDeployCall( + testAccount.Address(), + proxyContract.ByteCode, + math.MaxUint64, + big.NewInt(0), + testAccount.Nonce()) + + res, err := blk.DirectCall(call) + require.NoError(t, err) + require.NotNil(t, res.DeployedContractAddress) + testAccount.SetNonce(testAccount.Nonce() + 1) + proxyContract.DeployedAt = *res.DeployedContractAddress + }) + + RunWithNewBlockView(t, emu, func(blk types.BlockView) { + // set proxy contract reference the test contract + tx := testAccount.PrepareAndSignTx( + t, + proxyContract.DeployedAt.ToCommon(), + proxyContract.MakeCallData(t, "setImplementation", testContract.DeployedAt.ToCommon()), + big.NewInt(0), + 1_000_000, + big.NewInt(0), + ) + res, err := blk.RunTransaction(tx) + requireSuccessfulExecution(t, err, res) + }) + + blk, uploader, tracer := blockWithTracer(t, emu) + + var txID gethCommon.Hash + var trace json.RawMessage + + blockID := flow.Identifier{0x02} + uploaded := make(chan struct{}) + + uploader.UploadFunc = func(id string, message json.RawMessage) error { + uploaded <- struct{}{} + require.Equal(t, debug.TraceID(txID, blockID), id) + require.Equal(t, trace, message) + require.Greater(t, len(message), 0) + return nil + } + + // make call to the proxy + tx := testAccount.PrepareAndSignTx( + t, + proxyContract.DeployedAt.ToCommon(), + testContract.MakeCallData(t, "store", big.NewInt(2)), + big.NewInt(0), + 1_000_000, + big.NewInt(0), + ) + + // interact and record trace + res, err := blk.RunTransaction(tx) + requireSuccessfulExecution(t, err, res) + txID = res.TxHash + trace, err = tracer.TxTracer().GetResult() + require.NoError(t, err) + require.NotEmpty(t, trace) + tracer.WithBlockID(blockID) + + tracer.Collect(txID) + testAccount.SetNonce(testAccount.Nonce() + 1) + require.Eventuallyf(t, func() bool { + <-uploaded + return true + }, time.Second, time.Millisecond*100, "upload did not execute") + }) + + }) t.Run("contract interaction run failed transaction", func(t *testing.T) { runWithDeployedContract(t, func(testContract *testutils.TestContract, testAccount *testutils.EOATestAccount, emu *emulator.Emulator) { blk, _, tracer := blockWithTracer(t, emu) diff --git a/fvm/evm/testutils/contract.go b/fvm/evm/testutils/contract.go index 1c084a4262d..4089fca7701 100644 --- a/fvm/evm/testutils/contract.go +++ b/fvm/evm/testutils/contract.go @@ -52,6 +52,13 @@ func GetDummyKittyTestContract(t testing.TB) *TestContract { } } +func GetProxyContract(t testing.TB) *TestContract { + return &TestContract{ + ABI: contracts.ProxyContractABIJSON, + ByteCode: contracts.ProxyContractBytes, + } +} + func RunWithDeployedContract(t testing.TB, tc *TestContract, led atree.Ledger, flowEVMRootAddress flow.Address, f func(*TestContract)) { DeployContract(t, RandomAddress(t), tc, led, flowEVMRootAddress) f(tc) diff --git a/fvm/evm/testutils/contracts/contracts.go b/fvm/evm/testutils/contracts/contracts.go index adce36b692e..bf489fd1332 100644 --- a/fvm/evm/testutils/contracts/contracts.go +++ b/fvm/evm/testutils/contracts/contracts.go @@ -20,3 +20,11 @@ var DummyKittyContractBytes, _ = hex.DecodeString(dummyKittyContractBytesInHex) //go:embed dummy_kitty_abi.json var DummyKittyContractABIJSON string + +//go:embed proxy_bytes.hex +var proxyContractBytesInHex string + +var ProxyContractBytes, _ = hex.DecodeString(proxyContractBytesInHex) + +//go:embed proxy_abi.json +var ProxyContractABIJSON string diff --git a/fvm/evm/testutils/contracts/proxy.sol b/fvm/evm/testutils/contracts/proxy.sol new file mode 100644 index 00000000000..5dc3f80107b --- /dev/null +++ b/fvm/evm/testutils/contracts/proxy.sol @@ -0,0 +1,25 @@ +pragma solidity ^0.8.0; + +contract Proxy { + address private _implementation; + + function setImplementation(address implementation) external { + _implementation = implementation; + } + + fallback() external payable { + address impl = _implementation; + assembly { + calldatacopy(0, 0, calldatasize()) + let result := delegatecall(gas(), impl, 0, calldatasize(), 0, 0) + returndatacopy(0, 0, returndatasize()) + switch result + case 0 { + revert(0, returndatasize()) + } + default { + return(0, returndatasize()) + } + } + } +} \ No newline at end of file diff --git a/fvm/evm/testutils/contracts/proxy_abi.json b/fvm/evm/testutils/contracts/proxy_abi.json new file mode 100644 index 00000000000..aaba81ba703 --- /dev/null +++ b/fvm/evm/testutils/contracts/proxy_abi.json @@ -0,0 +1,19 @@ +[ + { + "stateMutability": "payable", + "type": "fallback" + }, + { + "inputs": [ + { + "internalType": "address", + "name": "implementation", + "type": "address" + } + ], + "name": "setImplementation", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + } +] \ No newline at end of file diff --git a/fvm/evm/testutils/contracts/proxy_bytes.hex b/fvm/evm/testutils/contracts/proxy_bytes.hex new file mode 100644 index 00000000000..700cc41f67d --- /dev/null +++ b/fvm/evm/testutils/contracts/proxy_bytes.hex @@ -0,0 +1 @@ +6080604052348015600e575f80fd5b5061018e8061001c5f395ff3fe608060405260043610610021575f3560e01c8063d784d4261461006557610022565b5b5f805f9054906101000a900473ffffffffffffffffffffffffffffffffffffffff169050365f80375f80365f845af43d5f803e805f8114610061573d5ff35b3d5ffd5b348015610070575f80fd5b5061008b6004803603810190610086919061012d565b61008d565b005b805f806101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff16021790555050565b5f80fd5b5f73ffffffffffffffffffffffffffffffffffffffff82169050919050565b5f6100fc826100d3565b9050919050565b61010c816100f2565b8114610116575f80fd5b50565b5f8135905061012781610103565b92915050565b5f60208284031215610142576101416100cf565b5b5f61014f84828501610119565b9150509291505056fea2646970667358221220dc83726b5fd29997fb44f2a424f0959fe05c9736efc3bf92f405182d14cdb5fb64736f6c634300081a0033 \ No newline at end of file From 0d1d79f8b2f7194314888c6b4490d4c5c1bf7697 Mon Sep 17 00:00:00 2001 From: ramtinms Date: Wed, 7 Aug 2024 01:03:23 -0400 Subject: [PATCH 3/7] fix test --- fvm/evm/debug/tracer.go | 2 +- fvm/evm/debug/tracer_test.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/fvm/evm/debug/tracer.go b/fvm/evm/debug/tracer.go index 00a0231ae92..e3f441ad3dd 100644 --- a/fvm/evm/debug/tracer.go +++ b/fvm/evm/debug/tracer.go @@ -19,7 +19,7 @@ import ( ) const ( - tracerConfig = `{ "onlyTopCall": true }` + tracerConfig = `{"onlyTopCall": true }` tracerName = "callTracer" ) diff --git a/fvm/evm/debug/tracer_test.go b/fvm/evm/debug/tracer_test.go index 9518bc8ff44..480d5d17ba6 100644 --- a/fvm/evm/debug/tracer_test.go +++ b/fvm/evm/debug/tracer_test.go @@ -46,6 +46,7 @@ func Test_CallTracer(t *testing.T) { tx := gethTypes.NewTransaction(nonce, to, amount, 100, big.NewInt(10), data) tr.OnTxStart(nil, tx, from) tr.OnEnter(1, byte(vm.ADD), from, to, data, 20, big.NewInt(2)) + tr.OnExit(1, nil, 10, nil, false) tr.OnTxEnd(&gethTypes.Receipt{}, nil) tr.OnExit(0, []byte{0x02}, 200, nil, false) From d3d2db2073c2b7a2b846733f788437d48794c79f Mon Sep 17 00:00:00 2001 From: ramtinms Date: Wed, 7 Aug 2024 01:17:42 -0400 Subject: [PATCH 4/7] add debug logs for tracer --- fvm/evm/debug/tracer.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/fvm/evm/debug/tracer.go b/fvm/evm/debug/tracer.go index e3f441ad3dd..48164b4e75e 100644 --- a/fvm/evm/debug/tracer.go +++ b/fvm/evm/debug/tracer.go @@ -148,6 +148,7 @@ func NewSafeTxTracer(ct *CallTracer) *tracers.Tracer { } }() ct.tracer.OnTxStart(vm, tx, from) + l.Debug().Msg("tracing OnTxStart is called") } } @@ -165,6 +166,7 @@ func NewSafeTxTracer(ct *CallTracer) *tracers.Tracer { } }() ct.tracer.OnTxEnd(receipt, err) + l.Debug().Msg("tracing OnTxEnd is called") } } @@ -189,6 +191,7 @@ func NewSafeTxTracer(ct *CallTracer) *tracers.Tracer { } }() ct.tracer.OnEnter(depth, typ, from, to, input, gas, value) + l.Debug().Int("depth", depth).Msg("tracing OnEnter is called") } } @@ -206,6 +209,7 @@ func NewSafeTxTracer(ct *CallTracer) *tracers.Tracer { } }() ct.tracer.OnExit(depth, output, gasUsed, err, reverted) + l.Debug().Int("depth", depth).Msg("tracing OnExit is called") } } @@ -231,6 +235,7 @@ func NewSafeTxTracer(ct *CallTracer) *tracers.Tracer { } }() ct.tracer.OnOpcode(pc, op, gas, cost, scope, rData, depth, err) + l.Debug().Msg("tracing OnOpcode is called") } } @@ -254,6 +259,7 @@ func NewSafeTxTracer(ct *CallTracer) *tracers.Tracer { } }() ct.tracer.OnFault(pc, op, gas, cost, scope, depth, err) + l.Debug().Msg("tracing OnFault is called") } } @@ -271,6 +277,7 @@ func NewSafeTxTracer(ct *CallTracer) *tracers.Tracer { } }() ct.tracer.OnGasChange(old, new, reason) + l.Debug().Msg("tracing OnGasChange is called") } } From a2579528eadab8592ed696db9eea88127ea1073a Mon Sep 17 00:00:00 2001 From: ramtinms Date: Wed, 7 Aug 2024 01:20:25 -0400 Subject: [PATCH 5/7] . --- fvm/evm/debug/tracer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fvm/evm/debug/tracer.go b/fvm/evm/debug/tracer.go index 48164b4e75e..f86712c79df 100644 --- a/fvm/evm/debug/tracer.go +++ b/fvm/evm/debug/tracer.go @@ -19,7 +19,7 @@ import ( ) const ( - tracerConfig = `{"onlyTopCall": true }` + tracerConfig = `{ "onlyTopCall": true }` tracerName = "callTracer" ) From 050505dafe119754f1081417af2b575c5e1db6cb Mon Sep 17 00:00:00 2001 From: ramtinms Date: Wed, 7 Aug 2024 10:49:16 -0400 Subject: [PATCH 6/7] add tracer reset --- fvm/evm/debug/tracer.go | 40 +++++++++++++++++++++++++++------------- 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/fvm/evm/debug/tracer.go b/fvm/evm/debug/tracer.go index f86712c79df..8df049233f7 100644 --- a/fvm/evm/debug/tracer.go +++ b/fvm/evm/debug/tracer.go @@ -32,31 +32,37 @@ type EVMTracer interface { var _ EVMTracer = &CallTracer{} type CallTracer struct { - logger zerolog.Logger - tracer *tracers.Tracer - uploader Uploader - blockID flow.Identifier - txTracer *tracers.Tracer + logger zerolog.Logger + tracer *tracers.Tracer + tracerConfig []byte + uploader Uploader + blockID flow.Identifier } func NewEVMCallTracer(uploader Uploader, logger zerolog.Logger) (*CallTracer, error) { - tracer, err := tracers.DefaultDirectory.New(tracerName, &tracers.Context{}, json.RawMessage(tracerConfig)) + tracerConfig := json.RawMessage(tracerConfig) + + tracer, err := tracers.DefaultDirectory.New(tracerName, &tracers.Context{}, tracerConfig) if err != nil { return nil, err } return &CallTracer{ - logger: logger.With().Str("module", "evm-tracer").Logger(), - tracer: tracer, - uploader: uploader, + logger: logger.With().Str("module", "evm-tracer").Logger(), + tracer: tracer, + tracerConfig: tracerConfig, + uploader: uploader, }, nil } func (t *CallTracer) TxTracer() *tracers.Tracer { - if t.txTracer == nil { - t.txTracer = NewSafeTxTracer(t) - } - return t.txTracer + return NewSafeTxTracer(t) +} + +func (t *CallTracer) ResetTracer() error { + var err error + t.tracer, err = tracers.DefaultDirectory.New(tracerName, &tracers.Context{}, json.RawMessage(tracerConfig)) + return err } func (t *CallTracer) WithBlockID(id flow.Identifier) { @@ -98,9 +104,17 @@ func (t *CallTracer) Collect(txID gethCommon.Hash) { Msg("failed to upload trace results, no more retries") return } + // reset tracing to have fresh state + if err := t.ResetTracer(); err != nil { + l.Error().Err(err). + Str("traces", string(res)). + Msg("failed to reset trace") + return + } l.Debug().Msg("evm traces uploaded successfully") }() + } var NopTracer = &nopTracer{} From b46a84b62381afd23ff4d4fb6bce2756d2470e40 Mon Sep 17 00:00:00 2001 From: ramtinms Date: Wed, 7 Aug 2024 10:52:59 -0400 Subject: [PATCH 7/7] move reset to higher level --- fvm/evm/debug/tracer.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/fvm/evm/debug/tracer.go b/fvm/evm/debug/tracer.go index 8df049233f7..58df192073e 100644 --- a/fvm/evm/debug/tracer.go +++ b/fvm/evm/debug/tracer.go @@ -98,17 +98,18 @@ func (t *CallTracer) Collect(txID gethCommon.Hash) { l.Error().Err(err).Msg("failed to produce trace results") } - if err = t.uploader.Upload(TraceID(txID, t.blockID), res); err != nil { + // reset tracing to have fresh state + if err := t.ResetTracer(); err != nil { l.Error().Err(err). Str("traces", string(res)). - Msg("failed to upload trace results, no more retries") + Msg("failed to reset trace") return } - // reset tracing to have fresh state - if err := t.ResetTracer(); err != nil { + + if err = t.uploader.Upload(TraceID(txID, t.blockID), res); err != nil { l.Error().Err(err). Str("traces", string(res)). - Msg("failed to reset trace") + Msg("failed to upload trace results, no more retries") return }