From 7f435fb01715e70f3712fda28d548828dc36d055 Mon Sep 17 00:00:00 2001 From: Rapol Date: Tue, 24 Sep 2024 20:06:50 +0200 Subject: [PATCH 1/3] added flag + test --- db/substate_db.go | 31 ++++----- db/substate_db_test.go | 11 ++- db/substate_encoding.go | 80 +++++++++++++++++++++ db/substate_encoding_test.go | 130 +++++++++++++++++++++++++++++++++++ db/substate_iterator.go | 11 +-- db/substate_iterator_test.go | 5 ++ 6 files changed, 241 insertions(+), 27 deletions(-) create mode 100644 db/substate_encoding.go create mode 100644 db/substate_encoding_test.go diff --git a/db/substate_db.go b/db/substate_db.go index 3f91598..904d8c1 100644 --- a/db/substate_db.go +++ b/db/substate_db.go @@ -43,6 +43,12 @@ type SubstateDB interface { // GetLastSubstate returns last substate (block and transaction wise) inside given DB. GetLastSubstate() (*substate.Substate, error) + + // SetSubstateEncoding sets the decoder func to the provided encoding + SetSubstateEncoding(encoding string) (*substateDB, error) + + // GetSubstateEncoding returns the currently configured encoding + GetSubstateEncoding() string } // NewDefaultSubstateDB creates new instance of SubstateDB with default options. @@ -57,11 +63,11 @@ func NewSubstateDB(path string, o *opt.Options, wo *opt.WriteOptions, ro *opt.Re } func MakeDefaultSubstateDB(db *leveldb.DB) SubstateDB { - return &substateDB{&codeDB{&baseDB{backend: db}}} + return &substateDB{&codeDB{&baseDB{backend: db}}, nil} } func MakeDefaultSubstateDBFromBaseDB(db BaseDB) SubstateDB { - return &substateDB{&codeDB{&baseDB{backend: db.getBackend()}}} + return &substateDB{&codeDB{&baseDB{backend: db.getBackend()}}, nil} } // NewReadOnlySubstateDB creates a new instance of read-only SubstateDB. @@ -70,7 +76,7 @@ func NewReadOnlySubstateDB(path string) (SubstateDB, error) { } func MakeSubstateDB(db *leveldb.DB, wo *opt.WriteOptions, ro *opt.ReadOptions) SubstateDB { - return &substateDB{&codeDB{&baseDB{backend: db, wo: wo, ro: ro}}} + return &substateDB{&codeDB{&baseDB{backend: db, wo: wo, ro: ro}}, nil} } func newSubstateDB(path string, o *opt.Options, wo *opt.WriteOptions, ro *opt.ReadOptions) (*substateDB, error) { @@ -78,11 +84,12 @@ func newSubstateDB(path string, o *opt.Options, wo *opt.WriteOptions, ro *opt.Re if err != nil { return nil, err } - return &substateDB{base}, nil + return &substateDB{base, nil}, nil } type substateDB struct { *codeDB + encoding *substateEncoding } func (db *substateDB) GetFirstSubstate() *substate.Substate { @@ -108,12 +115,7 @@ func (db *substateDB) GetSubstate(block uint64, tx int) (*substate.Substate, err return nil, fmt.Errorf("cannot get substate block: %v, tx: %v from db; %w", block, tx, err) } - rlpSubstate, err := rlp.Decode(val) - if err != nil { - return nil, fmt.Errorf("cannot decode data into rlp block: %v, tx %v; %w", block, tx, err) - } - - return rlpSubstate.ToSubstate(db.GetCode, block, tx) + return db.decodeToSubstate(val, block, tx) } // GetBlockSubstates returns substates for given block if exists within DB. @@ -138,14 +140,9 @@ func (db *substateDB) GetBlockSubstates(block uint64) (map[int]*substate.Substat return nil, fmt.Errorf("record-replay: GetBlockSubstates(%v) iterated substates from block %v", block, b) } - rlpSubstate, err := rlp.Decode(value) - if err != nil { - return nil, fmt.Errorf("cannot decode data into rlp block: %v, tx %v; %w", block, tx, err) - } - - sbstt, err := rlpSubstate.ToSubstate(db.GetCode, block, tx) + sbstt, err := db.decodeToSubstate(value, block, tx) if err != nil { - return nil, fmt.Errorf("cannot decode data into substate: %w", err) + return nil, fmt.Errorf("failed to decode substate, block %v, tx: %v; %w", block, tx, err) } txSubstate[tx] = sbstt diff --git a/db/substate_db_test.go b/db/substate_db_test.go index 3a90ccb..a3f1105 100644 --- a/db/substate_db_test.go +++ b/db/substate_db_test.go @@ -72,6 +72,10 @@ func TestSubstateDB_GetSubstate(t *testing.T) { t.Fatal(err) } + testSubstateDB_GetSubstate(db, t) +} + +func testSubstateDB_GetSubstate(db *substateDB, t *testing.T) { ss, err := db.GetSubstate(37_534_834, 1) if err != nil { t.Fatalf("get substate returned error; %v", err) @@ -84,6 +88,7 @@ func TestSubstateDB_GetSubstate(t *testing.T) { if err = ss.Equal(testSubstate); err != nil { t.Fatalf("substates are different; %v", err) } + } func TestSubstateDB_DeleteSubstate(t *testing.T) { @@ -200,13 +205,17 @@ func createDbAndPutSubstate(dbPath string) (*substateDB, error) { } func addSubstate(db *substateDB, blk uint64) error { + return addCustomSubstate(db, blk, testSubstate) +} + +func addCustomSubstate(db *substateDB, blk uint64, ss *substate.Substate) error { h1 := types.Hash{} h1.SetBytes(nil) h2 := types.Hash{} h2.SetBytes(nil) - s := *testSubstate + s := *ss s.InputSubstate[types.Address{1}] = substate.NewAccount(1, new(big.Int).SetUint64(1), h1[:]) s.OutputSubstate[types.Address{2}] = substate.NewAccount(2, new(big.Int).SetUint64(2), h2[:]) diff --git a/db/substate_encoding.go b/db/substate_encoding.go new file mode 100644 index 0000000..035dcc2 --- /dev/null +++ b/db/substate_encoding.go @@ -0,0 +1,80 @@ +package db + +import ( + "fmt" + + "github.com/Fantom-foundation/Substate/rlp" + "github.com/Fantom-foundation/Substate/substate" + "github.com/Fantom-foundation/Substate/types" +) + +// SetSubstateEncoding sets the runtime encoding/decoding behavior of substateDB +// intended usage: +// +// db := &substateDB{..} +// db, err := db.SetSubstateEncoding() // set encoding +// db.GetSubstateDecoder() // returns configured encoding +func (db *substateDB) SetSubstateEncoding(schema string) (*substateDB, error) { + encoding, err := newSubstateEncoding(schema, db.GetCode) + if err != nil { + return nil, fmt.Errorf("Failed to set decoder; %w", err) + } + + db.encoding = encoding + return db, nil +} + +// GetDecoder returns the encoding in use +func (db *substateDB) GetSubstateEncoding() string { + if db.encoding == nil { + return "" + } + return db.encoding.schema +} + +type substateEncoding struct { + schema string + decode decoderFunc +} + +// decoderFunc aliases the common function used to decode substate +type decoderFunc func([]byte, uint64, int) (*substate.Substate, error) + +// codeLookup aliases codehash->code lookup necessary to decode substate +type codeLookup = func(types.Hash) ([]byte, error) + +// newSubstateDecoder returns requested SubstateDecoder +func newSubstateEncoding(encoding string, lookup codeLookup) (*substateEncoding, error) { + switch encoding { + + case "default", "rlp": + return &substateEncoding{ + schema: "rlp", + decode: func(bytes []byte, block uint64, tx int) (*substate.Substate, error) { + return decodeRlp(bytes, lookup, block, tx) + }, + }, nil + + default: + return nil, fmt.Errorf("Encoding not supported: %s", encoding) + + } +} + +// decodeSubstate defensively defaults to "default" if nil +func (db *substateDB) decodeToSubstate(bytes []byte, block uint64, tx int) (*substate.Substate, error) { + if db.encoding == nil { + db.SetSubstateEncoding("default") + } + return db.encoding.decode(bytes, block, tx) +} + +// decodeRlp decodes into substate the provided rlp-encoded bytecode +func decodeRlp(bytes []byte, lookup codeLookup, block uint64, tx int) (*substate.Substate, error) { + rlpSubstate, err := rlp.Decode(bytes) + if err != nil { + return nil, fmt.Errorf("cannot decode substate data from rlp block: %v, tx %v; %w", block, tx, err) + } + + return rlpSubstate.ToSubstate(lookup, block, tx) +} diff --git a/db/substate_encoding_test.go b/db/substate_encoding_test.go new file mode 100644 index 0000000..4bec5c3 --- /dev/null +++ b/db/substate_encoding_test.go @@ -0,0 +1,130 @@ +package db + +import ( + "strings" + "testing" + + "github.com/Fantom-foundation/Substate/rlp" + trlp "github.com/Fantom-foundation/Substate/types/rlp" +) + +var ( + testRlp, _ = trlp.EncodeToBytes(rlp.NewRLP(testSubstate)) + testBlk = testSubstate.Block + testTx = testSubstate.Transaction + + supportedEncoding = map[string][]byte{ + "rlp": testRlp, + } +) + +func TestSubstateEncoding_NilEncodingDefaultsToRlp(t *testing.T) { + path := t.TempDir() + "test-db" + db, err := newSubstateDB(path, nil, nil, nil) + if err != nil { + t.Errorf("cannot open db; %v", err) + } + + if got := db.GetSubstateEncoding(); got != "" { + t.Fatalf("substate encoding should be nil, got: %s", got) + } + + // purposely never set encoding + _, err = db.decodeToSubstate(testRlp, testBlk, testTx) + if err != nil { + t.Fatal(err) + } + + if got := db.GetSubstateEncoding(); got != "rlp" { + t.Fatalf("db should default to rlp, got: %s", got) + } +} + +func TestSubstateEncoding_DefaultEncodingDefaultsToRlp(t *testing.T) { + path := t.TempDir() + "test-db" + db, err := newSubstateDB(path, nil, nil, nil) + if err != nil { + t.Errorf("cannot open db; %v", err) + } + + _, err = db.SetSubstateEncoding("default") + if err != nil { + t.Fatal("default is supportet, but error") + } + + _, err = db.decodeToSubstate(testRlp, testBlk, testTx) + if err != nil { + t.Fatal(err) + } + + if got := db.GetSubstateEncoding(); got != "rlp" { + t.Fatalf("db should default to rlp, got: %s", got) + } +} + +func TestSubstateEncoding_UnsupportedEncodingThrowsError(t *testing.T) { + path := t.TempDir() + "test-db" + db, err := newSubstateDB(path, nil, nil, nil) + if err != nil { + t.Errorf("cannot open db; %v", err) + } + + _, err = db.SetSubstateEncoding("EncodingNotSupported") + if err == nil || !strings.Contains(err.Error(), "Encoding not supported") { + t.Error("Encoding not supported, but no error") + } +} + +func TestSubstateEncoding_TestDb(t *testing.T) { + path := t.TempDir() + "test-db" + db, err := newSubstateDB(path, nil, nil, nil) + if err != nil { + t.Errorf("cannot open db; %v", err) + } + + for encoding, bytes := range supportedEncoding { + _, err = db.SetSubstateEncoding(encoding) + if err != nil { + t.Error(err) + } + + ss, err := db.decodeToSubstate(bytes, testBlk, testTx) + if err != nil { + t.Error(err) + } + + err = addCustomSubstate(db, testBlk, ss) + if err != nil { + t.Error(err) + } + + testSubstateDB_GetSubstate(db, t) + } +} + +func TestSubstateEncoding_TestIterator(t *testing.T) { + path := t.TempDir() + "test-db" + db, err := newSubstateDB(path, nil, nil, nil) + if err != nil { + t.Errorf("cannot open db; %v", err) + } + + for encoding, bytes := range supportedEncoding { + _, err = db.SetSubstateEncoding(encoding) + if err != nil { + t.Error(err) + } + + ss, err := db.decodeToSubstate(bytes, testBlk, testTx) + if err != nil { + t.Error(err) + } + + err = addCustomSubstate(db, testBlk, ss) + if err != nil { + t.Error(err) + } + + testSubstatorIterator_Value(db, t) + } +} diff --git a/db/substate_iterator.go b/db/substate_iterator.go index 99a145c..a801e93 100644 --- a/db/substate_iterator.go +++ b/db/substate_iterator.go @@ -3,10 +3,8 @@ package db import ( "fmt" - "github.com/syndtr/goleveldb/leveldb/util" - - "github.com/Fantom-foundation/Substate/rlp" "github.com/Fantom-foundation/Substate/substate" + "github.com/syndtr/goleveldb/leveldb/util" ) func newSubstateIterator(db *substateDB, start []byte) *substateIterator { @@ -33,12 +31,7 @@ func (i *substateIterator) decode(data rawEntry) (*substate.Substate, error) { return nil, fmt.Errorf("invalid substate key: %v; %w", key, err) } - rlpSubstate, err := rlp.Decode(value) - if err != nil { - return nil, err - } - - return rlpSubstate.ToSubstate(i.db.GetCode, block, tx) + return i.db.decodeToSubstate(value, block, tx) } func (i *substateIterator) start(numWorkers int) { diff --git a/db/substate_iterator_test.go b/db/substate_iterator_test.go index f4dd139..29d394d 100644 --- a/db/substate_iterator_test.go +++ b/db/substate_iterator_test.go @@ -29,6 +29,10 @@ func TestSubstateIterator_Value(t *testing.T) { return } + testSubstatorIterator_Value(db, t) +} + +func testSubstatorIterator_Value(db *substateDB, t *testing.T) { iter := db.NewSubstateIterator(0, 10) if !iter.Next() { @@ -48,6 +52,7 @@ func TestSubstateIterator_Value(t *testing.T) { if tx.Transaction != 1 { t.Fatalf("iterator returned transaction with different transaction number\ngot: %v\n want: %v", tx.Transaction, 1) } + } func TestSubstateIterator_Release(t *testing.T) { From 0f0919d680bd11311e4f060820625e0220f83500 Mon Sep 17 00:00:00 2001 From: RT <146690782+rpl-ffl@users.noreply.github.com> Date: Wed, 25 Sep 2024 16:38:23 +0700 Subject: [PATCH 2/3] Rename Funcs Co-authored-by: cabrador <84449820+cabrador@users.noreply.github.com> --- db/substate_encoding.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db/substate_encoding.go b/db/substate_encoding.go index 035dcc2..deedb37 100644 --- a/db/substate_encoding.go +++ b/db/substate_encoding.go @@ -38,10 +38,10 @@ type substateEncoding struct { } // decoderFunc aliases the common function used to decode substate -type decoderFunc func([]byte, uint64, int) (*substate.Substate, error) +type decodeFunc func([]byte, uint64, int) (*substate.Substate, error) // codeLookup aliases codehash->code lookup necessary to decode substate -type codeLookup = func(types.Hash) ([]byte, error) +type codeLookupFunc = func(types.Hash) ([]byte, error) // newSubstateDecoder returns requested SubstateDecoder func newSubstateEncoding(encoding string, lookup codeLookup) (*substateEncoding, error) { From 80c981501dc37fcc985926949b58c1333a85cb56 Mon Sep 17 00:00:00 2001 From: Rapol Date: Wed, 25 Sep 2024 12:59:28 +0200 Subject: [PATCH 3/3] codeLookup->codeLookupFunc decoderFunc->decodeFunc --- db/substate_encoding.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/db/substate_encoding.go b/db/substate_encoding.go index deedb37..921fa61 100644 --- a/db/substate_encoding.go +++ b/db/substate_encoding.go @@ -34,17 +34,17 @@ func (db *substateDB) GetSubstateEncoding() string { type substateEncoding struct { schema string - decode decoderFunc + decode decodeFunc } -// decoderFunc aliases the common function used to decode substate +// decodeFunc aliases the common function used to decode substate type decodeFunc func([]byte, uint64, int) (*substate.Substate, error) -// codeLookup aliases codehash->code lookup necessary to decode substate +// codeLookupFunc aliases codehash->code lookup necessary to decode substate type codeLookupFunc = func(types.Hash) ([]byte, error) // newSubstateDecoder returns requested SubstateDecoder -func newSubstateEncoding(encoding string, lookup codeLookup) (*substateEncoding, error) { +func newSubstateEncoding(encoding string, lookup codeLookupFunc) (*substateEncoding, error) { switch encoding { case "default", "rlp": @@ -70,7 +70,7 @@ func (db *substateDB) decodeToSubstate(bytes []byte, block uint64, tx int) (*sub } // decodeRlp decodes into substate the provided rlp-encoded bytecode -func decodeRlp(bytes []byte, lookup codeLookup, block uint64, tx int) (*substate.Substate, error) { +func decodeRlp(bytes []byte, lookup codeLookupFunc, block uint64, tx int) (*substate.Substate, error) { rlpSubstate, err := rlp.Decode(bytes) if err != nil { return nil, fmt.Errorf("cannot decode substate data from rlp block: %v, tx %v; %w", block, tx, err)