From 5d04cba786d2888e84de5a750694ab57a49bd5e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Tue, 30 Apr 2024 10:54:40 -0700 Subject: [PATCH] add predicate function and test for atree's PersistentSlabStorage.FixLoadedBrokenReferences --- migrations/broken_dictionary.go | 47 +++++++++ migrations/migration_test.go | 99 +++++++++++++++++++ .../testdata/missing-slabs-payloads.csv | 25 +++++ runtime/tests/runtime_utils/testledger.go | 39 ++++++-- 4 files changed, 203 insertions(+), 7 deletions(-) create mode 100644 migrations/broken_dictionary.go create mode 100644 migrations/testdata/missing-slabs-payloads.csv diff --git a/migrations/broken_dictionary.go b/migrations/broken_dictionary.go new file mode 100644 index 0000000000..a342920770 --- /dev/null +++ b/migrations/broken_dictionary.go @@ -0,0 +1,47 @@ +/* + * Cadence - The resource-oriented smart contract programming language + * + * Copyright Dapper Labs, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package migrations + +import ( + "github.com/onflow/atree" + + "github.com/onflow/cadence/runtime/interpreter" +) + +// ShouldFixBrokenCompositeKeyedDictionary returns true if the given value is a dictionary with a composite key type. +// +// It is useful for use with atree's PersistentSlabStorage.FixLoadedBrokenReferences. +// +// NOTE: The intended use case is to enable migration programs in onflow/flow-go to fix broken references. +// As of April 2024, only 10 registers in testnet (not mainnet) were found to have broken references, +// and they seem to have resulted from a bug that was fixed 2 years ago by https://github.com/onflow/cadence/pull/1565. +func ShouldFixBrokenCompositeKeyedDictionary(atreeValue atree.Value) bool { + orderedMap, ok := atreeValue.(*atree.OrderedMap) + if !ok { + return false + } + + dictionaryStaticType, ok := orderedMap.Type().(*interpreter.DictionaryStaticType) + if !ok { + return false + } + + _, ok = dictionaryStaticType.KeyType.(*interpreter.CompositeStaticType) + return ok +} diff --git a/migrations/migration_test.go b/migrations/migration_test.go index 3895e4a8be..73818a8411 100644 --- a/migrations/migration_test.go +++ b/migrations/migration_test.go @@ -19,6 +19,10 @@ package migrations import ( + "bytes" + _ "embed" + "encoding/csv" + "encoding/hex" "errors" "fmt" "testing" @@ -2659,3 +2663,98 @@ func TestDictionaryKeyConflict(t *testing.T) { test(t, false) }) } + +//go:embed testdata/missing-slabs-payloads.csv +var missingSlabsPayloadsData []byte + +// '$' + 8 byte index +const slabKeyLength = 9 + +func isSlabStorageKey(key []byte) bool { + return len(key) == slabKeyLength && key[0] == '$' +} + +func TestFixLoadedBrokenReferences(t *testing.T) { + + t.Parallel() + + // Read CSV file with test data + + reader := csv.NewReader(bytes.NewReader(missingSlabsPayloadsData)) + + // account, key, value + reader.FieldsPerRecord = 3 + + records, err := reader.ReadAll() + require.NoError(t, err) + + // Load data into ledger. Skip header + + ledger := NewTestLedger(nil, nil) + + for _, record := range records[1:] { + account, err := hex.DecodeString(record[0]) + require.NoError(t, err) + + key, err := hex.DecodeString(record[1]) + require.NoError(t, err) + + value, err := hex.DecodeString(record[2]) + require.NoError(t, err) + + err = ledger.SetValue(account, key, value) + require.NoError(t, err) + } + + storage := runtime.NewStorage(ledger, nil) + + // Check health. + // Retrieve all slabs before migration + + err = ledger.ForEach(func(owner, key, value []byte) error { + + if !isSlabStorageKey(key) { + return nil + } + + // Convert the owner/key to a storage ID. + + var storageIndex atree.StorageIndex + copy(storageIndex[:], key[1:]) + + storageID := atree.NewStorageID(atree.Address(owner), storageIndex) + + // Retrieve the slab. + _, _, err = storage.Retrieve(storageID) + require.NoError(t, err) + + return nil + }) + require.NoError(t, err) + + address, err := common.HexToAddress("0x5d63c34d7f05e5a4") + require.NoError(t, err) + + for _, domain := range common.AllPathDomains { + _ = storage.GetStorageMap(address, domain.Identifier(), false) + } + + err = storage.CheckHealth() + require.Error(t, err) + + require.ErrorContains(t, err, "slab (0x0.49) not found: slab not found during slab iteration") + + // Fix the broken slab references + + fixedSlabs, skippedSlabIDs, err := storage.PersistentSlabStorage. + FixLoadedBrokenReferences(ShouldFixBrokenCompositeKeyedDictionary) + require.NoError(t, err) + + require.NotEmpty(t, fixedSlabs) + require.Empty(t, skippedSlabIDs) + + // Re-run health check. This time it should pass. + + err = storage.CheckHealth() + require.NoError(t, err) +} diff --git a/migrations/testdata/missing-slabs-payloads.csv b/migrations/testdata/missing-slabs-payloads.csv new file mode 100644 index 0000000000..c64aa50fa8 --- /dev/null +++ b/migrations/testdata/missing-slabs-payloads.csv @@ -0,0 +1,25 @@ +owner,key,value +5d63c34d7f05e5a4,240000000000000009,008883d88483d8c082487e60df042a9c086869466c6f77546f6b656e6f466c6f77546f6b656e2e5661756c7402021b6f605c4ae015732a008883005b00000000000000100727f5bf434b68c18638e048b5b7cdb49b0000000000000002826475756964d8a41a00a4a65d826762616c616e6365d8bc1b0000001748783b3a +5d63c34d7f05e5a4,612e73,0000000000000012b9000000000000002b0000000000000001 +5d63c34d7f05e5a4,24000000000000000f,00c883d8d982d8d41830d8d582d8c08248631e88ae7f1d7c20704e6f6e46756e6769626c65546f6b656e744e6f6e46756e6769626c65546f6b656e2e4e4654031bb9d0e9f36650574100c883005b00000000000000180a5d0065f54d3b70efeefcbe52b8e6bbfe71d66e8625c1019b000000000000000382d8a419bfedd8ff505d63c34d7f05e5a4000000000000001682d8a419bfecd8ff505d63c34d7f05e5a4000000000000001082d8a419c026d8ff505d63c34d7f05e5a40000000000000028 +5d63c34d7f05e5a4,240000000000000005,00c883d88483d8c08248a47a2d3a3b7e9133734469676974616c436f6e74656e744173736574781b4469676974616c436f6e74656e7441737365742e4e46544461746101041bdb477d045820e23f00c883005b000000000000002001e9dead718715827af7a9558c406fd09c308d168ef2bca4bbd2c23d2c1c98209b0000000000000004826c73657269616c4e756d626572d8a30182666974656d4964d8876e746573742d6974656d2d69642d3282686d65746164617461d8ff505d63c34d7f05e5a40000000000000006826b6974656d56657273696f6ed8a301 +5d63c34d7f05e5a4,240000000000000006,008883d8d982d8d408d8d408011bd3ad92ee82a688cf008883005b00000000000000087d4660d52656b0f19b000000000000000182d887666578496e666fd887781b4164646974696f6e616c20696e666f206561636820746f6b656e2e +5d63c34d7f05e5a4,24000000000000002a,008883d8d982d8d408d8d408001b7d63d5c2a25d9570008883005b00000000000000009b0000000000000000 +5d63c34d7f05e5a4,24000000000000000d,008883f6051bbe10d8ef134f8081008883005b00000000000000283c74aa8be0b9f0e484cd59f126f4e5908c69f0b6aca96e5f9c1af60840e775e3c06c0966bef02b939b0000000000000005826d444341436f6c6c656374696f6ed8cb82d8c882016d444341436f6c6c656374696f6ed8db82f4d8dc82d8d40581d8d682d8c08248a47a2d3a3b7e9133734469676974616c436f6e74656e74417373657478244469676974616c436f6e74656e7441737365742e436f6c6c656374696f6e5075626c6963827046616e546f705065726d697373696f6ed8cb82d8c882017046616e546f705065726d697373696f6ed8db82f4d8dc82d8d40581d8d682d8c08248a47a2d3a3b7e91337046616e546f705065726d697373696f6e781946616e546f705065726d697373696f6e2e52656365697665728270666c6f77546f6b656e42616c616e6365d8cb82d8c882016e666c6f77546f6b656e5661756c74d8db82f4d8dc82d8d582d8c082487e60df042a9c086869466c6f77546f6b656e6f466c6f77546f6b656e2e5661756c7481d8d682d8c082489a0766d93b6608b76d46756e6769626c65546f6b656e7546756e6769626c65546f6b656e2e42616c616e63658271666c6f77546f6b656e5265636569766572d8cb82d8c882016e666c6f77546f6b656e5661756c74d8db82f4d8dc82d8d582d8c082487e60df042a9c086869466c6f77546f6b656e6f466c6f77546f6b656e2e5661756c7481d8d682d8c082489a0766d93b6608b76d46756e6769626c65546f6b656e7646756e6769626c65546f6b656e2e5265636569766572827546616e546f70546f6b656e436f6c6c656374696f6ed8cb82d8c882017546616e546f70546f6b656e436f6c6c656374696f6ed8db82f4d8dc82d8d40581d8d682d8c08248a47a2d3a3b7e91336b46616e546f70546f6b656e781c46616e546f70546f6b656e2e436f6c6c656374696f6e5075626c6963 +5d63c34d7f05e5a4,7075626c6963,000000000000000d +5d63c34d7f05e5a4,240000000000000029,00c883d88483d8c08248a47a2d3a3b7e91336b46616e546f70546f6b656e7346616e546f70546f6b656e2e4e46544461746101041bee2eefad66dc79bb00c883005b000000000000002038fc822d312df7f38ef037256b43eed596a75665ca80dfe6aec0bc45428aa9e59b0000000000000004826b6974656d56657273696f6ed8a301826c73657269616c4e756d626572d8a30182686d65746164617461d8ff505d63c34d7f05e5a4000000000000002a82666974656d4964d887781a746573742d6974656d2d69642d31363534393133393635393138 +5d63c34d7f05e5a4,73746f72616765,000000000000000c +5d63c34d7f05e5a4,24000000000000000c,00c883f6041bd6bd24f14bb87b1d00c883005b000000000000002011e0799c7eee7e48e100d7590eb37bbcfc5fbcac8f0e7c49fdc92797c5d69e9f9b0000000000000004826e666c6f77546f6b656e5661756c74d8ff505d63c34d7f05e5a40000000000000009827546616e546f70546f6b656e436f6c6c656374696f6ed8ff505d63c34d7f05e5a4000000000000000e827046616e546f705065726d697373696f6ed8ff505d63c34d7f05e5a4000000000000000a826d444341436f6c6c656374696f6ed8ff505d63c34d7f05e5a40000000000000008 +5d63c34d7f05e5a4,240000000000000028,00c883d88483d8c08248a47a2d3a3b7e91336b46616e546f70546f6b656e6f46616e546f70546f6b656e2e4e465402041bb77700225b8d6b5200c883005b00000000000000202183e3a62217fe8a36d1395a6a775e199d29ed46c773a0acc6d7ec7f9e97bfc79b0000000000000004826464617461d8ff505d63c34d7f05e5a40000000000000029826475756964d8a41a05cb6bc982657265664964d8877819746573742d7265662d69642d3136353439313339363539313882626964d8a419c026 +5d63c34d7f05e5a4,240000000000000016,00c883d88483d8c08248a47a2d3a3b7e91336b46616e546f70546f6b656e6f46616e546f70546f6b656e2e4e465402041bb77700225b8d6b5200c883005b00000000000000202183e3a62217fe8a36d1395a6a775e199d29ed46c773a0acc6d7ec7f9e97bfc79b0000000000000004826464617461d8ff505d63c34d7f05e5a40000000000000017826475756964d8a41a05c8448882657265664964d8877819746573742d7265662d69642d3136353436363835373330383182626964d8a419bfed +5d63c34d7f05e5a4,24000000000000000b,00c883d8d982d8d582d8c08248a47a2d3a3b7e91337046616e546f705065726d697373696f6e7546616e546f705065726d697373696f6e2e526f6c65d8ddf6011b535c9de83a38cab000c883005b00000000000000089bd8ae8dd553d9479b000000000000000182d8ff5000000000000000000000000000000031d8c983d88348a47a2d3a3b7e9133d8c882026c46616e546f704d696e746572d8db82f4d8d582d8c08248a47a2d3a3b7e91337046616e546f705065726d697373696f6e7746616e546f705065726d697373696f6e2e4d696e746572 +5d63c34d7f05e5a4,240000000000000007,00c883d8d982d8d41830d8d582d8c08248631e88ae7f1d7c20704e6f6e46756e6769626c65546f6b656e744e6f6e46756e6769626c65546f6b656e2e4e4654011b095f1b185a2800d900c883005b00000000000000085521726dbf8b37549b000000000000000182d8a4182bd8ff505d63c34d7f05e5a40000000000000004 +5d63c34d7f05e5a4,240000000000000004,00c883d88483d8c08248a47a2d3a3b7e9133734469676974616c436f6e74656e744173736574774469676974616c436f6e74656e7441737365742e4e465402041bf62e63ac21a3216600c883005b00000000000000200507c593895292f5ab364b56d50468c7d0766f8c688ebb74f8c1a4433ffbccf29b000000000000000482626964d8a4182b826464617461d8ff505d63c34d7f05e5a40000000000000005826475756964d8a41a00a5641882657265664964d8877830746573742d7265662d69642d61353662363764392d306634352d346339622d613264352d636534306439376339333230 +5d63c34d7f05e5a4,240000000000000008,00c883d88483d8c08248a47a2d3a3b7e9133734469676974616c436f6e74656e744173736574781e4469676974616c436f6e74656e7441737365742e436f6c6c656374696f6e02021bf7f58ec9906ea65700c883005b0000000000000010e5df9d08d0a2430ce91971a214b62eaa9b0000000000000002826475756964d8a41a00a4a66682696f776e65644e465473d8ff505d63c34d7f05e5a40000000000000007 +5d63c34d7f05e5a4,240000000000000017,00c883d88483d8c08248a47a2d3a3b7e91336b46616e546f70546f6b656e7346616e546f70546f6b656e2e4e46544461746101041bee2eefad66dc79bb00c883005b000000000000002038fc822d312df7f38ef037256b43eed596a75665ca80dfe6aec0bc45428aa9e59b0000000000000004826b6974656d56657273696f6ed8a301826c73657269616c4e756d626572d8a30182686d65746164617461d8ff505d63c34d7f05e5a4000000000000001882666974656d4964d887781a746573742d6974656d2d69642d31363534363638353733303831 +5d63c34d7f05e5a4,240000000000000018,008883d8d982d8d408d8d408001b7d63d5c2a25d9570008883005b00000000000000009b0000000000000000 +5d63c34d7f05e5a4,240000000000000010,00c883d88483d8c08248a47a2d3a3b7e91336b46616e546f70546f6b656e6f46616e546f70546f6b656e2e4e465402041bb77700225b8d6b5200c883005b00000000000000202183e3a62217fe8a36d1395a6a775e199d29ed46c773a0acc6d7ec7f9e97bfc79b0000000000000004826464617461d8ff505d63c34d7f05e5a40000000000000011826475756964d8a41a05c838bf82657265664964d8877819746573742d7265662d69642d3136353436363539323439343382626964d8a419bfec +5d63c34d7f05e5a4,240000000000000011,00c883d88483d8c08248a47a2d3a3b7e91336b46616e546f70546f6b656e7346616e546f70546f6b656e2e4e46544461746101041bee2eefad66dc79bb00c883005b000000000000002038fc822d312df7f38ef037256b43eed596a75665ca80dfe6aec0bc45428aa9e59b0000000000000004826b6974656d56657273696f6ed8a301826c73657269616c4e756d626572d8a30182686d65746164617461d8ff505d63c34d7f05e5a4000000000000001282666974656d4964d887781a746573742d6974656d2d69642d31363534363635393234393433 +5d63c34d7f05e5a4,24000000000000000a,00c883d88483d8c08248a47a2d3a3b7e91337046616e546f705065726d697373696f6e7746616e546f705065726d697373696f6e2e486f6c64657202031bb9d0e9f36650574100c883005b000000000000001820d6c23f2e85e694b0070dbc21a9822de5725916c4a005e99b0000000000000003826475756964d8a41a00d858bc8265726f6c6573d8ff505d63c34d7f05e5a4000000000000000b8269726563697069656e74d883485d63c34d7f05e5a4 +5d63c34d7f05e5a4,240000000000000012,008883d8d982d8d408d8d408001b7d63d5c2a25d9570008883005b00000000000000009b0000000000000000 +5d63c34d7f05e5a4,24000000000000000e,00c883d88483d8c08248a47a2d3a3b7e91336b46616e546f70546f6b656e7646616e546f70546f6b656e2e436f6c6c656374696f6e02021b85d7c70d054429ef00c883005b000000000000001012dab75ddc75f021eec8d5b5338fb70a9b000000000000000282696f776e65644e465473d8ff505d63c34d7f05e5a4000000000000000f826475756964d8a41a05c83883 diff --git a/runtime/tests/runtime_utils/testledger.go b/runtime/tests/runtime_utils/testledger.go index 082936a253..86b1e9bff7 100644 --- a/runtime/tests/runtime_utils/testledger.go +++ b/runtime/tests/runtime_utils/testledger.go @@ -22,6 +22,7 @@ import ( "encoding/binary" "encoding/hex" "fmt" + "sort" "strconv" "strings" @@ -54,6 +55,34 @@ func (s TestLedger) AllocateStorageIndex(owner []byte) (atree.StorageIndex, erro return s.OnAllocateStorageIndex(owner) } +const testLedgerKeySeparator = "|" + +func (s TestLedger) ForEach(f func(owner, key, value []byte) error) error { + + var keys []string + for key := range s.StoredValues { //nolint:maprange + keys = append(keys, key) + } + sort.Strings(keys) + + for _, key := range keys { //nolint:maprange + value := s.StoredValues[key] + + keyParts := strings.Split(key, testLedgerKeySeparator) + owner := []byte(keyParts[0]) + key := []byte(keyParts[1]) + + if err := f(owner, key, value); err != nil { + return err + } + } + return nil +} + +func TestStorageKey(owner, key string) string { + return strings.Join([]string{owner, key}, testLedgerKeySeparator) +} + func (s TestLedger) Dump() { // Only used for testing/debugging purposes for key, data := range s.StoredValues { //nolint:maprange @@ -68,10 +97,6 @@ func NewTestLedger( onWrite func(owner, key, value []byte), ) TestLedger { - storageKey := func(owner, key string) string { - return strings.Join([]string{owner, key}, "|") - } - storedValues := map[string][]byte{} storageIndices := map[string]uint64{} @@ -79,18 +104,18 @@ func NewTestLedger( return TestLedger{ StoredValues: storedValues, OnValueExists: func(owner, key []byte) (bool, error) { - value := storedValues[storageKey(string(owner), string(key))] + value := storedValues[TestStorageKey(string(owner), string(key))] return len(value) > 0, nil }, OnGetValue: func(owner, key []byte) (value []byte, err error) { - value = storedValues[storageKey(string(owner), string(key))] + value = storedValues[TestStorageKey(string(owner), string(key))] if onRead != nil { onRead(owner, key, value) } return value, nil }, OnSetValue: func(owner, key, value []byte) (err error) { - storedValues[storageKey(string(owner), string(key))] = value + storedValues[TestStorageKey(string(owner), string(key))] = value if onWrite != nil { onWrite(owner, key, value) }