diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0691820c6..db4a222a0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,6 +25,7 @@ jobs: # integer overflow). - name: test & coverage report creation run: | + cd cmd/legacydump && go build -o legacydump main.go && cd ../.. go test ./... -mod=readonly -timeout 10m -short -race -coverprofile=coverage.txt -covermode=atomic go test ./... -mod=readonly -timeout 15m GOARCH=386 go test ./... -mod=readonly -timeout 15m diff --git a/.gitignore b/.gitignore index 7fff5ca4f..28f008576 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +cmd/legacydump/legacydump vendor .glide *.swp diff --git a/Makefile b/Makefile index 7eb7e6ff8..1a2d937e7 100644 --- a/Makefile +++ b/Makefile @@ -24,9 +24,12 @@ test-short: @go test ./... $(LDFLAGS) -v --race --short .PHONY: test-short -test: +cmd/legacydump/legacydump: + cd cmd/legacydump && go build -o legacydump main.go + +test: cmd/legacydump/legacydump @echo "--> Running go test" - @go test ./... $(LDFLAGS) -v + @go test ./... $(LDFLAGS) .PHONY: test format: diff --git a/cmd/legacydump/legacydump b/cmd/legacydump/legacydump deleted file mode 100755 index 9992570af..000000000 Binary files a/cmd/legacydump/legacydump and /dev/null differ diff --git a/docs/node/nodedb.md b/docs/node/nodedb.md index 04aef2a4c..f5abc447b 100644 --- a/docs/node/nodedb.md +++ b/docs/node/nodedb.md @@ -19,7 +19,7 @@ When a version `v` is deleted, all nodes which removed in the current version wi ```golang // DeleteVersionsFrom permanently deletes all tree versions from the given version upwards. func (ndb *nodeDB) DeleteVersionsFrom(fromVersion int64) error { - latest, err := ndb.getLatestVersion() + _, latest, err := ndb.getLatestVersion() if err != nil { return err } diff --git a/immutable_tree.go b/immutable_tree.go index 54ccdf991..b3b640ec3 100644 --- a/immutable_tree.go +++ b/immutable_tree.go @@ -301,7 +301,7 @@ func (t *ImmutableTree) IsFastCacheEnabled() (bool, error) { } func (t *ImmutableTree) isLatestTreeVersion() (bool, error) { - latestVersion, err := t.ndb.getLatestVersion() + _, latestVersion, err := t.ndb.getLatestVersion() if err != nil { return false, err } diff --git a/iterator_test.go b/iterator_test.go index c4b03f2e5..06262d502 100644 --- a/iterator_test.go +++ b/iterator_test.go @@ -182,7 +182,7 @@ func TestIterator_WithDelete_Full_Ascending_Success(t *testing.T) { err = tree.DeleteVersionsTo(1) require.NoError(t, err) - latestVersion, err := tree.ndb.getLatestVersion() + _, latestVersion, err := tree.ndb.getLatestVersion() require.NoError(t, err) immutableTree, err := tree.GetImmutable(latestVersion) require.NoError(t, err) @@ -253,7 +253,7 @@ func setupIteratorAndMirror(t *testing.T, config *iteratorTestConfig) (corestore _, _, err := tree.SaveVersion() require.NoError(t, err) - latestVersion, err := tree.ndb.getLatestVersion() + _, latestVersion, err := tree.ndb.getLatestVersion() require.NoError(t, err) immutableTree, err := tree.GetImmutable(latestVersion) require.NoError(t, err) diff --git a/migrate_test.go b/migrate_test.go index 573cbfe75..a4403e2f2 100644 --- a/migrate_test.go +++ b/migrate_test.go @@ -177,7 +177,7 @@ func TestDeleteVersions(t *testing.T) { // Test LoadVersionForOverwriting for the legacy version err = tree.LoadVersionForOverwriting(int64(targetVersion)) require.NoError(t, err) - latestVersion, err := tree.ndb.getLatestVersion() + _, latestVersion, err := tree.ndb.getLatestVersion() require.NoError(t, err) require.Equal(t, int64(targetVersion), latestVersion) legacyLatestVersion, err := tree.ndb.getLegacyLatestVersion() diff --git a/mutable_tree.go b/mutable_tree.go index b26f0c8ab..ae7dc6280 100644 --- a/mutable_tree.go +++ b/mutable_tree.go @@ -40,6 +40,7 @@ type MutableTree struct { unsavedFastNodeRemovals *sync.Map // map[string]interface{} FastNodes that have not yet been removed from disk ndb *nodeDB skipFastStorageUpgrade bool // If true, the tree will work like no fast storage and always not upgrade fast storage + initialVersionSet bool mtx sync.Mutex } @@ -62,6 +63,7 @@ func NewMutableTree(db corestore.KVStoreWithBatch, cacheSize int, skipFastStorag unsavedFastNodeRemovals: &sync.Map{}, ndb: ndb, skipFastStorageUpgrade: skipFastStorageUpgrade, + initialVersionSet: opts.initialVersionSet, } } @@ -73,7 +75,8 @@ func (tree *MutableTree) IsEmpty() bool { // GetLatestVersion returns the latest version of the tree. func (tree *MutableTree) GetLatestVersion() (int64, error) { - return tree.ndb.getLatestVersion() + _, v, err := tree.ndb.getLatestVersion() + return v, err } // VersionExists returns whether or not a version exists. @@ -90,10 +93,13 @@ func (tree *MutableTree) VersionExists(version int64) bool { if err != nil { return false } - latestVersion, err := tree.ndb.getLatestVersion() + found, latestVersion, err := tree.ndb.getLatestVersion() if err != nil { return false } + if !found { + return false + } return firstVersion <= version && version <= latestVersion } @@ -104,7 +110,7 @@ func (tree *MutableTree) AvailableVersions() []int { if err != nil { return nil } - latestVersion, err := tree.ndb.getLatestVersion() + _, latestVersion, err := tree.ndb.getLatestVersion() if err != nil { return nil } @@ -146,7 +152,7 @@ func (tree *MutableTree) WorkingHash() []byte { func (tree *MutableTree) WorkingVersion() int64 { version := tree.version + 1 - if version == 1 && tree.ndb.opts.InitialVersion > 0 { + if version == 1 && tree.initialVersionSet { version = int64(tree.ndb.opts.InitialVersion) } return version @@ -449,21 +455,16 @@ func (tree *MutableTree) LoadVersion(targetVersion int64) (int64, error) { tree.ndb.opts.InitialVersion, firstVersion) } - latestVersion, err := tree.ndb.getLatestVersion() + ok, latestVersion, err := tree.ndb.getLatestVersion() if err != nil { return 0, err } - if firstVersion > 0 && firstVersion < int64(tree.ndb.opts.InitialVersion) { - return latestVersion, fmt.Errorf("initial version set to %v, but found earlier version %v", - tree.ndb.opts.InitialVersion, firstVersion) - } - if latestVersion < targetVersion { return latestVersion, fmt.Errorf("wanted to load target %d but only found up to %d", targetVersion, latestVersion) } - if firstVersion == 0 { + if !ok { if targetVersion <= 0 { if !tree.skipFastStorageUpgrade { tree.mtx.Lock() @@ -604,7 +605,7 @@ func (tree *MutableTree) enableFastStorageAndCommit() error { return err } - latestVersion, err := tree.ndb.getLatestVersion() + _, latestVersion, err := tree.ndb.getLatestVersion() if err != nil { return err } @@ -707,6 +708,7 @@ func (tree *MutableTree) UnsetCommitting() { // the tree. Returns the hash and new version number. func (tree *MutableTree) SaveVersion() ([]byte, int64, error) { version := tree.WorkingVersion() + tree.initialVersionSet = false if tree.VersionExists(version) { // If the version already exists, return an error as we're attempting to overwrite. @@ -871,6 +873,7 @@ func (tree *MutableTree) saveFastNodeRemovals() error { // and is otherwise ignored. func (tree *MutableTree) SetInitialVersion(version uint64) { tree.ndb.opts.InitialVersion = version + tree.initialVersionSet = true } // DeleteVersionsTo removes versions upto the given version from the MutableTree. diff --git a/mutable_tree_test.go b/mutable_tree_test.go index 78f153fdd..3d92f888c 100644 --- a/mutable_tree_test.go +++ b/mutable_tree_test.go @@ -769,6 +769,8 @@ func TestUpgradeStorageToFast_LatestVersion_Success(t *testing.T) { require.False(t, isUpgradeable) require.NoError(t, err) + _, _, err = tree.SaveVersion() + require.NoError(t, err) isFastCacheEnabled, err = tree.IsFastCacheEnabled() require.NoError(t, err) require.True(t, isFastCacheEnabled) @@ -907,7 +909,7 @@ func TestFastStorageReUpgradeProtection_NoForceUpgrade_Success(t *testing.T) { // Pretend that we called Load and have the latest state in the tree tree.version = latestTreeVersion - latestVersion, err := tree.ndb.getLatestVersion() + _, latestVersion, err := tree.ndb.getLatestVersion() require.NoError(t, err) require.Equal(t, latestVersion, int64(latestTreeVersion)) @@ -1001,7 +1003,7 @@ func TestFastStorageReUpgradeProtection_ForceUpgradeFirstTime_NoForceSecondTime_ // Pretend that we called Load and have the latest state in the tree tree.version = latestTreeVersion - latestVersion, err := tree.ndb.getLatestVersion() + _, latestVersion, err := tree.ndb.getLatestVersion() require.NoError(t, err) require.Equal(t, latestVersion, int64(latestTreeVersion)) @@ -1479,3 +1481,16 @@ func TestMutableTreeClose(t *testing.T) { require.NoError(t, tree.Close()) } + +func TestMutableTree_InitialVersionZero(t *testing.T) { + db := dbm.NewMemDB() + + tree := NewMutableTree(db, 0, false, NewNopLogger(), InitialVersionOption(0)) + + _, err := tree.Set([]byte("hello"), []byte("world")) + require.NoError(t, err) + + _, version, err := tree.SaveVersion() + require.NoError(t, err) + require.Equal(t, int64(0), version) +} diff --git a/nodedb.go b/nodedb.go index 829c5697d..0eb7003c7 100644 --- a/nodedb.go +++ b/nodedb.go @@ -331,7 +331,7 @@ func (ndb *nodeDB) shouldForceFastStorageUpgrade() (bool, error) { versions := strings.Split(ndb.storageVersion, fastStorageVersionDelimiter) if len(versions) == 2 { - latestVersion, err := ndb.getLatestVersion() + _, latestVersion, err := ndb.getLatestVersion() if err != nil { // TODO: should be true or false as default? (removed panic here) return false, err @@ -525,7 +525,7 @@ func (ndb *nodeDB) deleteLegacyVersions(legacyLatestVersion int64) error { // DeleteVersionsFrom permanently deletes all tree versions from the given version upwards. func (ndb *nodeDB) DeleteVersionsFrom(fromVersion int64) error { - latest, err := ndb.getLatestVersion() + _, latest, err := ndb.getLatestVersion() if err != nil { return err } @@ -642,7 +642,7 @@ func (ndb *nodeDB) deleteVersionsTo(toVersion int64) error { return err } - latest, err := ndb.getLatestVersion() + _, latest, err := ndb.getLatestVersion() if err != nil { return err } @@ -727,7 +727,7 @@ func (ndb *nodeDB) getFirstVersion() (int64, error) { return version, nil } // Find the first version - latestVersion, err := ndb.getLatestVersion() + _, latestVersion, err := ndb.getLatestVersion() if err != nil { return 0, err } @@ -797,13 +797,13 @@ func (ndb *nodeDB) resetLegacyLatestVersion(version int64) { ndb.legacyLatestVersion = version } -func (ndb *nodeDB) getLatestVersion() (int64, error) { +func (ndb *nodeDB) getLatestVersion() (bool, int64, error) { ndb.mtx.Lock() latestVersion := ndb.latestVersion ndb.mtx.Unlock() if latestVersion > 0 { - return latestVersion, nil + return true, latestVersion, nil } itr, err := ndb.db.ReverseIterator( @@ -811,7 +811,7 @@ func (ndb *nodeDB) getLatestVersion() (int64, error) { nodeKeyPrefixFormat.KeyInt64(int64(math.MaxInt64)), ) if err != nil { - return 0, err + return false, 0, err } defer itr.Close() @@ -821,24 +821,25 @@ func (ndb *nodeDB) getLatestVersion() (int64, error) { nodeKeyFormat.Scan(k, &nk) latestVersion = GetNodeKey(nk).version ndb.resetLatestVersion(latestVersion) - return latestVersion, nil + return true, latestVersion, nil } if err := itr.Error(); err != nil { - return 0, err + return false, 0, err } // If there are no versions, try to get the latest version from the legacy format. latestVersion, err = ndb.getLegacyLatestVersion() if err != nil { - return 0, err + return false, 0, err } if latestVersion > 0 { ndb.resetLatestVersion(latestVersion) - return latestVersion, nil + return true, latestVersion, nil } - return 0, nil + return false, 0, nil + // return -1, nil } func (ndb *nodeDB) resetLatestVersion(version int64) { @@ -1246,7 +1247,7 @@ func (ndb *nodeDB) traverseStateChanges(startVersion, endVersion int64, fn func( if startVersion < firstVersion { startVersion = firstVersion } - latestVersion, err := ndb.getLatestVersion() + _, latestVersion, err := ndb.getLatestVersion() if err != nil { return err } diff --git a/nodedb_test.go b/nodedb_test.go index a1f67d96e..170261110 100644 --- a/nodedb_test.go +++ b/nodedb_test.go @@ -87,7 +87,7 @@ func TestSetStorageVersion_Success(t *testing.T) { ndb := newNodeDB(db, 0, DefaultOptions(), NewNopLogger()) require.Equal(t, defaultStorageVersionValue, ndb.getStorageVersion()) - latestVersion, err := ndb.getLatestVersion() + _, latestVersion, err := ndb.getLatestVersion() require.NoError(t, err) err = ndb.SetFastStorageVersionToBatch(latestVersion) @@ -404,7 +404,7 @@ func TestDeleteVersionsFromNoDeadlock(t *testing.T) { err := ndb.SetFastStorageVersionToBatch(ndb.latestVersion) require.NoError(t, err) - latestVersion, err := ndb.getLatestVersion() + _, latestVersion, err := ndb.getLatestVersion() require.NoError(t, err) require.Equal(t, expectedVersion+fastStorageVersionDelimiter+strconv.Itoa(int(latestVersion)), ndb.getStorageVersion()) require.NoError(t, ndb.batch.Write()) diff --git a/options.go b/options.go index 520c2170f..ca65d0ea4 100644 --- a/options.go +++ b/options.go @@ -87,6 +87,8 @@ type Options struct { // AsyncPruning is a flag to enable async pruning AsyncPruning bool + + initialVersionSet bool } // DefaultOptions returns the default options for IAVL. @@ -105,6 +107,7 @@ func SyncOption(sync bool) Option { func InitialVersionOption(iv uint64) Option { return func(opts *Options) { opts.InitialVersion = iv + opts.initialVersionSet = true } } diff --git a/tree_random_test.go b/tree_random_test.go index 02aaede3c..2f583e0e5 100644 --- a/tree_random_test.go +++ b/tree_random_test.go @@ -283,7 +283,7 @@ func assertEmptyDatabase(t *testing.T, tree *MutableTree) { storageVersionValue, err := tree.ndb.db.Get([]byte(firstKey)) require.NoError(t, err) - latestVersion, err := tree.ndb.getLatestVersion() + _, latestVersion, err := tree.ndb.getLatestVersion() require.NoError(t, err) require.Equal(t, fastStorageVersionValue+fastStorageVersionDelimiter+strconv.Itoa(int(latestVersion)), string(storageVersionValue)) @@ -347,7 +347,7 @@ func assertMirror(t *testing.T, tree *MutableTree, mirror map[string]string, ver // Checks that fast node cache matches live state. func assertFastNodeCacheIsLive(t *testing.T, tree *MutableTree, mirror map[string]string, version int64) { - latestVersion, err := tree.ndb.getLatestVersion() + _, latestVersion, err := tree.ndb.getLatestVersion() require.NoError(t, err) if latestVersion != version { // The fast node cache check should only be done to the latest version @@ -364,7 +364,7 @@ func assertFastNodeCacheIsLive(t *testing.T, tree *MutableTree, mirror map[strin // Checks that fast nodes on disk match live state. func assertFastNodeDiskIsLive(t *testing.T, tree *MutableTree, mirror map[string]string, version int64) { - latestVersion, err := tree.ndb.getLatestVersion() + _, latestVersion, err := tree.ndb.getLatestVersion() require.NoError(t, err) if latestVersion != version { // The fast node disk check should only be done to the latest version