Skip to content

Commit

Permalink
add Repair013Orphans to repair 0.13 database orphans (#301)
Browse files Browse the repository at this point in the history
Cherry-picked from 2142759 in 0.14.x branch.
  • Loading branch information
erikgrinaker authored Jul 30, 2020
1 parent d750a99 commit 49b2749
Show file tree
Hide file tree
Showing 12 changed files with 273 additions and 0 deletions.
68 changes: 68 additions & 0 deletions repair.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
package iavl

import (
"math"

"github.com/pkg/errors"
dbm "github.com/tendermint/tm-db"
)

// Repair013Orphans repairs incorrect orphan entries written by IAVL 0.13 pruning. To use it, close
// a database using IAVL 0.13, make a backup copy, and then run this function before opening the
// database with IAVL 0.14 or later. It returns the number of faulty orphan entries removed. If the
// 0.13 database was written with KeepEvery:1 (the default) or the last version _ever_ saved to the
// tree was a multiple of `KeepEvery` and thus saved to disk, this repair is not necessary.
//
// Note that this cannot be used directly on Cosmos SDK databases, since they store multiple IAVL
// trees in the same underlying database via a prefix scheme.
//
// The pruning functionality enabled with Options.KeepEvery > 1 would write orphans entries to disk
// for versions that should only have been saved in memory, and these orphan entries were clamped
// to the last version persisted to disk instead of the version that generated them (so a delete at
// version 749 might generate an orphan entry ending at version 700 for KeepEvery:100). If the
// database is reopened at the last persisted version and this version is later deleted, the
// orphaned nodes can be deleted prematurely or incorrectly, causing data loss and database
// corruption.
//
// This function removes these incorrect orphan entries by deleting all orphan entries that have a
// to-version equal to or greater than the latest persisted version. Correct orphans will never
// have this, since they must have been deleted in a future (non-existent) version for that to be
// the case.
func Repair013Orphans(db dbm.DB) (uint64, error) {
ndb := newNodeDB(db, 0, &Options{Sync: true})
version := ndb.getLatestVersion()
if version == 0 {
return 0, errors.New("no versions found")
}

var (
repaired uint64
err error
)
batch := db.NewBatch()
defer batch.Close()
ndb.traverseRange(orphanKeyFormat.Key(version), orphanKeyFormat.Key(math.MaxInt64), func(k, v []byte) {
// Sanity check so we don't remove stuff we shouldn't
var toVersion int64
orphanKeyFormat.Scan(k, &toVersion)
if toVersion < version {
err = errors.Errorf("Found unexpected orphan with toVersion=%v, lesser than latest version %v",
toVersion, version)
return
}
repaired++
err = batch.Delete(k)
if err != nil {
return
}
})
if err != nil {
return 0, err
}
err = batch.WriteSync()
if err != nil {
return 0, err
}

return repaired, nil
}
191 changes: 191 additions & 0 deletions repair_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,191 @@
package iavl

import (
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"testing"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
dbm "github.com/tendermint/tm-db"
)

func TestRepair013Orphans(t *testing.T) {
dir, err := ioutil.TempDir("", "test-iavl-repair")
require.NoError(t, err)
defer os.RemoveAll(dir)

// There is also 0.13-orphans-v6.db containing a database closed immediately after writing
// version 6, which should not contain any broken orphans.
err = copyDB("testdata/0.13-orphans.db", filepath.Join(dir, "0.13-orphans.db"))
require.NoError(t, err)

db, err := dbm.NewGoLevelDB("0.13-orphans", dir)
require.NoError(t, err)

// Repair the database.
repaired, err := Repair013Orphans(db)
require.NoError(t, err)
assert.EqualValues(t, 8, repaired)

// Load the database.
tree, err := NewMutableTreeWithOpts(db, 0, &Options{Sync: true})
require.NoError(t, err)
version, err := tree.Load()
require.NoError(t, err)
require.EqualValues(t, 6, version)

// We now generate two empty versions, and check all persisted versions.
_, version, err = tree.SaveVersion()
require.NoError(t, err)
require.EqualValues(t, 7, version)
_, version, err = tree.SaveVersion()
require.NoError(t, err)
require.EqualValues(t, 8, version)

// Check all persisted versions.
require.Equal(t, []int{3, 6, 7, 8}, tree.AvailableVersions())
assertVersion(t, tree, 0)
assertVersion(t, tree, 3)
assertVersion(t, tree, 6)
assertVersion(t, tree, 7)
assertVersion(t, tree, 8)

// We then delete version 6 (the last persisted one with 0.13).
err = tree.DeleteVersion(6)
require.NoError(t, err)

// Reading "rm7" (which should not have been deleted now) would panic with a broken database.
_, value := tree.Get([]byte("rm7"))
require.Equal(t, []byte{1}, value)

// Check all persisted versions.
require.Equal(t, []int{3, 7, 8}, tree.AvailableVersions())
assertVersion(t, tree, 0)
assertVersion(t, tree, 3)
assertVersion(t, tree, 7)
assertVersion(t, tree, 8)

// Delete all historical versions, and check the latest.
err = tree.DeleteVersion(3)
require.NoError(t, err)
err = tree.DeleteVersion(7)
require.NoError(t, err)

require.Equal(t, []int{8}, tree.AvailableVersions())
assertVersion(t, tree, 0)
assertVersion(t, tree, 8)
}

// assertVersion checks the given version (or current if 0) against the expected values.
func assertVersion(t *testing.T, tree *MutableTree, version int64) {
var err error
itree := tree.ImmutableTree
if version > 0 {
itree, err = tree.GetImmutable(version)
require.NoError(t, err)
}
version = itree.version

// The "current" value should have the current version for <= 6, then 6 afterwards
_, value := itree.Get([]byte("current"))
if version >= 6 {
require.EqualValues(t, []byte{6}, value)
} else {
require.EqualValues(t, []byte{byte(version)}, value)
}

// The "addX" entries should exist for 1-6 in the respective versions, and the
// "rmX" entries should have been removed for 1-6 in the respective versions.
for i := byte(1); i < 8; i++ {
_, value = itree.Get([]byte(fmt.Sprintf("add%v", i)))
if i <= 6 && int64(i) <= version {
require.Equal(t, []byte{i}, value)
} else {
require.Nil(t, value)
}

_, value = itree.Get([]byte(fmt.Sprintf("rm%v", i)))
if i <= 6 && version >= int64(i) {
require.Nil(t, value)
} else {
require.Equal(t, []byte{1}, value)
}
}
}

// Generate013Orphans generates a GoLevelDB orphan database in testdata/0.13-orphans.db
// for testing Repair013Orphans(). It must be run with IAVL 0.13.x.
/*func TestGenerate013Orphans(t *testing.T) {
err := os.RemoveAll("testdata/0.13-orphans.db")
require.NoError(t, err)
db, err := dbm.NewGoLevelDB("0.13-orphans", "testdata")
require.NoError(t, err)
tree, err := NewMutableTreeWithOpts(db, dbm.NewMemDB(), 0, &Options{
KeepEvery: 3,
KeepRecent: 1,
Sync: true,
})
require.NoError(t, err)
version, err := tree.Load()
require.NoError(t, err)
require.EqualValues(t, 0, version)
// We generate 8 versions. In each version, we create a "addX" key, delete a "rmX" key,
// and update the "current" key, where "X" is the current version. Values are the version in
// which the key was last set.
tree.Set([]byte("rm1"), []byte{1})
tree.Set([]byte("rm2"), []byte{1})
tree.Set([]byte("rm3"), []byte{1})
tree.Set([]byte("rm4"), []byte{1})
tree.Set([]byte("rm5"), []byte{1})
tree.Set([]byte("rm6"), []byte{1})
tree.Set([]byte("rm7"), []byte{1})
tree.Set([]byte("rm8"), []byte{1})
for v := byte(1); v <= 8; v++ {
tree.Set([]byte("current"), []byte{v})
tree.Set([]byte(fmt.Sprintf("add%v", v)), []byte{v})
tree.Remove([]byte(fmt.Sprintf("rm%v", v)))
_, version, err = tree.SaveVersion()
require.NoError(t, err)
require.EqualValues(t, v, version)
}
// At this point, the database will contain incorrect orphans in version 6 that, when
// version 6 is deleted, will cause "current", "rm7", and "rm8" to go missing.
}*/

// copyDB makes a shallow copy of the source database directory.
func copyDB(src, dest string) error {
entries, err := ioutil.ReadDir(src)
if err != nil {
return err
}
err = os.MkdirAll(dest, 0777)
if err != nil {
return err
}
for _, entry := range entries {
out, err := os.Create(filepath.Join(dest, entry.Name()))
if err != nil {
return err
}
defer out.Close()

in, err := os.Open(filepath.Join(src, entry.Name()))
defer in.Close() // nolint
if err != nil {
return err
}

_, err = io.Copy(out, in)
if err != nil {
return err
}
}
return nil
}
Binary file added testdata/0.13-orphans-v6.db/000001.log
Binary file not shown.
1 change: 1 addition & 0 deletions testdata/0.13-orphans-v6.db/CURRENT
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
MANIFEST-000000
Empty file.
6 changes: 6 additions & 0 deletions testdata/0.13-orphans-v6.db/LOG
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
=============== Jun 25, 2020 (CEST) ===============
14:30:10.673317 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed
14:30:10.688689 db@open opening
14:30:10.689548 version@stat F·[] S·0B[] Sc·[]
14:30:10.702481 db@janitor F·2 G·0
14:30:10.702564 db@open done T·13.82376ms
Binary file added testdata/0.13-orphans-v6.db/MANIFEST-000000
Binary file not shown.
Binary file added testdata/0.13-orphans.db/000001.log
Binary file not shown.
1 change: 1 addition & 0 deletions testdata/0.13-orphans.db/CURRENT
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
MANIFEST-000000
Empty file added testdata/0.13-orphans.db/LOCK
Empty file.
6 changes: 6 additions & 0 deletions testdata/0.13-orphans.db/LOG
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
=============== Jun 25, 2020 (CEST) ===============
13:31:22.162368 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed
13:31:22.173177 db@open opening
13:31:22.173961 version@stat F·[] S·0B[] Sc·[]
13:31:22.189072 db@janitor F·2 G·0
13:31:22.189117 db@open done T·15.875399ms
Binary file added testdata/0.13-orphans.db/MANIFEST-000000
Binary file not shown.

0 comments on commit 49b2749

Please sign in to comment.