From 21427595e542af089991cf68e19c0015c6e44c8e Mon Sep 17 00:00:00 2001 From: Erik Grinaker Date: Fri, 26 Jun 2020 11:32:30 +0200 Subject: [PATCH] add Repair013Orphans to repair 0.13 database orphans (#282) --- CHANGELOG.md | 4 + repair.go | 65 +++++++ repair_test.go | 191 ++++++++++++++++++++ testdata/0.13-orphans-v6.db/000001.log | Bin 0 -> 3590 bytes testdata/0.13-orphans-v6.db/CURRENT | 1 + testdata/0.13-orphans-v6.db/LOCK | 0 testdata/0.13-orphans-v6.db/LOG | 6 + testdata/0.13-orphans-v6.db/MANIFEST-000000 | Bin 0 -> 54 bytes testdata/0.13-orphans.db/000001.log | Bin 0 -> 4300 bytes testdata/0.13-orphans.db/CURRENT | 1 + testdata/0.13-orphans.db/LOCK | 0 testdata/0.13-orphans.db/LOG | 6 + testdata/0.13-orphans.db/MANIFEST-000000 | Bin 0 -> 54 bytes 13 files changed, 274 insertions(+) create mode 100644 repair.go create mode 100644 repair_test.go create mode 100644 testdata/0.13-orphans-v6.db/000001.log create mode 100644 testdata/0.13-orphans-v6.db/CURRENT create mode 100644 testdata/0.13-orphans-v6.db/LOCK create mode 100644 testdata/0.13-orphans-v6.db/LOG create mode 100644 testdata/0.13-orphans-v6.db/MANIFEST-000000 create mode 100644 testdata/0.13-orphans.db/000001.log create mode 100644 testdata/0.13-orphans.db/CURRENT create mode 100644 testdata/0.13-orphans.db/LOCK create mode 100644 testdata/0.13-orphans.db/LOG create mode 100644 testdata/0.13-orphans.db/MANIFEST-000000 diff --git a/CHANGELOG.md b/CHANGELOG.md index 78e547c21..c9a1ed078 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +### Improvements + +- Add `Repair013Orphans()` to repair faulty orphans in a database last written to by IAVL 0.13.x + ### Bug Fixes - Remove unnecessary Protobuf dependencies diff --git a/repair.go b/repair.go new file mode 100644 index 000000000..157eb1fc3 --- /dev/null +++ b/repair.go @@ -0,0 +1,65 @@ +package iavl + +import ( + "math" + + "github.com/pkg/errors" + dbm "github.com/tendermint/tm-db" +) + +// Repair013Orphans repairs incorrect orphan entries written by IAVL 0.13 pruning. To use it, close +// a database using IAVL 0.13, make a backup copy, and then run this function before opening the +// database with IAVL 0.14 or later. It returns the number of faulty orphan entries removed. If the +// 0.13 database was written with KeepEvery:1 (the default) or the last version _ever_ saved to the +// tree was a multiple of `KeepEvery` and thus saved to disk, this repair is not necessary. +// +// Note that this cannot be used directly on Cosmos SDK databases, since they store multiple IAVL +// trees in the same underlying database via a prefix scheme. +// +// The pruning functionality enabled with Options.KeepEvery > 1 would write orphans entries to disk +// for versions that should only have been saved in memory, and these orphan entries were clamped +// to the last version persisted to disk instead of the version that generated them (so a delete at +// version 749 might generate an orphan entry ending at version 700 for KeepEvery:100). If the +// database is reopened at the last persisted version and this version is later deleted, the +// orphaned nodes can be deleted prematurely or incorrectly, causing data loss and database +// corruption. +// +// This function removes these incorrect orphan entries by deleting all orphan entries that have a +// to-version equal to or greater than the latest persisted version. Correct orphans will never +// have this, since they must have been deleted in a future (non-existent) version for that to be +// the case. +func Repair013Orphans(db dbm.DB) (uint64, error) { + ndb := newNodeDB(db, 0, &Options{Sync: true}) + version := ndb.getLatestVersion() + if version == 0 { + return 0, errors.New("no versions found") + } + + var ( + repaired uint64 + err error + ) + batch := db.NewBatch() + defer batch.Close() + ndb.traverseRange(orphanKeyFormat.Key(version), orphanKeyFormat.Key(math.MaxInt64), func(k, v []byte) { + // Sanity check so we don't remove stuff we shouldn't + var toVersion int64 + orphanKeyFormat.Scan(k, &toVersion) + if toVersion < version { + err = errors.Errorf("Found unexpected orphan with toVersion=%v, lesser than latest version %v", + toVersion, version) + return + } + repaired++ + batch.Delete(k) + }) + if err != nil { + return 0, err + } + err = batch.WriteSync() + if err != nil { + return 0, err + } + + return repaired, nil +} diff --git a/repair_test.go b/repair_test.go new file mode 100644 index 000000000..ec6b598b3 --- /dev/null +++ b/repair_test.go @@ -0,0 +1,191 @@ +package iavl + +import ( + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + dbm "github.com/tendermint/tm-db" +) + +func TestRepair013Orphans(t *testing.T) { + dir, err := ioutil.TempDir("", "test-iavl-repair") + require.NoError(t, err) + defer os.RemoveAll(dir) + + // There is also 0.13-orphans-v6.db containing a database closed immediately after writing + // version 6, which should not contain any broken orphans. + err = copyDB("testdata/0.13-orphans.db", filepath.Join(dir, "0.13-orphans.db")) + require.NoError(t, err) + + db, err := dbm.NewGoLevelDB("0.13-orphans", dir) + require.NoError(t, err) + + // Repair the database. + repaired, err := Repair013Orphans(db) + require.NoError(t, err) + assert.EqualValues(t, 8, repaired) + + // Load the database. + tree, err := NewMutableTreeWithOpts(db, 0, &Options{Sync: true}) + require.NoError(t, err) + version, err := tree.Load() + require.NoError(t, err) + require.EqualValues(t, 6, version) + + // We now generate two empty versions, and check all persisted versions. + _, version, err = tree.SaveVersion() + require.NoError(t, err) + require.EqualValues(t, 7, version) + _, version, err = tree.SaveVersion() + require.NoError(t, err) + require.EqualValues(t, 8, version) + + // Check all persisted versions. + require.Equal(t, []int{3, 6, 7, 8}, tree.AvailableVersions()) + assertVersion(t, tree, 0) + assertVersion(t, tree, 3) + assertVersion(t, tree, 6) + assertVersion(t, tree, 7) + assertVersion(t, tree, 8) + + // We then delete version 6 (the last persisted one with 0.13). + err = tree.DeleteVersion(6) + require.NoError(t, err) + + // Reading "rm7" (which should not have been deleted now) would panic with a broken database. + _, value := tree.Get([]byte("rm7")) + require.Equal(t, []byte{1}, value) + + // Check all persisted versions. + require.Equal(t, []int{3, 7, 8}, tree.AvailableVersions()) + assertVersion(t, tree, 0) + assertVersion(t, tree, 3) + assertVersion(t, tree, 7) + assertVersion(t, tree, 8) + + // Delete all historical versions, and check the latest. + err = tree.DeleteVersion(3) + require.NoError(t, err) + err = tree.DeleteVersion(7) + require.NoError(t, err) + + require.Equal(t, []int{8}, tree.AvailableVersions()) + assertVersion(t, tree, 0) + assertVersion(t, tree, 8) +} + +// assertVersion checks the given version (or current if 0) against the expected values. +func assertVersion(t *testing.T, tree *MutableTree, version int64) { + var err error + itree := tree.ImmutableTree + if version > 0 { + itree, err = tree.GetImmutable(version) + require.NoError(t, err) + } + version = itree.version + + // The "current" value should have the current version for <= 6, then 6 afterwards + _, value := itree.Get([]byte("current")) + if version >= 6 { + require.EqualValues(t, []byte{6}, value) + } else { + require.EqualValues(t, []byte{byte(version)}, value) + } + + // The "addX" entries should exist for 1-6 in the respective versions, and the + // "rmX" entries should have been removed for 1-6 in the respective versions. + for i := byte(1); i < 8; i++ { + _, value = itree.Get([]byte(fmt.Sprintf("add%v", i))) + if i <= 6 && int64(i) <= version { + require.Equal(t, []byte{i}, value) + } else { + require.Nil(t, value) + } + + _, value = itree.Get([]byte(fmt.Sprintf("rm%v", i))) + if i <= 6 && version >= int64(i) { + require.Nil(t, value) + } else { + require.Equal(t, []byte{1}, value) + } + } +} + +// Generate013Orphans generates a GoLevelDB orphan database in testdata/0.13-orphans.db +// for testing Repair013Orphans(). It must be run with IAVL 0.13.x. +/*func TestGenerate013Orphans(t *testing.T) { + err := os.RemoveAll("testdata/0.13-orphans.db") + require.NoError(t, err) + db, err := dbm.NewGoLevelDB("0.13-orphans", "testdata") + require.NoError(t, err) + tree, err := NewMutableTreeWithOpts(db, dbm.NewMemDB(), 0, &Options{ + KeepEvery: 3, + KeepRecent: 1, + Sync: true, + }) + require.NoError(t, err) + version, err := tree.Load() + require.NoError(t, err) + require.EqualValues(t, 0, version) + + // We generate 8 versions. In each version, we create a "addX" key, delete a "rmX" key, + // and update the "current" key, where "X" is the current version. Values are the version in + // which the key was last set. + tree.Set([]byte("rm1"), []byte{1}) + tree.Set([]byte("rm2"), []byte{1}) + tree.Set([]byte("rm3"), []byte{1}) + tree.Set([]byte("rm4"), []byte{1}) + tree.Set([]byte("rm5"), []byte{1}) + tree.Set([]byte("rm6"), []byte{1}) + tree.Set([]byte("rm7"), []byte{1}) + tree.Set([]byte("rm8"), []byte{1}) + + for v := byte(1); v <= 8; v++ { + tree.Set([]byte("current"), []byte{v}) + tree.Set([]byte(fmt.Sprintf("add%v", v)), []byte{v}) + tree.Remove([]byte(fmt.Sprintf("rm%v", v))) + _, version, err = tree.SaveVersion() + require.NoError(t, err) + require.EqualValues(t, v, version) + } + + // At this point, the database will contain incorrect orphans in version 6 that, when + // version 6 is deleted, will cause "current", "rm7", and "rm8" to go missing. +}*/ + +// copyDB makes a shallow copy of the source database directory. +func copyDB(src, dest string) error { + entries, err := ioutil.ReadDir(src) + if err != nil { + return err + } + err = os.MkdirAll(dest, 0777) + if err != nil { + return err + } + for _, entry := range entries { + out, err := os.Create(filepath.Join(dest, entry.Name())) + if err != nil { + return err + } + defer out.Close() + + in, err := os.Open(filepath.Join(src, entry.Name())) + defer in.Close() // nolint + if err != nil { + return err + } + + _, err = io.Copy(out, in) + if err != nil { + return err + } + } + return nil +} diff --git a/testdata/0.13-orphans-v6.db/000001.log b/testdata/0.13-orphans-v6.db/000001.log new file mode 100644 index 0000000000000000000000000000000000000000..13bc49ab48d66ec10f0a1aa9c3fadff3086764e5 GIT binary patch literal 3590 zcmds3TU1j=6y?Gt#1y4GjUloBvd1S41kQBYd} zu}~2#snnKAe4z+}C=qNxKrJZMl`bP7k0MwlC_(My8vKE^#^3h-+@F1B?{nsyGuw+^ zBUq-7VLER<6ZpgC3;SqWXg!-q741oON~828nKh9aDcRByb_>1LfjqDBC`CtC7w7T$ z?idEq+<`l?a=FDfBg2lhQGvU$*XpZ7hX$Pj_#~HxnliBhqHzt{1Ji|Qb+&{X{SbWR z=qwwIvOaN8u8wJ*U4MRp+}x)eeJ`mqKo=(wXn;}yP{fdJ%i_yzVS-IYT?ZyA>c+1< zll&6;pps{B%!Nly-y{}Gpf-YSA*Kh>;d@Uq7xd^Hxq7xQM=bMqcwSLu9{gSysNUCg_B4cFhHdNP_$54TkGMXc4l}i zHRJbxHf(%($r2QITiDH9?evC zF83(Zf|&%E?|-4E!=va?9MiVVZ2Z2?;%Jk?YpM(#16uW&)H1fTD$I zaQ_O<Q;Pe zbh}+)=KL&1ScCJDWkn8*{UB5e_9nnazg7u`eC=KtXH?s?o>HbsLg>p&Pxk*Aw<=-I z=`qC#x-=!FZo$v*3l~}&tiQd|O?g}uJhrWbg_AWk z^8%>i0+eh>ChT_TQq)}xO-`(~^!Ui8bL9>zd(dni{Pg~U%KEC9Fc#TF)71=s>PUc+ zjm;8i_o5y^u^LPGLR=R!)dr9>i_|@CDtzjX>TGoDPd^hmyePm7I#HVly{e|^#85i~ zy=11HsfW51BagORa*|tgup3AJ?{Tki=nc-wJbvB*OYL5t0~uOSWyEKvwF+FYQBAzN9R zSkRsNw5istc9A-&=J97NJlBj(izw%wON`OGToHhi4K-{F(N8!^IAjZn@J{VYtzZ~G z%xTwm4*%FLk?}149Phi;rRq6@^-k+yC5YS@m_9_`D0oDRpC?(Bb9SsKx!7j!iugd$ zgSBCe1J?`IKH)TaLNuX`iIE^WZF435+(lk}ynCyUoYPVg7adK{I6@sbbTn$Rtgv(* ztlJwjK&b#IV#p?l@He+wP1|hIAz62?rOv6hQE@lcmWplN*clk&XxImBqv4bbB;?H& zzc4ZD%h^0$?_%NT7b$BQkVl57Cisfz4>qX%={1_Z&cY@ x_C@x3sxD<<*4IE*d2y{^BtJHQY(n%91EA&xpmfi$c2VXL?JnP%7ZO0F@-G-${zm`+ literal 0 HcmV?d00001 diff --git a/testdata/0.13-orphans-v6.db/CURRENT b/testdata/0.13-orphans-v6.db/CURRENT new file mode 100644 index 000000000..feda7d6b2 --- /dev/null +++ b/testdata/0.13-orphans-v6.db/CURRENT @@ -0,0 +1 @@ +MANIFEST-000000 diff --git a/testdata/0.13-orphans-v6.db/LOCK b/testdata/0.13-orphans-v6.db/LOCK new file mode 100644 index 000000000..e69de29bb diff --git a/testdata/0.13-orphans-v6.db/LOG b/testdata/0.13-orphans-v6.db/LOG new file mode 100644 index 000000000..f890e80b8 --- /dev/null +++ b/testdata/0.13-orphans-v6.db/LOG @@ -0,0 +1,6 @@ +=============== Jun 25, 2020 (CEST) =============== +14:30:10.673317 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed +14:30:10.688689 db@open opening +14:30:10.689548 version@stat F·[] S·0B[] Sc·[] +14:30:10.702481 db@janitor F·2 G·0 +14:30:10.702564 db@open done T·13.82376ms diff --git a/testdata/0.13-orphans-v6.db/MANIFEST-000000 b/testdata/0.13-orphans-v6.db/MANIFEST-000000 new file mode 100644 index 0000000000000000000000000000000000000000..9d54f6733b1364dc8d53dd15ca59a6ec36a1c29d GIT binary patch literal 54 zcmdmC5aOo9z{n_-lUkOVlai$8R9TW*o>`pgoS$2eSd>_jU&O?~%*ev9Y~pbaHU>r} JMrI}!1^~s!4paaD literal 0 HcmV?d00001 diff --git a/testdata/0.13-orphans.db/000001.log b/testdata/0.13-orphans.db/000001.log new file mode 100644 index 0000000000000000000000000000000000000000..95ef16dcaf9eb82f3f7e3d1fa4e6ab313229b121 GIT binary patch literal 4300 zcmds5Z8Vf=7@n6grdee}zM5m6Sl?EW8C2UkX|p4guid0DF;Uh>!-^)XNKx&U(z1z` z&9-YZm6V8-iFOFpQnrpgO~}WJwu~~WeP)bK=Ztgi-JiSj=lyd%&vo6`^W67)z2j(7 z&0V2J5TvP}7Jd*5qWc(I89kdRWo-$jV$HN=8P%cbNm+ssuV#}LbL#x^<1~_r3YilY zW=9Yh9XxPXSSr%{YGl}=Cfsv3@kV-W=*Xa@dl<#KzPgy-jnQNo+MZCsXlYhSaKt0@ zg@u)HFx>FeL6J1FX?ER(32IZHc=Wx54tEtYMUF<03Iq{jwyjIAv<7lFYjz%(D61X6 z{=DR;fQRKA^%K?{dfH|_zXWTOvn?i6F*<1PY1X11(y?n5ec61WoB4~fN?rf0I$nq7 zWoCSw*Kal(qt!#dtz>n})SFH0Gqy`DX__ zaWT^i5Tpe`^jM+Ui#J`xEyHoto&2mx6JgR_uh;D!%bmWLJjwFCaO(55v+&8N@}eB% z2a_Y{T|XLGb`YcK@t!| zi^~Z9!!N`8rA=xbY17V3=Rz;59Z#)x5AFHZrkl5~sWvlJq0CXP?6HlVCHK~xvxg0C z$0kR#ndW6I$YciATP<6WZ_eD00u*2;Iau@OO74)0>Fc@aRYomm#3?2ICM%0h_x~Q_ z5jSUa<;mTOGMo@30YS9549%nW8F#GgXbIPrCneP``tkkf#fIt|?yR;IpOpBGZ7*b# zsWQ(T5o9icWMi`EF7wXr+Die6@m2cv3ynHf?=&z&O}c*19xN)atBed}Q?+Ebnu(wr zi6Gg;ES_R6sv?Nhr>vewRw1+%3n;QdY8NxTT=68JK7t!fX-`#187N&@rz&V21ejq@ zsH@?&SFRl#To7OeozN5p$F%r*{aPwP7`$`%$}|KxJ|MshdxGr+0cO|}Y7(%E?2QN0 z3e*U?!VX!7;4X$>_S4Zpjh2grX zviRdbkg-d~ujZ~_dO{E_T~)Y46v^|xnpmf!swT)WBA4!(pcRO4Sfa_%aJYboFm zN7-#^RzV+`#xtMCp67hCrbs$xu+DM=c0yholTgFxo4Jn}vGYqjvMa{&6AO&?u8Q^K zJ@g4|7`Tz^^VGY+0i)&GSOf*5Q?`_oUs#K(PIhf`7I`-p#zaJzq#vUX96267S)5li zANw|iMvw{w5o0!Ph?}m#8pamA_LBAYn`sxGEj0RYY}A-efpU4X z`7gC}`m(o-*IDaXxP}Uw2SlO%l8LbHhz~YO-AsBs$&|MX5L63-=&?fRCI27E83tlb zkFQ&!An-%V70+OSB$f7?WxJtpZT)kO_W4WUQZ|LAA(8~7Mw7Ml8 zblw-*>ma$DP8eQCnWY6a+>ygk?o=)L9bzJ=ZxEz=hOrBUM>t)+4KEY~l_E%WlB`o9 z7Ai)xx@<(hec8&QjO<51mk5lKGwccFJ@}xMbW)&dK|7RnQlJz37e_qpCk4Eu5MTzK m&@6z5g|eUE0D=G&dOaQ?B!V?{k4UmV8`pgoS$2eSd>_jU&O?~%*ev9Y~pbaHU>r} JMrI}!1^~s!4paaD literal 0 HcmV?d00001