From 912390d5fc4a32113ea1aacc98b77b2649aea4c2 Mon Sep 17 00:00:00 2001 From: cool-developer <51834436+cool-develope@users.noreply.github.com> Date: Mon, 13 Nov 2023 07:27:42 -0500 Subject: [PATCH] feat(store/v2): parallel writes in storage sqlite backends (#18320) --- store/go.mod | 16 +---- store/go.sum | 46 +------------ store/pruning/manager_test.go | 30 ++++---- store/storage/sqlite/db.go | 6 +- store/storage/sqlite/db_test.go | 115 +++++++++++++++++++++++++++++++ store/storage/sqlite/iterator.go | 1 - 6 files changed, 138 insertions(+), 76 deletions(-) diff --git a/store/go.mod b/store/go.mod index 75deaf9d8cd1..a99b61f04b4c 100644 --- a/store/go.mod +++ b/store/go.mod @@ -13,11 +13,11 @@ require ( github.com/cosmos/gogoproto v1.4.11 github.com/cosmos/iavl v1.0.0 github.com/cosmos/ics23/go v0.10.0 + github.com/mattn/go-sqlite3 v1.14.17 github.com/linxGnu/grocksdb v1.8.5 github.com/stretchr/testify v1.8.4 github.com/tidwall/btree v1.7.0 golang.org/x/exp v0.0.0-20231006140011-7918f672742d - modernc.org/sqlite v1.27.0 ) require ( @@ -30,7 +30,6 @@ require ( github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect - github.com/dustin/go-humanize v1.0.1 // indirect github.com/emicklei/dot v1.6.0 // indirect github.com/getsentry/sentry-go v0.25.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect @@ -38,8 +37,6 @@ require ( github.com/golang/snappy v0.0.4 // indirect github.com/google/btree v1.1.2 // indirect github.com/google/go-cmp v0.6.0 // indirect - github.com/google/uuid v1.4.0 // indirect - github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/klauspost/compress v1.17.2 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect @@ -54,12 +51,12 @@ require ( github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.45.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect - github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rogpeppe/go-internal v1.11.0 // indirect github.com/rs/zerolog v1.31.0 // indirect github.com/sasha-s/go-deadlock v0.3.1 // indirect github.com/spf13/cast v1.5.1 // indirect github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect + golang.org/x/sync v0.4.0 // indirect golang.org/x/crypto v0.15.0 // indirect golang.org/x/mod v0.13.0 // indirect golang.org/x/net v0.17.0 // indirect @@ -71,13 +68,4 @@ require ( google.golang.org/protobuf v1.31.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect gotest.tools/v3 v3.5.1 // indirect - lukechampine.com/uint128 v1.3.0 // indirect - modernc.org/cc/v3 v3.41.0 // indirect - modernc.org/ccgo/v3 v3.16.15 // indirect - modernc.org/libc v1.29.0 // indirect - modernc.org/mathutil v1.6.0 // indirect - modernc.org/memory v1.7.2 // indirect - modernc.org/opt v0.1.3 // indirect - modernc.org/strutil v1.2.0 // indirect - modernc.org/token v1.1.0 // indirect ) diff --git a/store/go.sum b/store/go.sum index 78f222dc74c5..72597a4ee505 100644 --- a/store/go.sum +++ b/store/go.sum @@ -46,8 +46,6 @@ github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 h1:YLtO71vCjJRCBcrPMtQ9nqBsqpA1m5sE92cU+pd5Mcc= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs= -github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= -github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/emicklei/dot v1.6.0 h1:vUzuoVE8ipzS7QkES4UfxdpCwdU2U97m2Pb2tQCoYRY= github.com/emicklei/dot v1.6.0/go.mod h1:DeV7GvQtIw4h2u73RKBkkFdvVAz0D9fzeJrgPW6gy/s= github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= @@ -85,13 +83,7 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/pprof v0.0.0-20230228050547-1710fef4ab10 h1:CqYfpuYIjnlNxM3msdyPRKabhXZWbKjf3Q8BWROFBso= -github.com/google/pprof v0.0.0-20230228050547-1710fef4ab10/go.mod h1:79YE0hCXdHag9sBkw2o+N/YnZtTkXi0UT9Nnixa5eYk= -github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= -github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= -github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4= @@ -108,8 +100,8 @@ github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/ github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y= -github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= +github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM= +github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= @@ -141,8 +133,6 @@ github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lne github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= -github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= -github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= @@ -170,8 +160,6 @@ golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY= -golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -211,8 +199,6 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= -golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -243,31 +229,3 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= -lukechampine.com/uint128 v1.3.0 h1:cDdUVfRwDUDovz610ABgFD17nXD4/uDgVHl2sC3+sbo= -lukechampine.com/uint128 v1.3.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= -modernc.org/cc/v3 v3.41.0 h1:QoR1Sn3YWlmA1T4vLaKZfawdVtSiGx8H+cEojbC7v1Q= -modernc.org/cc/v3 v3.41.0/go.mod h1:Ni4zjJYJ04CDOhG7dn640WGfwBzfE0ecX8TyMB0Fv0Y= -modernc.org/ccgo/v3 v3.16.15 h1:KbDR3ZAVU+wiLyMESPtbtE/Add4elztFyfsWoNTgxS0= -modernc.org/ccgo/v3 v3.16.15/go.mod h1:yT7B+/E2m43tmMOT51GMoM98/MtHIcQQSleGnddkUNI= -modernc.org/ccorpus v1.11.6 h1:J16RXiiqiCgua6+ZvQot4yUuUy8zxgqbqEEUuGPlISk= -modernc.org/ccorpus v1.11.6/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ= -modernc.org/httpfs v1.0.6 h1:AAgIpFZRXuYnkjftxTAZwMIiwEqAfk8aVB2/oA6nAeM= -modernc.org/httpfs v1.0.6/go.mod h1:7dosgurJGp0sPaRanU53W4xZYKh14wfzX420oZADeHM= -modernc.org/libc v1.29.0 h1:tTFRFq69YKCF2QyGNuRUQxKBm1uZZLubf6Cjh/pVHXs= -modernc.org/libc v1.29.0/go.mod h1:DaG/4Q3LRRdqpiLyP0C2m1B8ZMGkQ+cCgOIjEtQlYhQ= -modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= -modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= -modernc.org/memory v1.7.2 h1:Klh90S215mmH8c9gO98QxQFsY+W451E8AnzjoE2ee1E= -modernc.org/memory v1.7.2/go.mod h1:NO4NVCQy0N7ln+T9ngWqOQfi7ley4vpwvARR+Hjw95E= -modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= -modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= -modernc.org/sqlite v1.27.0 h1:MpKAHoyYB7xqcwnUwkuD+npwEa0fojF0B5QRbN+auJ8= -modernc.org/sqlite v1.27.0/go.mod h1:Qxpazz0zH8Z1xCFyi5GSL3FzbtZ3fvbjmywNogldEW0= -modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= -modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= -modernc.org/tcl v1.15.2 h1:C4ybAYCGJw968e+Me18oW55kD/FexcHbqH2xak1ROSY= -modernc.org/tcl v1.15.2/go.mod h1:3+k/ZaEbKrC8ePv8zJWPtBSW0V7Gg9g8rkmhI1Kfs3c= -modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= -modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= -modernc.org/z v1.7.3 h1:zDJf6iHjrnB+WRD88stbXokugjyc0/pB91ri1gO6LZY= -modernc.org/z v1.7.3/go.mod h1:Ipv4tsdxZRbQyLq9Q1M6gdbkxYzdlrciF2Hi/lS7nWE= diff --git a/store/pruning/manager_test.go b/store/pruning/manager_test.go index 10c5966241c3..da4bbe1da551 100644 --- a/store/pruning/manager_test.go +++ b/store/pruning/manager_test.go @@ -44,13 +44,15 @@ func (s *PruningTestSuite) TearDownTest() { } func (s *PruningTestSuite) TestPruning() { - s.manager.SetCommitmentOptions(Options{4, 2, false}) - s.manager.SetStorageOptions(Options{3, 3, true}) + s.manager.SetCommitmentOptions(Options{4, 2, true}) + s.manager.SetStorageOptions(Options{3, 3, false}) s.manager.Start() + latestVersion := uint64(100) + // write 10 batches - for i := 0; i < 10; i++ { - version := uint64(i + 1) + for i := uint64(0); i < latestVersion; i++ { + version := i + 1 cs := store.NewChangeset() cs.Add([]byte("key"), []byte(fmt.Sprintf("value%d", version))) err := s.sc.WriteBatch(cs) @@ -59,27 +61,27 @@ func (s *PruningTestSuite) TestPruning() { s.Require().NoError(err) err = s.ss.ApplyChangeset(version, cs) s.Require().NoError(err) - s.manager.Prune(uint64(i + 1)) + s.manager.Prune(version) } // wait for pruning to finish s.manager.Stop() - // check the store for the version 6 - val, err := s.ss.Get("", 6, []byte("key")) + // check the store for the version 96 + val, err := s.ss.Get("", latestVersion-4, []byte("key")) s.Require().NoError(err) - s.Require().Equal([]byte("value6"), val) - // check the store for the version 5 - val, err = s.ss.Get("", 5, []byte("key")) + s.Require().Equal([]byte("value96"), val) + // check the store for the version 50 + val, err = s.ss.Get("", 50, []byte("key")) s.Require().NoError(err) s.Require().Nil(val) - // check the commitment for the version 6 - proof, err := s.sc.GetProof(6, []byte("key")) + // check the commitment for the version 96 + proof, err := s.sc.GetProof(latestVersion-4, []byte("key")) s.Require().NoError(err) s.Require().NotNil(proof.GetExist()) - // check the commitment for the version 5 - proof, err = s.sc.GetProof(5, []byte("key")) + // check the commitment for the version 95 + proof, err = s.sc.GetProof(latestVersion-5, []byte("key")) s.Require().Error(err) s.Require().Nil(proof) } diff --git a/store/storage/sqlite/db.go b/store/storage/sqlite/db.go index 0886b023f808..ae369fa03586 100644 --- a/store/storage/sqlite/db.go +++ b/store/storage/sqlite/db.go @@ -8,14 +8,14 @@ import ( "path/filepath" "strings" - _ "modernc.org/sqlite" + _ "github.com/mattn/go-sqlite3" "cosmossdk.io/store/v2" ) const ( - driverName = "sqlite" - dbName = "ss.db" + driverName = "sqlite3" + dbName = "file:ss.db?cache=shared&mode=rwc&_journal_mode=WAL" reservedStoreKey = "_RESERVED_" keyLatestHeight = "latest_height" diff --git a/store/storage/sqlite/db_test.go b/store/storage/sqlite/db_test.go index 60d258626a1a..7d80232dfd33 100644 --- a/store/storage/sqlite/db_test.go +++ b/store/storage/sqlite/db_test.go @@ -2,7 +2,9 @@ package sqlite import ( "fmt" + "sync" "testing" + "time" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -87,3 +89,116 @@ func TestDatabase_ReverseIterator(t *testing.T) { require.Error(t, err) require.Nil(t, iter3) } + +func TestParallelWrites(t *testing.T) { + db, err := New(t.TempDir()) + require.NoError(t, err) + defer db.Close() + + latestVersion := 10 + kvCount := 100 + + wg := sync.WaitGroup{} + triggerStartCh := make(chan bool) + + // start 10 goroutines that write to the database + for i := 0; i < latestVersion; i++ { + wg.Add(1) + go func(i int) { + <-triggerStartCh + t.Log("start time", i, time.Now()) + defer wg.Done() + cs := new(store.Changeset) + for j := 0; j < kvCount; j++ { + key := fmt.Sprintf("key-%d-%03d", i, j) + val := fmt.Sprintf("val-%d-%03d", i, j) + + cs.AddKVPair(store.KVPair{StoreKey: storeKey1, Key: []byte(key), Value: []byte(val)}) + } + + require.NoError(t, db.ApplyChangeset(uint64(i+1), cs)) + t.Log("end time", i, time.Now()) + }(i) + + } + + // start the goroutines + close(triggerStartCh) + wg.Wait() + + // check that all the data is there + for i := 0; i < latestVersion; i++ { + for j := 0; j < kvCount; j++ { + version := uint64(i + 1) + key := fmt.Sprintf("key-%d-%03d", i, j) + val := fmt.Sprintf("val-%d-%03d", i, j) + + v, err := db.Get(storeKey1, version, []byte(key)) + require.NoError(t, err) + require.Equal(t, []byte(val), v) + } + } +} + +func TestParallelWriteAndPruning(t *testing.T) { + db, err := New(t.TempDir()) + require.NoError(t, err) + defer db.Close() + + latestVersion := 100 + kvCount := 100 + prunePeriod := 5 + + wg := sync.WaitGroup{} + triggerStartCh := make(chan bool) + + // start a goroutine that write to the database + wg.Add(1) + go func() { + <-triggerStartCh + defer wg.Done() + for i := 0; i < latestVersion; i++ { + cs := new(store.Changeset) + for j := 0; j < kvCount; j++ { + key := fmt.Sprintf("key-%d-%03d", i, j) + val := fmt.Sprintf("val-%d-%03d", i, j) + + cs.AddKVPair(store.KVPair{StoreKey: storeKey1, Key: []byte(key), Value: []byte(val)}) + } + + require.NoError(t, db.ApplyChangeset(uint64(i+1), cs)) + } + }() + // start a goroutine that prunes the database + wg.Add(1) + go func() { + <-triggerStartCh + defer wg.Done() + for i := 10; i < latestVersion; i += prunePeriod { + for { + v, err := db.GetLatestVersion() + require.NoError(t, err) + if v > uint64(i) { + t.Log("pruning version", v-1) + require.NoError(t, db.Prune(v-1)) + break + } + } + } + }() + + // start the goroutines + close(triggerStartCh) + wg.Wait() + + // check if the data is pruned + version := uint64(latestVersion - prunePeriod) + val, err := db.Get(storeKey1, version, []byte(fmt.Sprintf("key-%d-%03d", version-1, 0))) + require.NoError(t, err) + require.Nil(t, val) + + version = uint64(latestVersion) + val, err = db.Get(storeKey1, version, []byte(fmt.Sprintf("key-%d-%03d", version-1, 0))) + require.NoError(t, err) + require.Equal(t, []byte(fmt.Sprintf("val-%d-%03d", version-1, 0)), val) +} diff --git a/store/storage/sqlite/iterator.go b/store/storage/sqlite/iterator.go index 8e31200e36d3..8be734d871a3 100644 --- a/store/storage/sqlite/iterator.go +++ b/store/storage/sqlite/iterator.go @@ -7,7 +7,6 @@ import ( "strings" "golang.org/x/exp/slices" - _ "modernc.org/sqlite" "cosmossdk.io/store/v2" )