Problem: rocksdb is out of date (backport crypto-org-chain#1448)

* Problem: rocksdb is out of date Solution: - upgrade to v9.1.1 * tag * changelog * Apply suggestions from code review Signed-off-by: yihuang <[email protected]> * versiondb * fix versiondb integration test * add rocksdb.tools to shell for debugging --------- Signed-off-by: yihuang <[email protected]>
yihuang · Jul 8, 2024 · 266fac9 · 266fac9
1 parent e1d819c
commit 266fac9
Show file tree

Hide file tree

Showing 9 changed files with 59 additions and 44 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,12 @@
 
 ## UNRELEASED
 
+* (store) [#1448](https://github.com/crypto-org-chain/cronos/pull/1448) Upgrade rocksdb to `v9.1.1`.
+
+*Jul 7, 2024*
+
+## v1.3.0
+
 ### Improvements
 
 * (versiondb) [#1491](https://github.com/crypto-org-chain/cronos/pull/1491) Free slice data in HasAtVersion.

diff --git a/flake.nix b/flake.nix
@@ -50,25 +50,21 @@
           };
           defaultPackage = packages.cronosd;
           defaultApp = apps.cronosd;
-          devShells = {
+          devShells = rec {
             default = pkgs.mkShell {
               buildInputs = [
                 defaultPackage.go
                 pkgs.gomod2nix
               ];
             };
             rocksdb = pkgs.mkShell {
-              buildInputs = [
-                defaultPackage.go
-                pkgs.gomod2nix
+              buildInputs = default.buildInputs ++ [
                 pkgs.rocksdb
+                pkgs.rocksdb.tools
               ];
             };
             full = pkgs.mkShell {
-              buildInputs = [
-                defaultPackage.go
-                pkgs.gomod2nix
-                pkgs.rocksdb
+              buildInputs = rocksdb.buildInputs ++ [
                 pkgs.test-env
               ];
             };

diff --git a/go.mod b/go.mod
@@ -27,7 +27,7 @@ require (
 	github.com/golang/protobuf v1.5.4
 	github.com/gorilla/mux v1.8.0
 	github.com/grpc-ecosystem/grpc-gateway v1.16.0
-	github.com/linxGnu/grocksdb v1.8.4
+	github.com/linxGnu/grocksdb v1.9.1
 	github.com/peggyjv/gravity-bridge/module/v2 v2.0.0-20220420162017-838c0d25e974
 	github.com/spf13/cast v1.6.0
 	github.com/spf13/cobra v1.8.0
@@ -254,7 +254,6 @@ replace (
 	// TODO Remove it: https://github.com/cosmos/cosmos-sdk/issues/10409
 	github.com/gin-gonic/gin => github.com/gin-gonic/gin v1.9.0
 	github.com/jhump/protoreflect => github.com/jhump/protoreflect v1.9.0
-	github.com/linxGnu/grocksdb => github.com/linxGnu/grocksdb v1.8.0
 	github.com/peggyjv/gravity-bridge/module/v2 => github.com/crypto-org-chain/gravity-bridge/module/v2 v2.0.1-0.20230825054824-75403cd90c6e
 	github.com/syndtr/goleveldb => github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7
 	// stick with compatible version or x/exp in v0.47.x line

diff --git a/go.sum b/go.sum
@@ -1085,8 +1085,8 @@ github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6
 github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg=
 github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM=
 github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4=
-github.com/linxGnu/grocksdb v1.8.0 h1:H4L/LhP7GOMf1j17oQAElHgVlbEje2h14A8Tz9cM2BE=
-github.com/linxGnu/grocksdb v1.8.0/go.mod h1:09CeBborffXhXdNpEcOeZrLKEnRtrZFEpFdPNI9Zjjg=
+github.com/linxGnu/grocksdb v1.9.1 h1:LmwuHzsWglxJrIES9jvS2O1xTPD2nnKYhAQDx5dIyRo=
+github.com/linxGnu/grocksdb v1.9.1/go.mod h1:QYiYypR2d4v63Wj1adOOfzglnoII0gLj3PNh4fZkcFA=
 github.com/lufeee/execinquery v1.2.1/go.mod h1:EC7DrEKView09ocscGHC+apXMIaorh4xqSxS/dy8SbM=
 github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
 github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w=

diff --git a/gomod2nix.toml b/gomod2nix.toml
@@ -403,9 +403,8 @@ schema = 3
     version = "v0.1.0"
     hash = "sha256-wQqGTtRWsfR9n0O/SXHVgECebbnNmHddxJIbG63OJBQ="
   [mod."github.com/linxGnu/grocksdb"]
-    version = "v1.8.0"
-    hash = "sha256-o6zj18at4oN6pqSioFqd1AXduR/cx0xDgtL1rPPw+1M="
-    replaced = "github.com/linxGnu/grocksdb"
+    version = "v1.9.1"
+    hash = "sha256-yRSGvZ5YeDqGOEmGPRSKxPjNoCQ/Xk/DMtNtutkbPGg="
   [mod."github.com/magiconair/properties"]
     version = "v1.8.7"
     hash = "sha256-XQ2bnc2s7/IH3WxEO4GishZurMyKwEclZy1DXg+2xXc="

diff --git a/integration_tests/test_versiondb.py b/integration_tests/test_versiondb.py
@@ -17,7 +17,7 @@ def test_versiondb_migration(cronos: Cronos):
     - verify change set and save snapshot
     - restore pruned application.db from the snapshot
     - replace node1's application.db with the restored one
-    - build versiondb for node0
+    - rebuild versiondb for node0
     - start the nodes, now check:
       - the network can grow
       - node0 do support historical queries
@@ -60,6 +60,8 @@ def test_versiondb_migration(cronos: Cronos):
     print("restore versiondb for node0")
     sst_dir = tempfile.mkdtemp(dir=cronos.base_dir)
     print(cli0.changeset_build_versiondb_sst(changeset_dir, sst_dir))
+    # ingest-versiondb-sst expects an empty database
+    shutil.rmtree(cli0.data_dir / "data/versiondb")
     print(
         cli0.changeset_ingest_versiondb_sst(
             cli0.data_dir / "data/versiondb", sst_dir, maximum_version=latest_version

diff --git a/nix/rocksdb.nix b/nix/rocksdb.nix
@@ -14,20 +14,19 @@
   # see: https://github.com/NixOS/nixpkgs/issues/216479
 , enableJemalloc ? !stdenv.hostPlatform.isWindows && !stdenv.hostPlatform.isStatic
 , jemalloc
-, enableLite ? false
 , enableShared ? !stdenv.hostPlatform.isStatic
 , sse42Support ? stdenv.hostPlatform.sse4_2Support
 }:
 
-stdenv.mkDerivation rec {
+stdenv.mkDerivation (finalAttrs: {
   pname = "rocksdb";
-  version = "8.1.1";
+  version = "9.1.1";
 
   src = fetchFromGitHub {
     owner = "facebook";
-    repo = pname;
-    rev = "v${version}";
-    sha256 = "sha256-79hRtc5QSWLLyjRGCmuYZSoIc9IcIsnl8UCinz2sVw4=";
+    repo = finalAttrs.pname;
+    rev = "v${finalAttrs.version}";
+    hash = "sha256-/Xf0bzNJPclH9IP80QNaABfhj4IAR5LycYET18VFCXc=";
   };
 
   nativeBuildInputs = [ cmake ninja ];
@@ -42,16 +41,9 @@ stdenv.mkDerivation rec {
     "tools"
   ];
 
-  NIX_CFLAGS_COMPILE = lib.optionals stdenv.cc.isGNU [
-    "-Wno-error=deprecated-copy"
-    "-Wno-error=pessimizing-move"
-    # Needed with GCC 12
-    "-Wno-error=format-truncation"
-    "-Wno-error=maybe-uninitialized"
-  ] ++ lib.optionals stdenv.cc.isClang [
-    "-Wno-error=unused-private-field"
+  env.NIX_CFLAGS_COMPILE = toString (lib.optionals stdenv.cc.isClang [
     "-faligned-allocation"
-  ];
+  ]);
 
   cmakeFlags = [
     "-DPORTABLE=1"
@@ -70,18 +62,38 @@ stdenv.mkDerivation rec {
     "-DUSE_RTTI=1"
     "-DROCKSDB_INSTALL_ON_WINDOWS=YES" # harmless elsewhere
     (lib.optional sse42Support "-DFORCE_SSE42=1")
-    (lib.optional enableLite "-DROCKSDB_LITE=1")
-    "-DFAIL_ON_WARNINGS=${if stdenv.hostPlatform.isMinGW then "NO" else "YES"}"
+    "-DFAIL_ON_WARNINGS=NO"
   ] ++ lib.optional (!enableShared) "-DROCKSDB_BUILD_SHARED=0";
 
   # otherwise "cc1: error: -Wformat-security ignored without -Wformat [-Werror=format-security]"
   hardeningDisable = lib.optional stdenv.hostPlatform.isWindows "format";
 
+  postPatch = lib.optionalString (lib.versionOlder finalAttrs.version "9") ''
+    # Fix gcc-13 build failures due to missing <cstdint> and
+    # <system_error> includes, fixed upstream since 9.x
+    sed -e '1i #include <cstdint>' -i options/offpeak_time_info.h
+  '' + lib.optionalString (lib.versionOlder finalAttrs.version "8") ''
+    # Fix gcc-13 build failures due to missing <cstdint> and
+    # <system_error> includes, fixed upstream since 8.x
+    sed -e '1i #include <cstdint>' -i db/compaction/compaction_iteration_stats.h
+    sed -e '1i #include <cstdint>' -i table/block_based/data_block_hash_index.h
+    sed -e '1i #include <cstdint>' -i util/string_util.h
+    sed -e '1i #include <cstdint>' -i include/rocksdb/utilities/checkpoint.h
+  '' + lib.optionalString (lib.versionOlder finalAttrs.version "7") ''
+    # Fix gcc-13 build failures due to missing <cstdint> and
+    # <system_error> includes, fixed upstream since 7.x
+    sed -e '1i #include <system_error>' -i third-party/folly/folly/synchronization/detail/ProxyLockable-inl.h
+  '' + ''
+    # fixed in https://github.com/facebook/rocksdb/pull/12309
+    sed -e 's/ZSTD_INCLUDE_DIRS/zstd_INCLUDE_DIRS/' -i cmake/modules/Findzstd.cmake
+    sed -e 's/ZSTD_INCLUDE_DIRS/zstd_INCLUDE_DIRS/' -i CMakeLists.txt
+  '';
+
   preInstall = ''
     mkdir -p $tools/bin
     cp tools/{ldb,sst_dump}${stdenv.hostPlatform.extensions.executable} $tools/bin/
   '' + lib.optionalString stdenv.isDarwin ''
-    ls -1 $tools/bin/* | xargs -I{} ${stdenv.cc.bintools.targetPrefix}install_name_tool -change "@rpath/librocksdb.${lib.versions.major version}.dylib" $out/lib/librocksdb.dylib {}
+    ls -1 $tools/bin/* | xargs -I{} ${stdenv.cc.bintools.targetPrefix}install_name_tool -change "@rpath/librocksdb.${lib.versions.major finalAttrs.version}.dylib" $out/lib/librocksdb.dylib {}
   '' + lib.optionalString (stdenv.isLinux && enableShared) ''
     ls -1 $tools/bin/* | xargs -I{} patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib {}
   '';
@@ -94,15 +106,15 @@ stdenv.mkDerivation rec {
     fi
   '' + lib.optionalString stdenv.isDarwin ''
     ${stdenv.cc.targetPrefix}install_name_tool -change "@rpath/libsnappy.1.dylib" "${snappy}/lib/libsnappy.1.dylib" $out/lib/librocksdb.dylib
-    ${stdenv.cc.targetPrefix}install_name_tool -change "@rpath/librocksdb.${lib.versions.major version}.dylib" "$out/lib/librocksdb.${lib.versions.major version}.dylib" $out/lib/librocksdb.dylib
+    ${stdenv.cc.targetPrefix}install_name_tool -change "@rpath/librocksdb.${lib.versions.major finalAttrs.version}.dylib" "$out/lib/librocksdb.${lib.versions.major finalAttrs.version}.dylib" $out/lib/librocksdb.dylib
   '';
 
   meta = with lib; {
     homepage = "https://rocksdb.org";
     description = "A library that provides an embeddable, persistent key-value store for fast storage";
-    changelog = "https://github.com/facebook/rocksdb/raw/v${version}/HISTORY.md";
+    changelog = "https://github.com/facebook/rocksdb/raw/v${finalAttrs.version}/HISTORY.md";
     license = licenses.asl20;
     platforms = platforms.all;
     maintainers = with maintainers; [ adev magenbluten ];
   };
-}
+})
diff --git a/versiondb/go.mod b/versiondb/go.mod
@@ -14,10 +14,10 @@ require (
 	github.com/cosmos/iavl v0.21.0-alpha.1.0.20230904092046-df3db2d96583
 	github.com/crypto-org-chain/cronos/memiavl v0.0.3
 	github.com/golang/snappy v0.0.4
-	github.com/linxGnu/grocksdb v1.8.0
+	github.com/linxGnu/grocksdb v1.9.1
 	github.com/spf13/cast v1.5.0
 	github.com/spf13/cobra v1.6.1
-	github.com/stretchr/testify v1.8.4
+	github.com/stretchr/testify v1.9.0
 )
 
 require (

diff --git a/versiondb/go.sum b/versiondb/go.sum
@@ -515,8 +515,8 @@ github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6
 github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg=
 github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM=
 github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4=
-github.com/linxGnu/grocksdb v1.8.0 h1:H4L/LhP7GOMf1j17oQAElHgVlbEje2h14A8Tz9cM2BE=
-github.com/linxGnu/grocksdb v1.8.0/go.mod h1:09CeBborffXhXdNpEcOeZrLKEnRtrZFEpFdPNI9Zjjg=
+github.com/linxGnu/grocksdb v1.9.1 h1:LmwuHzsWglxJrIES9jvS2O1xTPD2nnKYhAQDx5dIyRo=
+github.com/linxGnu/grocksdb v1.9.1/go.mod h1:QYiYypR2d4v63Wj1adOOfzglnoII0gLj3PNh4fZkcFA=
 github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo=
@@ -737,8 +737,9 @@ github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5J
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
-github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
+github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
@@ -749,8 +750,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
-github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/subosito/gotenv v1.4.1 h1:jyEFiXpy21Wm81FBN71l9VoMMV8H8jG+qIK3GCpY6Qs=
 github.com/subosito/gotenv v1.4.1/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0=
 github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d h1:vfofYNRScrDdvS342BElfbETmL1Aiz3i2t0zfRj16Hs=