diff --git a/CHANGELOG.md b/CHANGELOG.md index a6959d51b7..ff96625c28 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ ## UNRELEASED +* (store) [#1448](https://github.com/crypto-org-chain/cronos/pull/1448) Upgrade rocksdb to `v9.1.1`. + +*Jul 7, 2024* + +## v1.3.0 + ### Improvements * (versiondb) [#1491](https://github.com/crypto-org-chain/cronos/pull/1491) Free slice data in HasAtVersion. diff --git a/flake.nix b/flake.nix index b27e354e0c..36fc0bfe97 100644 --- a/flake.nix +++ b/flake.nix @@ -50,7 +50,7 @@ }; defaultPackage = packages.cronosd; defaultApp = apps.cronosd; - devShells = { + devShells = rec { default = pkgs.mkShell { buildInputs = [ defaultPackage.go @@ -58,17 +58,13 @@ ]; }; rocksdb = pkgs.mkShell { - buildInputs = [ - defaultPackage.go - pkgs.gomod2nix + buildInputs = default.buildInputs ++ [ pkgs.rocksdb + pkgs.rocksdb.tools ]; }; full = pkgs.mkShell { - buildInputs = [ - defaultPackage.go - pkgs.gomod2nix - pkgs.rocksdb + buildInputs = rocksdb.buildInputs ++ [ pkgs.test-env ]; }; diff --git a/go.mod b/go.mod index cde3e2397b..e07256b0f6 100644 --- a/go.mod +++ b/go.mod @@ -27,7 +27,7 @@ require ( github.com/golang/protobuf v1.5.4 github.com/gorilla/mux v1.8.0 github.com/grpc-ecosystem/grpc-gateway v1.16.0 - github.com/linxGnu/grocksdb v1.8.4 + github.com/linxGnu/grocksdb v1.9.1 github.com/peggyjv/gravity-bridge/module/v2 v2.0.0-20220420162017-838c0d25e974 github.com/spf13/cast v1.6.0 github.com/spf13/cobra v1.8.0 @@ -254,7 +254,6 @@ replace ( // TODO Remove it: https://github.com/cosmos/cosmos-sdk/issues/10409 github.com/gin-gonic/gin => github.com/gin-gonic/gin v1.9.0 github.com/jhump/protoreflect => github.com/jhump/protoreflect v1.9.0 - github.com/linxGnu/grocksdb => github.com/linxGnu/grocksdb v1.8.0 github.com/peggyjv/gravity-bridge/module/v2 => github.com/crypto-org-chain/gravity-bridge/module/v2 v2.0.1-0.20230825054824-75403cd90c6e github.com/syndtr/goleveldb => github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // stick with compatible version or x/exp in v0.47.x line diff --git a/go.sum b/go.sum index e9f915b06c..5a096582df 100644 --- a/go.sum +++ b/go.sum @@ -1085,8 +1085,8 @@ github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6 github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= -github.com/linxGnu/grocksdb v1.8.0 h1:H4L/LhP7GOMf1j17oQAElHgVlbEje2h14A8Tz9cM2BE= -github.com/linxGnu/grocksdb v1.8.0/go.mod h1:09CeBborffXhXdNpEcOeZrLKEnRtrZFEpFdPNI9Zjjg= +github.com/linxGnu/grocksdb v1.9.1 h1:LmwuHzsWglxJrIES9jvS2O1xTPD2nnKYhAQDx5dIyRo= +github.com/linxGnu/grocksdb v1.9.1/go.mod h1:QYiYypR2d4v63Wj1adOOfzglnoII0gLj3PNh4fZkcFA= github.com/lufeee/execinquery v1.2.1/go.mod h1:EC7DrEKView09ocscGHC+apXMIaorh4xqSxS/dy8SbM= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w= diff --git a/gomod2nix.toml b/gomod2nix.toml index 3c6df35444..12000f5f7b 100644 --- a/gomod2nix.toml +++ b/gomod2nix.toml @@ -403,9 +403,8 @@ schema = 3 version = "v0.1.0" hash = "sha256-wQqGTtRWsfR9n0O/SXHVgECebbnNmHddxJIbG63OJBQ=" [mod."github.com/linxGnu/grocksdb"] - version = "v1.8.0" - hash = "sha256-o6zj18at4oN6pqSioFqd1AXduR/cx0xDgtL1rPPw+1M=" - replaced = "github.com/linxGnu/grocksdb" + version = "v1.9.1" + hash = "sha256-yRSGvZ5YeDqGOEmGPRSKxPjNoCQ/Xk/DMtNtutkbPGg=" [mod."github.com/magiconair/properties"] version = "v1.8.7" hash = "sha256-XQ2bnc2s7/IH3WxEO4GishZurMyKwEclZy1DXg+2xXc=" diff --git a/integration_tests/test_versiondb.py b/integration_tests/test_versiondb.py index 6e4d5f452f..d79dfa5ac3 100644 --- a/integration_tests/test_versiondb.py +++ b/integration_tests/test_versiondb.py @@ -17,7 +17,7 @@ def test_versiondb_migration(cronos: Cronos): - verify change set and save snapshot - restore pruned application.db from the snapshot - replace node1's application.db with the restored one - - build versiondb for node0 + - rebuild versiondb for node0 - start the nodes, now check: - the network can grow - node0 do support historical queries @@ -60,6 +60,8 @@ def test_versiondb_migration(cronos: Cronos): print("restore versiondb for node0") sst_dir = tempfile.mkdtemp(dir=cronos.base_dir) print(cli0.changeset_build_versiondb_sst(changeset_dir, sst_dir)) + # ingest-versiondb-sst expects an empty database + shutil.rmtree(cli0.data_dir / "data/versiondb") print( cli0.changeset_ingest_versiondb_sst( cli0.data_dir / "data/versiondb", sst_dir, maximum_version=latest_version diff --git a/nix/rocksdb.nix b/nix/rocksdb.nix index 9b3333d173..916a0a0eb2 100644 --- a/nix/rocksdb.nix +++ b/nix/rocksdb.nix @@ -14,20 +14,19 @@ # see: https://github.com/NixOS/nixpkgs/issues/216479 , enableJemalloc ? !stdenv.hostPlatform.isWindows && !stdenv.hostPlatform.isStatic , jemalloc -, enableLite ? false , enableShared ? !stdenv.hostPlatform.isStatic , sse42Support ? stdenv.hostPlatform.sse4_2Support }: -stdenv.mkDerivation rec { +stdenv.mkDerivation (finalAttrs: { pname = "rocksdb"; - version = "8.1.1"; + version = "9.1.1"; src = fetchFromGitHub { owner = "facebook"; - repo = pname; - rev = "v${version}"; - sha256 = "sha256-79hRtc5QSWLLyjRGCmuYZSoIc9IcIsnl8UCinz2sVw4="; + repo = finalAttrs.pname; + rev = "v${finalAttrs.version}"; + hash = "sha256-/Xf0bzNJPclH9IP80QNaABfhj4IAR5LycYET18VFCXc="; }; nativeBuildInputs = [ cmake ninja ]; @@ -42,16 +41,9 @@ stdenv.mkDerivation rec { "tools" ]; - NIX_CFLAGS_COMPILE = lib.optionals stdenv.cc.isGNU [ - "-Wno-error=deprecated-copy" - "-Wno-error=pessimizing-move" - # Needed with GCC 12 - "-Wno-error=format-truncation" - "-Wno-error=maybe-uninitialized" - ] ++ lib.optionals stdenv.cc.isClang [ - "-Wno-error=unused-private-field" + env.NIX_CFLAGS_COMPILE = toString (lib.optionals stdenv.cc.isClang [ "-faligned-allocation" - ]; + ]); cmakeFlags = [ "-DPORTABLE=1" @@ -70,18 +62,38 @@ stdenv.mkDerivation rec { "-DUSE_RTTI=1" "-DROCKSDB_INSTALL_ON_WINDOWS=YES" # harmless elsewhere (lib.optional sse42Support "-DFORCE_SSE42=1") - (lib.optional enableLite "-DROCKSDB_LITE=1") - "-DFAIL_ON_WARNINGS=${if stdenv.hostPlatform.isMinGW then "NO" else "YES"}" + "-DFAIL_ON_WARNINGS=NO" ] ++ lib.optional (!enableShared) "-DROCKSDB_BUILD_SHARED=0"; # otherwise "cc1: error: -Wformat-security ignored without -Wformat [-Werror=format-security]" hardeningDisable = lib.optional stdenv.hostPlatform.isWindows "format"; + postPatch = lib.optionalString (lib.versionOlder finalAttrs.version "9") '' + # Fix gcc-13 build failures due to missing and + # includes, fixed upstream since 9.x + sed -e '1i #include ' -i options/offpeak_time_info.h + '' + lib.optionalString (lib.versionOlder finalAttrs.version "8") '' + # Fix gcc-13 build failures due to missing and + # includes, fixed upstream since 8.x + sed -e '1i #include ' -i db/compaction/compaction_iteration_stats.h + sed -e '1i #include ' -i table/block_based/data_block_hash_index.h + sed -e '1i #include ' -i util/string_util.h + sed -e '1i #include ' -i include/rocksdb/utilities/checkpoint.h + '' + lib.optionalString (lib.versionOlder finalAttrs.version "7") '' + # Fix gcc-13 build failures due to missing and + # includes, fixed upstream since 7.x + sed -e '1i #include ' -i third-party/folly/folly/synchronization/detail/ProxyLockable-inl.h + '' + '' + # fixed in https://github.com/facebook/rocksdb/pull/12309 + sed -e 's/ZSTD_INCLUDE_DIRS/zstd_INCLUDE_DIRS/' -i cmake/modules/Findzstd.cmake + sed -e 's/ZSTD_INCLUDE_DIRS/zstd_INCLUDE_DIRS/' -i CMakeLists.txt + ''; + preInstall = '' mkdir -p $tools/bin cp tools/{ldb,sst_dump}${stdenv.hostPlatform.extensions.executable} $tools/bin/ '' + lib.optionalString stdenv.isDarwin '' - ls -1 $tools/bin/* | xargs -I{} ${stdenv.cc.bintools.targetPrefix}install_name_tool -change "@rpath/librocksdb.${lib.versions.major version}.dylib" $out/lib/librocksdb.dylib {} + ls -1 $tools/bin/* | xargs -I{} ${stdenv.cc.bintools.targetPrefix}install_name_tool -change "@rpath/librocksdb.${lib.versions.major finalAttrs.version}.dylib" $out/lib/librocksdb.dylib {} '' + lib.optionalString (stdenv.isLinux && enableShared) '' ls -1 $tools/bin/* | xargs -I{} patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib {} ''; @@ -94,15 +106,15 @@ stdenv.mkDerivation rec { fi '' + lib.optionalString stdenv.isDarwin '' ${stdenv.cc.targetPrefix}install_name_tool -change "@rpath/libsnappy.1.dylib" "${snappy}/lib/libsnappy.1.dylib" $out/lib/librocksdb.dylib - ${stdenv.cc.targetPrefix}install_name_tool -change "@rpath/librocksdb.${lib.versions.major version}.dylib" "$out/lib/librocksdb.${lib.versions.major version}.dylib" $out/lib/librocksdb.dylib + ${stdenv.cc.targetPrefix}install_name_tool -change "@rpath/librocksdb.${lib.versions.major finalAttrs.version}.dylib" "$out/lib/librocksdb.${lib.versions.major finalAttrs.version}.dylib" $out/lib/librocksdb.dylib ''; meta = with lib; { homepage = "https://rocksdb.org"; description = "A library that provides an embeddable, persistent key-value store for fast storage"; - changelog = "https://github.com/facebook/rocksdb/raw/v${version}/HISTORY.md"; + changelog = "https://github.com/facebook/rocksdb/raw/v${finalAttrs.version}/HISTORY.md"; license = licenses.asl20; platforms = platforms.all; maintainers = with maintainers; [ adev magenbluten ]; }; -} +}) diff --git a/versiondb/go.mod b/versiondb/go.mod index 0e3c2859af..a118246b1f 100644 --- a/versiondb/go.mod +++ b/versiondb/go.mod @@ -14,10 +14,10 @@ require ( github.com/cosmos/iavl v0.21.0-alpha.1.0.20230904092046-df3db2d96583 github.com/crypto-org-chain/cronos/memiavl v0.0.3 github.com/golang/snappy v0.0.4 - github.com/linxGnu/grocksdb v1.8.0 + github.com/linxGnu/grocksdb v1.9.1 github.com/spf13/cast v1.5.0 github.com/spf13/cobra v1.6.1 - github.com/stretchr/testify v1.8.4 + github.com/stretchr/testify v1.9.0 ) require ( diff --git a/versiondb/go.sum b/versiondb/go.sum index daad8687ea..13d913ea0f 100644 --- a/versiondb/go.sum +++ b/versiondb/go.sum @@ -515,8 +515,8 @@ github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6 github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= -github.com/linxGnu/grocksdb v1.8.0 h1:H4L/LhP7GOMf1j17oQAElHgVlbEje2h14A8Tz9cM2BE= -github.com/linxGnu/grocksdb v1.8.0/go.mod h1:09CeBborffXhXdNpEcOeZrLKEnRtrZFEpFdPNI9Zjjg= +github.com/linxGnu/grocksdb v1.9.1 h1:LmwuHzsWglxJrIES9jvS2O1xTPD2nnKYhAQDx5dIyRo= +github.com/linxGnu/grocksdb v1.9.1/go.mod h1:QYiYypR2d4v63Wj1adOOfzglnoII0gLj3PNh4fZkcFA= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo= @@ -737,8 +737,9 @@ github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5J github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -749,8 +750,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.4.1 h1:jyEFiXpy21Wm81FBN71l9VoMMV8H8jG+qIK3GCpY6Qs= github.com/subosito/gotenv v1.4.1/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d h1:vfofYNRScrDdvS342BElfbETmL1Aiz3i2t0zfRj16Hs=