diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index be8a44f14d..0294ce871e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -52,7 +52,7 @@ jobs: unittest: runs-on: ubuntu-latest - timeout-minutes: 10 + timeout-minutes: 40 steps: - uses: actions/checkout@v3 - uses: cachix/install-nix-action@v22 @@ -78,16 +78,8 @@ jobs: make test # run versiondb tests - nix registry add nixpkgs github:NixOS/nixpkgs/23.05 - nix profile install nixpkgs#rocksdb - export PKG_CONFIG_PATH=$HOME/.nix-profile/lib/pkgconfig - export CGO_CFLAGS="$(pkg-config --cflags rocksdb)" CGO_LDFLAGS="$(pkg-config --libs rocksdb)" - export LD_LIBRARY_PATH=$HOME/.nix-profile/lib - export GLIBC="$(nix build nixpkgs#stdenv.cc.libc.out --no-link --print-out-paths)" - export CGO_LDFLAGS="$CGO_LDFLAGS -Wl,--dynamic-linker=$GLIBC/lib/ld-linux-x86-64.so.2" - echo $CGO_LDFLAGS cd ./versiondb - go test -v -mod=readonly ./... -tags rocksdb,grocksdb_clean_link -coverprofile=coverage.txt + nix-shell -I nixpkgs=../nix -p rocksdb go_1_20 --run "go test -v -mod=readonly ./... -tags rocksdb,grocksdb_clean_link -coverprofile=coverage.txt" if: steps.changed-files.outputs.any_changed == 'true' - name: filter out proto files run: | diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index bea8652214..8c394c4002 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -17,7 +17,7 @@ jobs: golangci: name: Run golangci-lint runs-on: ubuntu-latest - timeout-minutes: 10 + timeout-minutes: 40 steps: - uses: actions/setup-go@v3 with: @@ -40,8 +40,8 @@ jobs: *.sum - name: run golangci-lint run: | - nix registry add nixpkgs github:NixOS/nixpkgs/23.05 - nix profile install nixpkgs#rocksdb nixpkgs#golangci-lint + nix profile install nixpkgs#golangci-lint + nix profile install -f ./nix rocksdb export PKG_CONFIG_PATH=$HOME/.nix-profile/lib/pkgconfig export CGO_CFLAGS="$(pkg-config --cflags rocksdb)" CGO_LDFLAGS="$(pkg-config --libs rocksdb)" golangci-lint version diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e047f5d31..a420a2b66d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ - Avoid channel get changed when concurrent subscribe happens ([ethermint commit](https://github.com/crypto-org-chain/ethermint/commit/72bbe0a80dfd3c586868e2f0b4fbed72593c45bf)). - call Close method on app to cleanup resource on graceful shutdown ([ethermint commit](https://github.com/crypto-org-chain/ethermint/commit/0ea7b86532a1144f229961f94b4524d5889e874d)). - [#1081](https://github.com/crypto-org-chain/cronos/pull/1081) Build with nativebyteorder by default, the released binaries only support little-endian machines, big-endian machines need to build custom binary for themselves. +- [#940](https://github.com/crypto-org-chain/cronos/pull/940) Update rocksdb dependency to 8.1.1. ### Bug Fixes diff --git a/cmd/cronosd/opendb/opendb_rocksdb.go b/cmd/cronosd/opendb/opendb_rocksdb.go index ee17812135..ec1f7eff70 100644 --- a/cmd/cronosd/opendb/opendb_rocksdb.go +++ b/cmd/cronosd/opendb/opendb_rocksdb.go @@ -107,6 +107,7 @@ func NewRocksdbOptions(opts *grocksdb.Options, sstFileWriter bool) *grocksdb.Opt // http://rocksdb.org/blog/2017/05/12/partitioned-index-filter.html bbto.SetIndexType(grocksdb.KTwoLevelIndexSearchIndexType) bbto.SetPartitionFilters(true) + bbto.SetOptimizeFiltersForMemory(true) // hash index is better for iavl tree which mostly do point lookup. bbto.SetDataBlockIndexType(grocksdb.KDataBlockIndexTypeBinarySearchAndHash) diff --git a/go.mod b/go.mod index 95c84ed885..6e2f1c1ed1 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/golang/protobuf v1.5.3 github.com/gorilla/mux v1.8.0 github.com/grpc-ecosystem/grpc-gateway v1.16.0 - github.com/linxGnu/grocksdb v1.7.16 + github.com/linxGnu/grocksdb v1.8.0 github.com/peggyjv/gravity-bridge/module/v2 v2.0.0-00010101000000-000000000000 github.com/rakyll/statik v0.1.7 github.com/spf13/cast v1.5.0 @@ -26,6 +26,7 @@ require ( github.com/tendermint/tendermint v0.34.28 github.com/tendermint/tm-db v0.6.7 golang.org/x/exp v0.0.0-20230310171629-522b1b587ee0 + golang.org/x/sys v0.8.0 google.golang.org/genproto v0.0.0-20230323212658-478b75c54725 google.golang.org/grpc v1.54.0 google.golang.org/protobuf v1.30.0 @@ -207,7 +208,6 @@ require ( golang.org/x/net v0.10.0 // indirect golang.org/x/oauth2 v0.6.0 // indirect golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.8.0 // indirect golang.org/x/term v0.8.0 // indirect golang.org/x/text v0.9.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect diff --git a/go.sum b/go.sum index cf63123b4a..8cc8dea442 100644 --- a/go.sum +++ b/go.sum @@ -1943,8 +1943,9 @@ github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-b github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/linuxkit/virtsock v0.0.0-20201010232012-f8cee7dfc7a3/go.mod h1:3r6x7q95whyfWQpmGZTu3gk3v2YkMi05HEzl7Tf7YEo= github.com/linxGnu/grocksdb v1.7.15/go.mod h1:pY55D0o+r8yUYLq70QmhdudxYvoDb9F+9puf4m3/W+U= -github.com/linxGnu/grocksdb v1.7.16 h1:Q2co1xrpdkr5Hx3Fp+f+f7fRGhQFQhvi/+226dtLmA8= github.com/linxGnu/grocksdb v1.7.16/go.mod h1:JkS7pl5qWpGpuVb3bPqTz8nC12X3YtPZT+Xq7+QfQo4= +github.com/linxGnu/grocksdb v1.8.0 h1:H4L/LhP7GOMf1j17oQAElHgVlbEje2h14A8Tz9cM2BE= +github.com/linxGnu/grocksdb v1.8.0/go.mod h1:09CeBborffXhXdNpEcOeZrLKEnRtrZFEpFdPNI9Zjjg= github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/lucasjones/reggen v0.0.0-20180717132126-cdb49ff09d77/go.mod h1:5ELEyG+X8f+meRWHuqUOewBOhvHkl7M76pdGEansxW4= github.com/lufeee/execinquery v1.2.1/go.mod h1:EC7DrEKView09ocscGHC+apXMIaorh4xqSxS/dy8SbM= diff --git a/gomod2nix.toml b/gomod2nix.toml index 2ce252fc9e..60c5fab18a 100644 --- a/gomod2nix.toml +++ b/gomod2nix.toml @@ -381,8 +381,8 @@ schema = 3 version = "v0.1.0" hash = "sha256-wQqGTtRWsfR9n0O/SXHVgECebbnNmHddxJIbG63OJBQ=" [mod."github.com/linxGnu/grocksdb"] - version = "v1.7.16" - hash = "sha256-XJG6LqvXvlwpkAsuSYR+IPTDzByNdcS7oQA+eg98+D4=" + version = "v1.8.0" + hash = "sha256-o6zj18at4oN6pqSioFqd1AXduR/cx0xDgtL1rPPw+1M=" [mod."github.com/magiconair/properties"] version = "v1.8.6" hash = "sha256-xToSfpuePctkTdhJtsuKIEkXwfMZbnkFT98ahIfd4wY=" diff --git a/nix/rocksdb.nix b/nix/rocksdb.nix index 601a71ed80..8bcbc35060 100644 --- a/nix/rocksdb.nix +++ b/nix/rocksdb.nix @@ -21,13 +21,13 @@ stdenv.mkDerivation rec { pname = "rocksdb"; - version = "7.10.2"; + version = "8.1.1"; src = fetchFromGitHub { owner = "facebook"; repo = pname; rev = "v${version}"; - sha256 = "sha256-U2ReSrJwjAXUdRmwixC0DQXht/h/6rV8SOf5e2NozIs="; + sha256 = "sha256-79hRtc5QSWLLyjRGCmuYZSoIc9IcIsnl8UCinz2sVw4="; }; nativeBuildInputs = [ cmake ninja ]; diff --git a/versiondb/go.mod b/versiondb/go.mod index cd90451a02..521cd1d4b4 100644 --- a/versiondb/go.mod +++ b/versiondb/go.mod @@ -11,10 +11,10 @@ require ( github.com/gogo/protobuf v1.3.3 github.com/golang/snappy v0.0.4 github.com/hashicorp/go-multierror v1.1.1 - github.com/linxGnu/grocksdb v1.7.16 + github.com/linxGnu/grocksdb v1.8.0 github.com/spf13/cast v1.5.0 github.com/spf13/cobra v1.6.1 - github.com/stretchr/testify v1.8.2 + github.com/stretchr/testify v1.8.3 github.com/tendermint/tendermint v0.34.28 github.com/tendermint/tm-db v0.6.7 ) diff --git a/versiondb/go.sum b/versiondb/go.sum index 94a61df616..a2bf6cd5ff 100644 --- a/versiondb/go.sum +++ b/versiondb/go.sum @@ -680,8 +680,8 @@ github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6 github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= -github.com/linxGnu/grocksdb v1.7.16 h1:Q2co1xrpdkr5Hx3Fp+f+f7fRGhQFQhvi/+226dtLmA8= -github.com/linxGnu/grocksdb v1.7.16/go.mod h1:JkS7pl5qWpGpuVb3bPqTz8nC12X3YtPZT+Xq7+QfQo4= +github.com/linxGnu/grocksdb v1.8.0 h1:H4L/LhP7GOMf1j17oQAElHgVlbEje2h14A8Tz9cM2BE= +github.com/linxGnu/grocksdb v1.8.0/go.mod h1:09CeBborffXhXdNpEcOeZrLKEnRtrZFEpFdPNI9Zjjg= github.com/lucasjones/reggen v0.0.0-20180717132126-cdb49ff09d77/go.mod h1:5ELEyG+X8f+meRWHuqUOewBOhvHkl7M76pdGEansxW4= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= @@ -971,8 +971,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= -github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.4.1 h1:jyEFiXpy21Wm81FBN71l9VoMMV8H8jG+qIK3GCpY6Qs= github.com/subosito/gotenv v1.4.1/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= diff --git a/versiondb/tsrocksdb/opts.go b/versiondb/tsrocksdb/opts.go index 55b9f941d1..f0145591f1 100644 --- a/versiondb/tsrocksdb/opts.go +++ b/versiondb/tsrocksdb/opts.go @@ -30,6 +30,7 @@ func NewVersionDBOpts(sstFileWriter bool) *grocksdb.Options { bbto.SetFilterPolicy(grocksdb.NewRibbonHybridFilterPolicy(9.9, 1)) bbto.SetIndexType(grocksdb.KBinarySearchWithFirstKey) + bbto.SetOptimizeFiltersForMemory(true) opts.SetBlockBasedTableFactory(bbto) // improve sst file creation speed: compaction or sst file writer. opts.SetCompressionOptionsParallelThreads(4)