gRPC crashes with concurrent delegations queries

[RiccardoM]

Summary of Bug

Querying the gRPC concurrently for validator delegations makes it crash with error fatal error: concurrent map writes.

Version

v0.40.1

Steps to Reproduce

Try querying the redelegations for multiple validators concurrently.

Log output

fatal error: concurrent map writes
goroutine 3385544 [running]:
runtime.throw(0x1cae431, 0x15)
        runtime/panic.go:1116 +0x72 fp=0xc023622c20 sp=0xc023622bf0 pc=0x43d292
runtime.mapassign(0x1a42020, 0xc00118ac60, 0xc023622f40, 0x104)
        runtime/map.go:585 +0x5c5 fp=0xc023622ca0 sp=0xc023622c20 pc=0x415b25
github.com/cosmos/cosmos-sdk/x/staking/keeper.Keeper.GetValidator(0x20f3820, 0xc000046720, 0x212a600, 0xc00221a320, 0x2118360, 0xc0001e2c60, 0x7f133d2d3118, 0xc0022782c0, 0x212a100, 0xc00000f1c0, ...)
        github.com/cosmos/cosmos-sdk@v0.40.1/x/staking/keeper/validator.go:51 +0x5a5 fp=0xc023623758 sp=0xc023622ca0 pc=0x147ef05
github.com/cosmos/cosmos-sdk/x/staking/keeper.DelegationToDelegationResponse(0x2114620, 0xc00011c010, 0x2129f80, 0xc0252a4a00, 0xb, 0x0, 0xc01335b430, 0xe, 0x4677e, 0x2d46171f, ...)
        github.com/cosmos/cosmos-sdk@v0.40.1/x/staking/keeper/querier.go:451 +0x138 fp=0xc023624010 sp=0xc023623758 pc=0x14738f8
github.com/cosmos/cosmos-sdk/x/staking/keeper.DelegationsToDelegationResponses(0x2114620, 0xc00011c010, 0x2129f80, 0xc0252a4a00, 0xb, 0x0, 0xc01335b430, 0xe, 0x4677e, 0x2d46171f, ...)
        github.com/cosmos/cosmos-sdk@v0.40.1/x/staking/keeper/querier.go:475 +0x188 fp=0xc023624408 sp=0xc023624010 pc=0x1474048
github.com/cosmos/cosmos-sdk/x/staking/keeper.Querier.ValidatorDelegations(0x20f3820, 0xc000046720, 0x212a600, 0xc00221a320, 0x2118360, 0xc0001e2c60, 0x7f133d2d3118, 0xc0022782c0, 0x212a100, 0xc00000f1c0, ...)
        github.com/cosmos/cosmos-sdk@v0.40.1/x/staking/keeper/grpc_query.go:126 +0x488 fp=0xc023624e38 sp=0xc023624408 pc=0x145f6e8
github.com/cosmos/cosmos-sdk/x/staking/keeper.(*Querier).ValidatorDelegations(0xc00003d200, 0x21146a0, 0xc025410540, 0xc02360ece0, 0xc00003d200, 0xc0252f1328, 0x6)
        <autogenerated>:1 +0xe8 fp=0xc023624fe8 sp=0xc023624e38 pc=0x14af248
github.com/cosmos/cosmos-sdk/x/staking/types._Query_ValidatorDelegations_Handler.func1(0x21146a0, 0xc025410540, 0x1be3580, 0xc02360ece0, 0x0, 0xc025412240, 0x21146a0, 0xc025410540)
        github.com/cosmos/cosmos-sdk@v0.40.1/x/staking/types/query.pb.go:1829 +0x89 fp=0xc023625030 sp=0xc023624fe8 pc=0x12e55c9
github.com/cosmos/cosmos-sdk/baseapp.(*BaseApp).RegisterGRPCServer.func1(0x21146a0, 0xc025410540, 0x1be3580, 0xc02360ece0, 0xc02360ed00, 0xc02360ed20, 0x0, 0x0, 0xc007aa39f0, 0x4133f0)
        github.com/cosmos/cosmos-sdk@v0.40.1/baseapp/grpcserver.go:65 +0x365 fp=0xc023625910 sp=0xc023625030 pc=0x11bee45
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x21146a0, 0xc02361b0b0, 0x1be3580, 0xc02360ece0, 0x7f13244cc6e8, 0x20, 0x20, 0x7f1366aaa108)
        github.com/grpc-ecosystem/go-grpc-middleware@v1.2.2/chain.go:25 +0x63 fp=0xc023625970 sp=0xc023625910 pc=0x11adc23
github.com/grpc-ecosystem/go-grpc-middleware/recovery.UnaryServerInterceptor.func1(0x21146a0, 0xc02361b0b0, 0x1be3580, 0xc02360ece0, 0xc02360ed00, 0xc02360ed40, 0x0, 0x0, 0x0, 0x0)
        github.com/grpc-ecosystem/go-grpc-middleware@v1.2.2/recovery/interceptors.go:33 +0xcb fp=0xc0236259f8 sp=0xc023625970 pc=0x11ae2ab
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x21146a0, 0xc02361b0b0, 0x1be3580, 0xc02360ece0, 0xc022a07500, 0x0, 0xc007aa3ab8, 0x413d58)
        github.com/grpc-ecosystem/go-grpc-middleware@v1.2.2/chain.go:25 +0x63 fp=0xc023625a58 sp=0xc0236259f8 pc=0x11adc23
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1(0x21146a0, 0xc02361b0b0, 0x1be3580, 0xc02360ece0, 0xc02360ed00, 0xc02360ed20, 0xc007aa3b01, 0xc02361b0e0, 0xc007aa3b28, 0x11adb4d)
        github.com/grpc-ecosystem/go-grpc-middleware@v1.2.2/chain.go:34 +0xd7 fp=0xc023625ac8 sp=0xc023625a58 pc=0x11ade17
github.com/cosmos/cosmos-sdk/x/staking/types._Query_ValidatorDelegations_Handler(0x1c858e0, 0xc00003d200, 0x21146a0, 0xc02361b0b0, 0xc02360d740, 0xc02361b0e0, 0xc02361b0b0, 0xc007aa3ba0, 0x52bb06, 0x1b6a940)
        github.com/cosmos/cosmos-sdk@v0.40.1/x/staking/types/query.pb.go:1831 +0x150 fp=0xc023625b38 sp=0xc023625ac8 pc=0x1292e70
github.com/cosmos/cosmos-sdk/baseapp.(*BaseApp).RegisterGRPCServer.func2(0x1c858e0, 0xc00003d200, 0x21146a0, 0xc02361b0b0, 0xc02360d740, 0x0, 0x21146a0, 0xc02361b0b0, 0xc020579e40, 0x3a)
        github.com/cosmos/cosmos-sdk@v0.40.1/baseapp/grpcserver.go:79 +0xf6 fp=0xc023625bb0 sp=0xc023625b38 pc=0x11bf1f6
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00227ec40, 0x2126b40, 0xc011cec000, 0xc022a07500, 0xc006d3bd40, 0xc006d0b210, 0x0, 0x0, 0x0)
        google.golang.org/grpc@v1.35.0/server.go:1217 +0x522 fp=0xc023625e40 sp=0xc023625bb0 pc=0xaadd22
google.golang.org/grpc.(*Server).handleStream(0xc00227ec40, 0x2126b40, 0xc011cec000, 0xc022a07500, 0x0)
        google.golang.org/grpc@v1.35.0/server.go:1540 +0xd05 fp=0xc023625f68 sp=0xc023625e40 pc=0xab1ea5
google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc01d9f2be0, 0xc00227ec40, 0x2126b40, 0xc011cec000, 0xc022a07500)
        google.golang.org/grpc@v1.35.0/server.go:878 +0xa5 fp=0xc023625fb8 sp=0xc023625f68 pc=0xabefa5
runtime.goexit()
        runtime/asm_amd64.s:1374 +0x1 fp=0xc023625fc0 sp=0xc023625fb8 pc=0x475c21
created by google.golang.org/grpc.(*Server).serveStreams.func1
        google.golang.org/grpc@v1.35.0/server.go:876 +0x1fd
goroutine 1 [chan receive, 1170 minutes]:
github.com/cosmos/cosmos-sdk/server.WaitForQuitSignals(0x212fb80)
        github.com/cosmos/cosmos-sdk@v0.40.1/server/util.go:297 +0xbf
github.com/cosmos/cosmos-sdk/server.startInProcess(0xc000e541e0, 0x0, 0x0, 0x0, 0x21362c0, 0xc006d2f0c0, 0x0, 0x0, 0x211ce80, 0xc000bc0cf0, ...)
        github.com/cosmos/cosmos-sdk@v0.40.1/server/start.go:334 +0x837
github.com/cosmos/cosmos-sdk/server.StartCmd.func2(0xc000dcdb80, 0x2dc8350, 0x0, 0x0, 0x0, 0x0)
        github.com/cosmos/cosmos-sdk@v0.40.1/server/start.go:120 +0x169
github.com/spf13/cobra.(*Command).execute(0xc000dcdb80, 0x2dc8350, 0x0, 0x0, 0xc000dcdb80, 0x2dc8350)
        github.com/spf13/cobra@v1.1.1/command.go:850 +0x47c
github.com/spf13/cobra.(*Command).ExecuteC(0xc000026dc0, 0x0, 0x0, 0xc000bbc4e0)
        github.com/spf13/cobra@v1.1.1/command.go:958 +0x375
github.com/spf13/cobra.(*Command).Execute(...)
        github.com/spf13/cobra@v1.1.1/command.go:895
github.com/spf13/cobra.(*Command).ExecuteContext(...)
        github.com/spf13/cobra@v1.1.1/command.go:888
github.com/cosmos/cosmos-sdk/server/cmd.Execute(0xc000026dc0, 0xc000bbc4e0, 0xd, 0x212fae0, 0xc000bc0cf0)
        github.com/cosmos/cosmos-sdk@v0.40.1/server/cmd/execute.go:36 +0x265
main.main()
        github.com/desmos-labs/desmos/app/desmos/main.go:17 +0x45
goroutine 7 [select, 1 minutes]:
github.com/syndtr/goleveldb/leveldb.(*DB).tCompaction(0xc00015c1c0)
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/db_compaction.go:836 +0x265
created by github.com/syndtr/goleveldb/leveldb.openDB
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/db.go:155 +0x5a5
goroutine 6 [select]:
github.com/syndtr/goleveldb/leveldb.(*DB).mpoolDrain(0xc00015c1c0)
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/db_state.go:101 +0x105
created by github.com/syndtr/goleveldb/leveldb.openDB
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/db.go:149 +0x44d
goroutine 5 [select, 1 minutes]:
github.com/syndtr/goleveldb/leveldb.(*DB).compactionError(0xc00015c1c0)
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/db_compaction.go:91 +0xd1
created by github.com/syndtr/goleveldb/leveldb.openDB
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/db.go:148 +0x42b
goroutine 29 [select]:
github.com/syndtr/goleveldb/leveldb/util.(*BufferPool).drain(0xc00015a460)
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/util/buffer_pool.go:209 +0x128
created by github.com/syndtr/goleveldb/leveldb/util.NewBufferPool
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/util/buffer_pool.go:240 +0x18c
goroutine 30 [select, 1 minutes]:
github.com/syndtr/goleveldb/leveldb.(*session).refLoop(0xc0003c7b30)
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/session_util.go:189 +0x60f
created by github.com/syndtr/goleveldb/leveldb.newSession
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/session.go:93 +0x2bd
goroutine 8 [select, 1 minutes]:
github.com/syndtr/goleveldb/leveldb.(*DB).mCompaction(0xc00015c1c0)
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/db_compaction.go:773 +0x145
created by github.com/syndtr/goleveldb/leveldb.openDB
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/db.go:156 +0x5c7
goroutine 9 [select]:
github.com/syndtr/goleveldb/leveldb/util.(*BufferPool).drain(0xc00015a1c0)
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/util/buffer_pool.go:209 +0x128
created by github.com/syndtr/goleveldb/leveldb/util.NewBufferPool
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/util/buffer_pool.go:240 +0x18c
goroutine 10 [select, 1 minutes]:
github.com/syndtr/goleveldb/leveldb.(*session).refLoop(0xc0003c6f00)
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/session_util.go:189 +0x60f
created by github.com/syndtr/goleveldb/leveldb.newSession
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/session.go:93 +0x2bd
goroutine 11 [select, 1170 minutes]:
github.com/syndtr/goleveldb/leveldb.(*DB).compactionError(0xc00015d340)
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/db_compaction.go:91 +0xd1
created by github.com/syndtr/goleveldb/leveldb.openDB
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/db.go:148 +0x42b
goroutine 12 [select]:
github.com/syndtr/goleveldb/leveldb.(*DB).mpoolDrain(0xc00015d340)
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/db_state.go:101 +0x105
created by github.com/syndtr/goleveldb/leveldb.openDB
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/db.go:149 +0x44d
goroutine 13 [select, 1170 minutes]:
github.com/syndtr/goleveldb/leveldb.(*DB).tCompaction(0xc00015d340)
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/db_compaction.go:836 +0x265
created by github.com/syndtr/goleveldb/leveldb.openDB
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/db.go:155 +0x5a5
goroutine 14 [select, 1170 minutes]:
github.com/syndtr/goleveldb/leveldb.(*DB).mCompaction(0xc00015d340)
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/db_compaction.go:773 +0x145
created by github.com/syndtr/goleveldb/leveldb.openDB
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/db.go:156 +0x5c7
goroutine 50 [select]:
github.com/syndtr/goleveldb/leveldb.(*DB).mpoolDrain(0xc00227e380)
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/db_state.go:101 +0x105
created by github.com/syndtr/goleveldb/leveldb.openDB
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/db.go:149 +0x44d
goroutine 16 [select, 7 minutes]:
github.com/syndtr/goleveldb/leveldb.(*DB).compactionError(0xc00227e380)
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/db_compaction.go:91 +0xd1
created by github.com/syndtr/goleveldb/leveldb.openDB
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/db.go:148 +0x42b
goroutine 33 [select]:
github.com/syndtr/goleveldb/leveldb/util.(*BufferPool).drain(0xc00015a540)
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/util/buffer_pool.go:209 +0x128
created by github.com/syndtr/goleveldb/leveldb/util.NewBufferPool
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/util/buffer_pool.go:240 +0x18c
goroutine 34 [select, 1 minutes]:
github.com/syndtr/goleveldb/leveldb.(*session).refLoop(0xc00d24a3c0)
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/session_util.go:189 +0x60f
created by github.com/syndtr/goleveldb/leveldb.newSession
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/session.go:93 +0x2bd
goroutine 35 [select, 10 minutes]:
github.com/syndtr/goleveldb/leveldb.(*DB).compactionError(0xc00227e1c0)
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/db_compaction.go:91 +0xd1
created by github.com/syndtr/goleveldb/leveldb.openDB
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/db.go:148 +0x42b
goroutine 36 [select]:
github.com/syndtr/goleveldb/leveldb.(*DB).mpoolDrain(0xc00227e1c0)
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/db_state.go:101 +0x105
created by github.com/syndtr/goleveldb/leveldb.openDB
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/db.go:149 +0x44d
goroutine 37 [select, 10 minutes]:
github.com/syndtr/goleveldb/leveldb.(*DB).tCompaction(0xc00227e1c0)
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/db_compaction.go:836 +0x265
created by github.com/syndtr/goleveldb/leveldb.openDB
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/db.go:155 +0x5a5
goroutine 38 [select, 11 minutes]:
github.com/syndtr/goleveldb/leveldb.(*DB).mCompaction(0xc00227e1c0)
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/db_compaction.go:773 +0x145
created by github.com/syndtr/goleveldb/leveldb.openDB
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/db.go:156 +0x5c7
goroutine 39 [select]:
github.com/syndtr/goleveldb/leveldb/util.(*BufferPool).drain(0xc00015a8c0)
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/util/buffer_pool.go:209 +0x128
created by github.com/syndtr/goleveldb/leveldb/util.NewBufferPool
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/util/buffer_pool.go:240 +0x18c
goroutine 40 [select, 2 minutes]:
github.com/syndtr/goleveldb/leveldb.(*session).refLoop(0xc00d24aa50)
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/session_util.go:189 +0x60f
created by github.com/syndtr/goleveldb/leveldb.newSession
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/session.go:93 +0x2bd
goroutine 51 [select, 7 minutes]:
github.com/syndtr/goleveldb/leveldb.(*DB).tCompaction(0xc00227e380)
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/db_compaction.go:836 +0x265
created by github.com/syndtr/goleveldb/leveldb.openDB
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/db.go:155 +0x5a5
goroutine 52 [select, 7 minutes]:
github.com/syndtr/goleveldb/leveldb.(*DB).mCompaction(0xc00227e380)
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/db_compaction.go:773 +0x145
created by github.com/syndtr/goleveldb/leveldb.openDB
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/db.go:156 +0x5c7
goroutine 53 [select, 1170 minutes]:
github.com/tendermint/tendermint/proxy.(*multiAppConn).killTMOnClientError(0xc0019f04e0)
        github.com/tendermint/tendermint@v0.34.3/proxy/multi_app_conn.go:138 +0x174
created by github.com/tendermint/tendermint/proxy.(*multiAppConn).OnStart
        github.com/tendermint/tendermint@v0.34.3/proxy/multi_app_conn.go:118 +0x34e
goroutine 54 [chan receive]:
github.com/tendermint/tendermint/libs/pubsub.(*Server).loop(0xc004c164d0, 0xc001ff6b40, 0xc001ff6b70)
        github.com/tendermint/tendermint@v0.34.3/libs/pubsub/pubsub.go:324 +0x8c
created by github.com/tendermint/tendermint/libs/pubsub.(*Server).OnStart
        github.com/tendermint/tendermint@v0.34.3/libs/pubsub/pubsub.go:310 +0x68
goroutine 55 [select]:
github.com/syndtr/goleveldb/leveldb/util.(*BufferPool).drain(0xc0019a01c0)
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/util/buffer_pool.go:209 +0x128
created by github.com/syndtr/goleveldb/leveldb/util.NewBufferPool
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/util/buffer_pool.go:240 +0x18c
goroutine 56 [select]:
github.com/syndtr/goleveldb/leveldb.(*session).refLoop(0xc001993950)
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/session_util.go:189 +0x60f
created by github.com/syndtr/goleveldb/leveldb.newSession
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/session.go:93 +0x2bd
goroutine 41 [select, 1170 minutes]:
github.com/syndtr/goleveldb/leveldb.(*DB).compactionError(0xc001ffc000)
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/db_compaction.go:91 +0xd1
created by github.com/syndtr/goleveldb/leveldb.openDB
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/db.go:148 +0x42b
goroutine 42 [select]:
github.com/syndtr/goleveldb/leveldb.(*DB).mpoolDrain(0xc001ffc000)
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/db_state.go:101 +0x105
created by github.com/syndtr/goleveldb/leveldb.openDB
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/db.go:149 +0x44d
goroutine 43 [select, 1170 minutes]:
github.com/syndtr/goleveldb/leveldb.(*DB).tCompaction(0xc001ffc000)
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/db_compaction.go:836 +0x265
created by github.com/syndtr/goleveldb/leveldb.openDB
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/db.go:155 +0x5a5
goroutine 44 [select, 1170 minutes]:
github.com/syndtr/goleveldb/leveldb.(*DB).mCompaction(0xc001ffc000)
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/db_compaction.go:773 +0x145
created by github.com/syndtr/goleveldb/leveldb.openDB
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/db.go:156 +0x5c7
goroutine 45 [chan receive]:
github.com/tendermint/tendermint/state/txindex.(*IndexerService).OnStart.func1(0x210be60, 0xc001f4f080, 0x210be60, 0xc001f4f0c0, 0xc001f6a320, 0xc0003c53e0)
        github.com/tendermint/tendermint@v0.34.3/state/txindex/indexer_service.go:53 +0x3af
created by github.com/tendermint/tendermint/state/txindex.(*IndexerService).OnStart
        github.com/tendermint/tendermint@v0.34.3/state/txindex/indexer_service.go:51 +0x1d1
goroutine 85 [select, 1170 minutes]:
github.com/syndtr/goleveldb/leveldb.(*DB).compactionError(0xc00227e700)
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/db_compaction.go:91 +0xd1
created by github.com/syndtr/goleveldb/leveldb.openDB
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/db.go:148 +0x42b
goroutine 86 [select]:
github.com/syndtr/goleveldb/leveldb.(*DB).mpoolDrain(0xc00227e700)
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/db_state.go:101 +0x105
created by github.com/syndtr/goleveldb/leveldb.openDB
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/db.go:149 +0x44d
goroutine 70 [select]:
github.com/syndtr/goleveldb/leveldb/util.(*BufferPool).drain(0xc006ac0000)
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/util/buffer_pool.go:209 +0x128
created by github.com/syndtr/goleveldb/leveldb/util.NewBufferPool
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/util/buffer_pool.go:240 +0x18c
goroutine 71 [select]:
github.com/syndtr/goleveldb/leveldb.(*session).refLoop(0xc005bc3860)
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/session_util.go:189 +0x60f
created by github.com/syndtr/goleveldb/leveldb.newSession
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/session.go:93 +0x2bd
goroutine 87 [select, 1170 minutes]:
github.com/syndtr/goleveldb/leveldb.(*DB).tCompaction(0xc00227e700)
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/db_compaction.go:836 +0x265
created by github.com/syndtr/goleveldb/leveldb.openDB
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/db.go:155 +0x5a5
goroutine 88 [select, 1170 minutes]:
github.com/syndtr/goleveldb/leveldb.(*DB).mCompaction(0xc00227e700)
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/db_compaction.go:773 +0x145
created by github.com/syndtr/goleveldb/leveldb.openDB
        github.com/syndtr/goleveldb@v1.0.1-0.20200815110645-5c35d600f0ca/leveldb/db.go:156 +0x5c7
goroutine 75 [IO wait, 1170 minutes]:
internal/poll.runtime_pollWait(0x7f133de1dce8, 0x72, 0x0)
        runtime/netpoll.go:222 +0x55
internal/poll.(*pollDesc).wait(0xc005f5d698, 0x72, 0x0, 0x0, 0x1c8e24b)
        internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
        internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Accept(0xc005f5d680, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
        internal/poll/fd_unix.go:394 +0x1fc
net.(*netFD).accept(0xc005f5d680, 0xc006ca97d0, 0x413d58, 0xc00005b000)
        net/fd_unix.go:172 +0x45
net.(*TCPListener).accept(0xc006c91160, 0xc0021a1dc0, 0x413d58, 0x30)
        net/tcpsock_posix.go:139 +0x32
net.(*TCPListener).Accept(0xc006c91160, 0x1b6a940, 0xc006ca97d0, 0x1a29e00, 0x2ce7d90)
        net/tcpsock.go:261 +0x65
net/http.(*Server).Serve(0xc0010efc00, 0x210e8a0, 0xc006c91160, 0x0, 0x0)
        net/http/server.go:2937 +0x266
net/http.(*Server).ListenAndServe(0xc0010efc00, 0xc0010efc00, 0x15)
        net/http/server.go:2866 +0xb7
net/http.ListenAndServe(...)
        net/http/server.go:3120
github.com/tendermint/tendermint/node.NewNode.func1(0x21152a0, 0xc000f7c900, 0xc000d9ef00)
        github.com/tendermint/tendermint@v0.34.3/node/node.go:801 +0x145
created by github.com/tendermint/tendermint/node.NewNode
        github.com/tendermint/tendermint@v0.34.3/node/node.go:799 +0x1a0b
goroutine 3385430 [chan receive]:
github.com/cosmos/cosmos-sdk/store/iavl.(*iavlIterator).receiveNext(0xc0135eb4d0)
        github.com/cosmos/cosmos-sdk@v0.40.1/store/iavl/store.go:525 +0x66
github.com/cosmos/cosmos-sdk/store/iavl.(*iavlIterator).Next(0xc0135eb4d0)
        github.com/cosmos/cosmos-sdk@v0.40.1/store/iavl/store.go:463 +0x85
github.com/cosmos/cosmos-sdk/store/cachekv.(*cacheMergeIterator).Next(0xc002a483c0)
        github.com/cosmos/cosmos-sdk@v0.40.1/store/cachekv/mergeiterator.go:74 +0x1b2
github.com/cosmos/cosmos-sdk/store/gaskv.(*gasIterator).Next(0xc01c9f0120)
        github.com/cosmos/cosmos-sdk@v0.40.1/store/gaskv/store.go:150 +0x52
github.com/cosmos/cosmos-sdk/store/prefix.(*prefixIterator).Next(0xc01c883500)
        github.com/cosmos/cosmos-sdk@v0.40.1/store/prefix/store.go:153 +0x42
github.com/cosmos/cosmos-sdk/types/query.FilteredPaginate(0x2127080, 0xc002a48390, 0xc02254da10, 0xc022564788, 0x0, 0x0, 0x0)
        github.com/cosmos/cosmos-sdk@v0.40.1/types/query/filtered_pagination.go:84 +0x4fc
github.com/cosmos/cosmos-sdk/x/staking/keeper.Querier.ValidatorDelegations(0x20f3820, 0xc000046720, 0x212a600, 0xc00221a320, 0x2118360, 0xc0001e2c60, 0x7f133d2d3118, 0xc0022782c0, 0x212a100, 0xc00000f1c0, ...)
        github.com/cosmos/cosmos-sdk@v0.40.1/x/staking/keeper/grpc_query.go:102 +0x2fb
github.com/cosmos/cosmos-sdk/x/staking/types._Query_ValidatorDelegations_Handler.func1(0x21146a0, 0xc002a48300, 0x1be3580, 0xc022535ca0, 0x0, 0xc006b0ad80, 0x21146a0, 0xc002a48300)
        github.com/cosmos/cosmos-sdk@v0.40.1/x/staking/types/query.pb.go:1829 +0x89
github.com/cosmos/cosmos-sdk/baseapp.(*BaseApp).RegisterGRPCServer.func1(0x21146a0, 0xc002a48300, 0x1be3580, 0xc022535ca0, 0xc022535cc0, 0xc022535ce0, 0x0, 0x0, 0xc0085029f0, 0x4133f0)
        github.com/cosmos/cosmos-sdk@v0.40.1/baseapp/grpcserver.go:65 +0x365
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x21146a0, 0xc02254d9b0, 0x1be3580, 0xc022535ca0, 0x7f133c1967e0, 0x20, 0x20, 0x7f1366aaa7d0)
        github.com/grpc-ecosystem/go-grpc-middleware@v1.2.2/chain.go:25 +0x63
github.com/grpc-ecosystem/go-grpc-middleware/recovery.UnaryServerInterceptor.func1(0x21146a0, 0xc02254d9b0, 0x1be3580, 0xc022535ca0, 0xc022535cc0, 0xc022535d00, 0x0, 0x0, 0x0, 0x0)
        github.com/grpc-ecosystem/go-grpc-middleware@v1.2.2/recovery/interceptors.go:33 +0xcb
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x21146a0, 0xc02254d9b0, 0x1be3580, 0xc022535ca0, 0xc02250c200, 0x0, 0xc008502ab8, 0x413d58)
        github.com/grpc-ecosystem/go-grpc-middleware@v1.2.2/chain.go:25 +0x63
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1(0x21146a0, 0xc02254d9b0, 0x1be3580, 0xc022535ca0, 0xc022535cc0, 0xc022535ce0, 0xc008502b01, 0xc02254d9e0, 0xc008502b28, 0x11adb4d)
        github.com/grpc-ecosystem/go-grpc-middleware@v1.2.2/chain.go:34 +0xd7
github.com/cosmos/cosmos-sdk/x/staking/types._Query_ValidatorDelegations_Handler(0x1c858e0, 0xc00003d200, 0x21146a0, 0xc02254d9b0, 0xc0225477a0, 0xc02254d9e0, 0xc02254d9b0, 0xc008502ba0, 0x52bb06, 0x1b6a940)
        github.com/cosmos/cosmos-sdk@v0.40.1/x/staking/types/query.pb.go:1831 +0x150
github.com/cosmos/cosmos-sdk/baseapp.(*BaseApp).RegisterGRPCServer.func2(0x1c858e0, 0xc00003d200, 0x21146a0, 0xc02254d9b0, 0xc0225477a0, 0x0, 0x21146a0, 0xc02254d9b0, 0xc0129a1840, 0x3a)

Code example

// UpdateValidatorsUnbondingDelegations updates the redelegations for all the validators provided
func UpdateValidatorsRedelegations(
	height int64, validators []stakingtypes.Validator, client stakingtypes.QueryClient, db *database.BigDipperDb,
) error {
	log.Debug().Str("module", "staking").Int64("height", height).Msg("updating validators redelegations")

	params, err := db.GetStakingParams()
	if err != nil {
		return err
	}

	var wg sync.WaitGroup
	var out = make(chan types.Redelegation)
	for _, val := range validators {
		wg.Add(1)
		go getRedelegations(val.OperatorAddress, params.BondName, height, client, out, &wg)
	}

	// We need to call wg.Wait inside another goroutine in order to solve the hanging bug that's described here:
	// https://dev.to/sophiedebenedetto/synchronizing-go-routines-with-channels-and-waitgroups-3ke2
	go func() {
		wg.Wait()
		close(out)
	}()

	var redelegations []types.Redelegation
	for del := range out {
		redelegations = append(redelegations, del)
	}

	return db.SaveRedelegations(redelegations)
}

func getRedelegations(
	validatorAddress string, bondDenom string, height int64, client stakingtypes.QueryClient,
	out chan<- types.Redelegation, wg *sync.WaitGroup,
) {
	defer wg.Done()

	header := utils.GetHeightRequestHeader(height)

	var nextKey []byte
	var stop = false
	for !stop {
		res, err := client.Redelegations(
			context.Background(),
			&stakingtypes.QueryRedelegationsRequest{
				SrcValidatorAddr: validatorAddress,
				Pagination: &query.PageRequest{
					Key:   nextKey,
					Limit: 100, // Query 100 unbonding delegations at a time
				},
			},
			header,
		)
		if err != nil {
			log.Error().Str("module", "staking").Err(err).
				Msg("error while getting validators redelegations")
			return
		}

		for _, delegation := range res.RedelegationResponses {
			for _, entry := range delegation.Entries {
				out <- types.NewRedelegation(
					delegation.Redelegation.DelegatorAddress,
					delegation.Redelegation.ValidatorSrcAddress,
					delegation.Redelegation.ValidatorDstAddress,
					sdk.NewCoin(bondDenom, entry.Balance),
					entry.RedelegationEntry.CompletionTime,
					entry.RedelegationEntry.CreationHeight,
				)
			}
		}

		nextKey = res.Pagination.NextKey
		stop = len(res.Pagination.NextKey) == 0
	}
}

---

For Admin Use

• Not duplicate issue
• Appropriate labels applied
• Appropriate contributors tagged
• Contributor assigned/self-assigned

[RiccardoM]

The problem seem to be the following line:

k.validatorCache[strValue] = newCachedValidator(validator, strValue)

This is used in order to cache the queried validator and avoid duplicate operations later on. There seem to be two ways of fix this:

1. Add a RWMutex on the map.
2. Remove the caching.

[ethanfrey]

I took a quick look and that caching seems quite odd.

First, it refers to caching the amino decoding for performance reasons, which don't seem to apply so much with protobuf (that no one updated the comment, means no one really thought about whether it still made sense).

Secondly, the cache is only updated on GetValidator. But not on SetValidator or RemoveValidator. After re-reading it, it seems that it stores a mapping of encoded bytes to decoded structure, and doesn't do any caching of the db queries, so I guess it is only a work-around for poor amino performance. It does look a rather like duct-tape to me, though.

I would highly suggest removing this caching. And if it does turn out with benchmarking that this is a bottleneck, then reimplement it much more robustly.

[RiccardoM]

@ethanfrey Thanks for confirming that I thought as well. I've just run the following benchmark test:

func BenchmarkGetValidator(b *testing.B) {
	// 900 is the max number we are allowed to use in order to avoid simapp.CreateTestPubKeys
	// panic: encoding/hex: odd length hex string
	var powersNumber = 900

	var totalPower int64 = 0
	var powers = make([]int64, powersNumber)
	for i := range powers {
		powers[i] = int64(i)
		totalPower += int64(i)
	}

	app, ctx, _, valAddrs, vals := initValidators(b, totalPower, len(powers), powers)

	for _, validator := range vals {
		app.StakingKeeper.SetValidator(ctx, validator)
	}

	b.ResetTimer()
	for _, addr := range valAddrs {
		_, _ = app.StakingKeeper.GetValidator(ctx, addr)
	}
}

Here are the results:

• With caching
  BenchmarkGetValidator-16 1000000000 0.00789 ns/op
• Without caching
  BenchmarkGetValidator-16 1000000000 0.00763 ns/op

[ethanfrey]

Nice benchmark. But those numbers are way too low (< 1ns??)

You need to add a loop to do this b.N times. The last lines should read:

b.ResetTimer()
for n := 0; n < b.N; n++ {
    for _, addr := range valAddrs {
        _, _ = app.StakingKeeper.GetValidator(ctx, addr)
    }
}

Can you update the benchmark and give the new results?

[RiccardoM]

@ethanfrey Sure thing, here you go:

Running it one time

• With caching:
  BenchmarkGetValidator-16 129 8589234 ns/op
• Without caching:
  BenchmarkGetValidator-16 297 4199785 ns/op

Running it three times, output the third time:

• With caching:
  BenchmarkGetValidator-16 139 8484213 ns/op
• Without caching:
  BenchmarkGetValidator-16 285 4110587 ns/op

[ethanfrey]

So, it not only crashes the node, but it makes it slower???

Time to go!

[clevinson]

Dropping the backport label, as it actually should be added to the PR that fixes this :)

Thanks for digging into this @RiccardoM !