cockroachdb hit SIGSEGV during test suite

[davepacheco]

While trying to reproduce #1130, I ran into a different SIGSEGV in CockroachDB:

I220616 22:45:17.402378 1 util/log/flags.go:201  [-] 1  stderr capture started
unexpected fault address 0x108226a
fatal error: fault
[signal SIGSEGV: segmentation violation code=0x1 addr=0x108226a pc=0x108226a]

goroutine 831 [running]:
runtime.throw(0x59bf4fa, 0x5)
        /opt/ooce/go-1.16/src/runtime/panic.go:1117 +0x72 fp=0xc002cb8480 sp=0xc002cb8450 pc=0x10a9512
runtime.sigpanic()
        /opt/ooce/go-1.16/src/runtime/signal_unix.go:741 +0x276 fp=0xc002cb84b8 sp=0xc002cb8480 pc=0x10c0e96
runtime.mapassign_fast32(0x5292480, 0xc0038201b0, 0x1, 0x20)
        /opt/ooce/go-1.16/src/runtime/map_fast32.go:126 +0xaa fp=0xc002cb84f8 sp=0xc002cb84b8 pc=0x108226a
github.com/cockroachdb/cockroach/pkg/sql/catalog.(*collectorState).addDirectReferences(0xc00392e000, 0xfffffc7fee647ec0, 0xc000106e10, 0x0, 0x0)
        /ws/gc/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/sql/catalog/validate.go:442 +0x5a fp=0xc002cb8540 sp=0xc002cb84f8 pc=0x26d231a
github.com/cockroachdb/cockroach/pkg/sql/catalog.collectDescriptorsForValidation(0x6fd9f90, 0xc0036fa1e0, 0x6f97070, 0xc0035cfe60, 0xc0035cfe50, 0x1, 0x1, 0xc000299b00, 0xc000cb4480, 0xc002cb8670)
        /ws/gc/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/sql/catalog/validate.go:500 +0x159 fp=0xc002cb85e8 sp=0xc002cb8540 pc=0x26d2b19
github.com/cockroachdb/cockroach/pkg/sql/catalog.Validate(0x6fd9f90, 0xc0036fa1e0, 0x6f97070, 0xc0035cfe60, 0x2, 0xc00000001f, 0xc0035cfe50, 0x1, 0x1, 0x4, ...)
        /ws/gc/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/sql/catalog/validate.go:55 +0x14f fp=0xc002cb8680 sp=0xc002cb85e8 pc=0x26cff2f
github.com/cockroachdb/cockroach/pkg/sql/catalog/descs.(*Collection).ValidateUncommittedDescriptors(0xc000cae298, 0x6fd9f90, 0xc0036fa1e0, 0xc002eb2dc0, 0x0, 0x62abb280)
        /ws/gc/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/sql/catalog/descs/validate.go:43 +0x15a fp=0xc002cb8710 sp=0xc002cb8680 pc=0x2d2325a
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc000cae000, 0x6fd9f90, 0xc0036fa1e0, 0x7002558, 0xc00283d9a0, 0xcc, 0xc002cb87f8)
        /ws/gc/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:883 +0xa5 fp=0xc002cb8758 sp=0xc002cb8710 pc=0x4187985
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal-fm(0x6fd9f90, 0xc0036fa1e0, 0x7002558, 0xc00283d9a0, 0x204d597, 0xc002eb2df8)
        /ws/gc/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:870 +0x5b fp=0xc002cb87a0 sp=0xc002cb8758 pc=0x440369b
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransaction(0xc000cae000, 0x6fd9f90, 0xc0036fa1e0, 0x7002558, 0xc00283d9a0, 0xc002cb8890, 0xfffffc7fedf982c8, 0x0, 0x52ca140, 0x95b44c8)
        /ws/gc/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:820 +0x9c fp=0xc002cb8808 sp=0xc002cb87a0 pc=0x418715c
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).handleAutoCommit(0xc000cae000, 0x6fd9f90, 0xc0036fa1e0, 0x7002558, 0xc00283d9a0, 0x60859b8, 0xc002cb8928, 0x40, 0xc000cb4480)
        /ws/gc/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1837 +0xdc fp=0xc002cb88b0 sp=0xc002cb8808 pc=0x418e83c
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState.func7(0xc000cae000, 0xc002cb92c8, 0xc002cb92b8, 0xc002cb9248, 0xc002cb8e20, 0xc002cb92a8, 0x6f974a8, 0x921b59b)
        /ws/gc/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:502 +0xc9 fp=0xc002cb8928 sp=0xc002cb88b0 pc=0x43748e9
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc000cae000, 0x6fd9f90, 0xc0036fa1e0, 0x7002558, 0xc00283d9a0, 0xc0018de306, 0x8, 0x0, 0x0, 0xc002926300, ...)
        /ws/gc/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:755 +0x1251 fp=0xc002cb9240 sp=0xc002cb8928 pc=0x4184671
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc000cae000, 0x6fd9ee8, 0xc002a4ba40, 0x7002558, 0xc00283d9a0, 0xc0018de306, 0x8, 0x0, 0x0, 0xc002926300, ...)
        /ws/gc/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:129 +0xb0a fp=0xc002cb94d0 sp=0xc002cb9240 pc=0x4182b8a
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc000cae000, 0x6fd9ee8, 0xc002a4ba40, 0xc002926300, 0x95b44c8, 0x0, 0x0, 0xc002a214de, 0x1, 0x1, ...)
        /ws/gc/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:216 +0x14e fp=0xc002cb9608 sp=0xc002cb94d0 pc=0x418316e
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2(0xc0018de6ed, 0x0, 0x0, 0x3a038e9f, 0xeda3da97f, 0x0, 0xc000cae000, 0xc002cb99e8, 0xc002cb99d8, 0xc0026e18d8, ...)
        /ws/gc/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1773 +0x4fb fp=0xc002cb97e8 sp=0xc002cb9608 pc=0x437313b
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc000cae000, 0x6fd9ee8, 0xc002a4ba40, 0x0, 0x0)
        /ws/gc/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1775 +0x4df fp=0xc002cb9cb0 sp=0xc002cb97e8 pc=0x417591f
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc000cae000, 0x6fd9ee8, 0xc002c54740, 0xc00110ce60, 0x5400, 0x15000, 0xc00110d0e0, 0xc002c4ea10, 0x0, 0x0)
        /ws/gc/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1629 +0x22f fp=0xc002cb9d50 sp=0xc002cb9cb0 pc=0x41751cf
github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc000559c00, 0x6fd9ee8, 0xc002c54740, 0xc000cae000, 0x5400, 0x15000, 0xc00110d0e0, 0xc002c4ea10, 0x0, 0x0)
        /ws/gc/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:668 +0xce fp=0xc002cb9dd8 sp=0xc002cb9d50 pc=0x4170d6e
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1(0xc001a4a214, 0xc002f44330, 0x6fd9ee8, 0xc002c54740, 0xc002c4ea10, 0xc000559c00, 0xc001b92900, 0x70492c0, 0xc001af82d0, 0xc002c3be60, ...)
        /ws/gc/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:648 +0x3d4 fp=0xc002cb9f28 sp=0xc002cb9dd8 pc=0x45f7914
runtime.goexit()
        /opt/ooce/go-1.16/src/runtime/asm_amd64.s:1371 +0x1 fp=0xc002cb9f30 sp=0xc002cb9f28 pc=0x10e35a1
created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync
        /ws/gc/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:576 +0x188

[davepacheco]

See also #1144, but this one's actually inside CockroachDB.

[davepacheco]

I'm attaching issue-1223.tgz.zip, which includes the whole test run temp directory, which includes our test suite log file as well as the CockroachDB data directory (which includes the above stderr file).

[knz]

This is not in cockroachdb directly. This is in the go runtime system, when assigning to a (primitive) map type.

I would recommend you attempt building with go 1.17 or 1.18. There were improvements in that area.

[davepacheco]

@knz I'm working on getting a build of Cockroach using Go 1.18. As I think you've seen, we've hit a half dozen or so different memory-related issues (including SIGSEGV, stack corruption, and panics in the Go memory allocator). We don't know if they're related. For most of them (including this one), we've only seen them once despite many iterations. So if we don't see this with Go 1.18, that won't really tell us whether the problem was somehow fixed.

[bnaecker]

I've seem to hit the same issue. Working on #1474, I ran a full cargo t. One of the tests failed, and the standard error output appears to be almost exactly the same. There is one difference near the top of the statck, which is the existence of the runtime.hashGrow function call, the 3rd frame in the stack below. There are other differences in the middle there, calls to runtime.panicmem(). It does look the same at the bottom though.

I220720 21:43:41.793625 1 util/log/file_sync_buffer.go:238 ⋮ [config]   file created at: 2022/07/20 21:43:41
I220720 21:43:41.793669 1 util/log/file_sync_buffer.go:238 ⋮ [config]   running on machine: ‹feldspar›
I220720 21:43:41.793676 1 util/log/file_sync_buffer.go:238 ⋮ [config]   binary: CockroachDB OSS v21.2.9 (x86_64-pc-solaris2.11, built 2022/04/28 04:02:42, go1.16.10)
I220720 21:43:41.793684 1 util/log/file_sync_buffer.go:238 ⋮ [config]   arguments: [‹cockroach› ‹start-single-node› ‹--insecure› ‹--http-addr=:0› ‹--store=path=/tmp/.tmpKAnZD9/data,ballast-size=0› ‹--listen-addr› ‹127.0.0.1:0› ‹--listening-url-file› ‹/tmp/.tmpKAnZD9/listen-url›]
I220720 21:43:41.793697 1 util/log/file_sync_buffer.go:238 ⋮ [config]   log format (utf8=✓): crdb-v2
I220720 21:43:41.793700 1 util/log/file_sync_buffer.go:238 ⋮ [config]   line format: [IWEF]yymmdd hh:mm:ss.uuuuuu goid [chan@]file:line redactionmark \[tags\] [counter] msg
I220720 21:43:41.793379 1 util/log/flags.go:201  [-] 1  stderr capture started
unexpected fault address 0x108154f
fatal error: fault
[signal SIGSEGV: segmentation violation code=0x1 addr=0x108154f pc=0x108154f]

goroutine 175 [running]:
runtime.throw(0x59bf4fa, 0x5)
    /opt/ooce/go-1.16/src/runtime/panic.go:1117 +0x72 fp=0xc00332b8d0 sp=0xc00332b8a0 pc=0x10a9512
runtime.sigpanic()
    /opt/ooce/go-1.16/src/runtime/signal_unix.go:741 +0x276 fp=0xc00332b908 sp=0xc00332b8d0 pc=0x10c0e96
runtime.hashGrow(0x5292480, 0xc003181080)
    /opt/ooce/go-1.16/src/runtime/map.go:1050 +0xef fp=0xc00332b958 sp=0xc00332b908 pc=0x108154f
runtime.mapassign_fast32(0x5292480, 0xc003181080, 0x1, 0x20)
    /opt/ooce/go-1.16/src/runtime/map_fast32.go:156 +0x113 fp=0xc00332b998 sp=0xc00332b958 pc=0x10822d3
github.com/cockroachdb/cockroach/pkg/sql/catalog.(*collectorState).addDirectReferences(0xc002c9bb00, 0xfffffc7fee8f3e18, 0xc000186630, 0x0, 0x0)
    /ws/gc/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/sql/catalog/validate.go:442 +0x5a fp=0xc00332b9e0 sp=0xc00332b998 pc=0x26d231a
github.com/cockroachdb/cockroach/pkg/sql/catalog.collectDescriptorsForValidation(0x6ff5af8, 0xc001793dd0, 0x6f97070, 0xc002fda530, 0xc002fda520, 0x1, 0x1, 0xc00050a480, 0x0, 0xc00332bb10)
    /ws/gc/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/sql/catalog/validate.go:500 +0x159 fp=0xc00332ba88 sp=0xc00332b9e0 pc=0x26d2b19
github.com/cockroachdb/cockroach/pkg/sql/catalog.Validate(0x6ff5af8, 0xc001793dd0, 0x6f97070, 0xc002fda530, 0x2, 0xc00000001f, 0xc002fda520, 0x1, 0x1, 0x4, ...)
    /ws/gc/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/sql/catalog/validate.go:55 +0x14f fp=0xc00332bb20 sp=0xc00332ba88 pc=0x26cff2f
github.com/cockroachdb/cockroach/pkg/sql/catalog/descs.(*Collection).ValidateUncommittedDescriptors(0xc00175ae98, 0x6ff5af8, 0xc001793dd0, 0xc0031614a0, 0x0, 0x62d8770f)
    /ws/gc/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/sql/catalog/descs/validate.go:43 +0x15a fp=0xc00332bbb0 sp=0xc00332bb20 pc=0x2d2325a
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc00175ac00, 0x6ff5af8, 0xc001793dd0, 0x7002558, 0xc002f6d770, 0xeda6a6e0f, 0x0)
    /ws/gc/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:883 +0xa5 fp=0xc00332bbf8 sp=0xc00332bbb0 pc=0x4187985
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal-fm(0x6ff5af8, 0xc001793dd0, 0x7002558, 0xc002f6d770, 0x204d597, 0xc0031614d8)
    /ws/gc/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:870 +0x5b fp=0xc00332bc40 sp=0xc00332bbf8 pc=0x440369b
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransaction(0xc00175ac00, 0x6ff5af8, 0xc001793dd0, 0x7002558, 0xc002f6d770, 0xc00332bd30, 0xc00269d460, 0xc00332bd48, 0x0, 0xc00332bd20)
    /ws/gc/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:820 +0x9c fp=0xc00332bca8 sp=0xc00332bc40 pc=0x418715c
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).handleAutoCommit(0xc00175ac00, 0x6ff5af8, 0xc001793dd0, 0x7002558, 0xc002f6d770, 0x1800000001, 0xc00332be08, 0x1085854, 0xc00332cb50)
    /ws/gc/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1837 +0xdc fp=0xc00332bd50 sp=0xc00332bca8 pc=0x418e83c
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState.func7(0xc00175ac00, 0xc00332d2c8, 0xc00332d2b8, 0xc00332d248, 0xc00332ce20, 0xc00332d2a8, 0x6f974a8, 0x921b59b)
    /ws/gc/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:502 +0xc9 fp=0xc00332bdc8 sp=0xc00332bd50 pc=0x43748e9
runtime.call64(0x0, 0x60859b8, 0xc00332cc68, 0x4000000040)
    /opt/ooce/go-1.16/src/runtime/asm_amd64.s:552 +0x3e fp=0xc00332be18 sp=0xc00332bdc8 pc=0x10e1bfe
panic(0x52cd740, 0x91ab4f0)
    /opt/ooce/go-1.16/src/runtime/panic.go:971 +0x499 fp=0xc00332bee0 sp=0xc00332be18 pc=0x10a9039
runtime.panicmem()
    /opt/ooce/go-1.16/src/runtime/panic.go:212 +0x5b fp=0xc00332bf00 sp=0xc00332bee0 pc=0x10a715b
runtime.sigpanic()
    /opt/ooce/go-1.16/src/runtime/signal_unix.go:734 +0x179 fp=0xc00332bf38 sp=0xc00332bf00 pc=0x10c0d99
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).planAndRunSubquery(0xc001b4b300, 0x6ff5af8, 0xc001793dd0, 0x2, 0x6f48ec0, 0xc002f6c690, 0x3, 0x1, 0x6fdae70, 0xc0035edf80, ...)
    /ws/gc/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1209 +0xe22 fp=0xc00332c378 sp=0xc00332bf38 pc=0x4203982
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRunSubqueries(0xc001b4b300, 0x6ff5af8, 0xc001793dd0, 0xc00175b1d8, 0xc00332c550, 0xc0006af080, 0x9, 0x9, 0xc0019ad500, 0xc00332c538, ...)
    /ws/gc/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1085 +0x19e fp=0xc00332c498 sp=0xc00332c378 pc=0x4202a5e
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc00175ac00, 0x6ff5af8, 0xc001793dd0, 0xc00175b1d8, 0x3, 0xfffffc7fef07de28, 0xc0018033a0, 0xc00332c600, 0xc0034ddd98, 0x0, ...)
    /ws/gc/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1372 +0x7bd fp=0xc00332c5c0 sp=0xc00332c498 pc=0x418b27d
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc00175ac00, 0x6ff5af8, 0xc001793dd0, 0xc00175b1d8, 0xfffffc7fef07de28, 0xc0018033a0, 0x0, 0x0)
    /ws/gc/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1072 +0x90b fp=0xc00332c928 sp=0xc00332c5c0 pc=0x418898b
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc00175ac00, 0x6ff5af8, 0xc001793dd0, 0x7002558, 0xc002f6d770, 0xc001794717, 0x5d4, 0x12, 0xa, 0xc001b14100, ...)
    /ws/gc/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:709 +0x11a6 fp=0xc00332d240 sp=0xc00332c928 pc=0x41845c6
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc00175ac00, 0x6fd9ee8, 0xc0016dc500, 0x7002558, 0xc002f6d770, 0xc001794717, 0x5d4, 0x12, 0xa, 0xc001b14100, ...)
    /ws/gc/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:129 +0xb0a fp=0xc00332d4d0 sp=0xc00332d240 pc=0x4182b8a
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc00175ac00, 0x6fd9ee8, 0xc0016dc500, 0xc001b14100, 0xc002a75c20, 0x12, 0x12, 0xc002bbe660, 0x9, 0x9, ...)
    /ws/gc/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:216 +0x14e fp=0xc00332d608 sp=0xc00332d4d0 pc=0x418316e
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2(0xc001794e46, 0x0, 0x0, 0x36cc0823, 0xeda6a6e0f, 0x0, 0xc00175ac00, 0xc00332d9e8, 0xc00332d9d8, 0xc00332d8d8, ...)
    /ws/gc/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1773 +0x4fb fp=0xc00332d7e8 sp=0xc00332d608 pc=0x437313b
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc00175ac00, 0x6fd9ee8, 0xc0016dc500, 0x0, 0x0)
    /ws/gc/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1775 +0x4df fp=0xc00332dcb0 sp=0xc00332d7e8 pc=0x417591f
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc00175ac00, 0x6fd9ee8, 0xc000924740, 0xc001727ae0, 0x5400, 0x15000, 0xc00217a0a0, 0xc001116e40, 0x0, 0x0)
    /ws/gc/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1629 +0x22f fp=0xc00332dd50 sp=0xc00332dcb0 pc=0x41751cf
github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc0005f7500, 0x6fd9ee8, 0xc000924740, 0xc00175ac00, 0x5400, 0x15000, 0xc00217a0a0, 0xc001116e40, 0x0, 0x0)
    /ws/gc/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:668 +0xce fp=0xc00332ddd8 sp=0xc00332dd50 pc=0x4170d6e
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1(0xc001bc8654, 0xc00138e690, 0x6fd9ee8, 0xc000924740, 0xc001116e40, 0xc0005f7500, 0xc001802d00, 0x70492c0, 0xc001848510, 0xc001901980, ...)
    /ws/gc/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:648 +0x3d4 fp=0xc00332df28 sp=0xc00332ddd8 pc=0x45f7914
runtime.goexit()
    /opt/ooce/go-1.16/src/runtime/asm_amd64.s:1371 +0x1 fp=0xc00332df30 sp=0xc00332df28 pc=0x10e35a1
created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync
    /ws/gc/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:576 +0x188

I'm attaching the whole directory here as well.
issue-1223-round-2.tgz.zip

[knz]

This last stack strace is also with go 1.16. let's look at what 1.17 or 1.18 has for us. (fyi there's a PR open on the CRDB repo containing the tweaks to make it buildable with the 1.18 compiler. You can perhaps use that.) -- Verstuurd vanaf mijn Android apparaat met K-9 Mail. Excuseer mijn beknoptheid.

[davepacheco]

The next step here seems to be updating Omicron to the latest CockroachDB, which uses Go 1.17. This is being done under oxidecomputer/garbage-compactor#5.

[davepacheco]

Okay, I've finally got an up-to-date CockroachDB build with an up-to-date Go. It's v22.1.9 (released just this week) on Go v1.17.13. Go v1.17.13 is the latest Go release of 1.17. It looks like Go 1.17 is what Cockroach v22.1.9 expects.

I ran this loop:

$ while cockroach version; do :; done
Build Tag:        v22.1.9-dirty
Build Time:       2022/10/21 16:56:49
Distribution:     OSS
Platform:         illumos amd64 (x86_64-pc-solaris2.11)
Go Version:       go1.17.13
C Compiler:       gcc 10.3.0
Build Commit ID:  e438c2f89282e607e0e6ca1d38b2e0a622f94493
Build Type:       release
...
Build Tag:        v22.1.9-dirty
Build Time:       2022/10/21 16:56:49
Distribution:     OSS
Platform:         illumos amd64 (x86_64-pc-solaris2.11)
Go Version:       go1.17.13
C Compiler:       gcc 10.3.0
Build Commit ID:  e438c2f89282e607e0e6ca1d38b2e0a622f94493
Build Type:       release
runtime: s.allocCount= 35 s.nelems= 73
fatal error: s.allocCount != s.nelems && freeIndex == s.nelems

goroutine 1 [running, locked to thread]:
runtime.throw({0x612f6ce, 0x31})
	/opt/ooce/go-1.17/src/runtime/panic.go:1198 +0x74 fp=0xc000f7ecf8 sp=0xc000f7ecc8 pc=0x127d6b4
runtime.(*mcache).nextFree(0xfffffc7fef044f18, 0x12)
	/opt/ooce/go-1.17/src/runtime/malloc.go:884 +0x228 fp=0xc000f7ed38 sp=0xc000f7ecf8 pc=0x124e6e8
runtime.mallocgc(0x70, 0x5ee6920, 0x1)
	/opt/ooce/go-1.17/src/runtime/malloc.go:1077 +0x530 fp=0xc000f7edb8 sp=0xc000f7ed38 pc=0x124ec30
runtime.newobject(0x5ee6920)
	/opt/ooce/go-1.17/src/runtime/malloc.go:1234 +0x38 fp=0xc000f7ede8 sp=0xc000f7edb8 pc=0x124f1d8
regexp/syntax.simplify1(...)
	/opt/ooce/go-1.17/src/regexp/syntax/simplify.go:148
regexp/syntax.(*Regexp).Simplify(0xc0006ceb60)
	/opt/ooce/go-1.17/src/regexp/syntax/simplify.go:100 +0xbbb fp=0xc000f7eef8 sp=0xc000f7ede8 pc=0x14a5dfb
regexp/syntax.(*Regexp).Simplify(0xc0006cea10)
	/opt/ooce/go-1.17/src/regexp/syntax/simplify.go:23 +0x56b fp=0xc000f7f008 sp=0xc000f7eef8 pc=0x14a57ab
regexp/syntax.(*Regexp).Simplify(0xc0006cfb90)
	/opt/ooce/go-1.17/src/regexp/syntax/simplify.go:23 +0x56b fp=0xc000f7f118 sp=0xc000f7f008 pc=0x14a57ab
regexp.compile({0x61488c0, 0x35}, 0xd4, 0x0)
	/opt/ooce/go-1.17/src/regexp/regexp.go:177 +0xd4 fp=0xc000f7f1b8 sp=0xc000f7f118 pc=0x14af194
regexp.Compile(...)
	/opt/ooce/go-1.17/src/regexp/regexp.go:133
regexp.MustCompile({0x61488c0, 0x35})
	/opt/ooce/go-1.17/src/regexp/regexp.go:309 +0x4b fp=0xc000f7f240 sp=0xc000f7f1b8 pc=0x14afe8b
go.opencensus.io/resource.init()
	/home/dap/garbage-compactor/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/vendor/go.opencensus.io/resource/resource.go:60 +0x36 fp=0xc000f7f268 sp=0xc000f7f240 pc=0x51e6956
runtime.doInit(0xaf5abe0)
	/opt/ooce/go-1.17/src/runtime/proc.go:6498 +0x129 fp=0xc000f7f3b8 sp=0xc000f7f268 pc=0x128e949
runtime.doInit(0xaf51960)
	/opt/ooce/go-1.17/src/runtime/proc.go:6475 +0x7e fp=0xc000f7f508 sp=0xc000f7f3b8 pc=0x128e89e
runtime.doInit(0xaf58f00)
	/opt/ooce/go-1.17/src/runtime/proc.go:6475 +0x7e fp=0xc000f7f658 sp=0xc000f7f508 pc=0x128e89e
runtime.doInit(0xaf67160)
	/opt/ooce/go-1.17/src/runtime/proc.go:6475 +0x7e fp=0xc000f7f7a8 sp=0xc000f7f658 pc=0x128e89e
runtime.doInit(0xaf6b9a0)
	/opt/ooce/go-1.17/src/runtime/proc.go:6475 +0x7e fp=0xc000f7f8f8 sp=0xc000f7f7a8 pc=0x128e89e
runtime.doInit(0xaf6ea20)
	/opt/ooce/go-1.17/src/runtime/proc.go:6475 +0x7e fp=0xc000f7fa48 sp=0xc000f7f8f8 pc=0x128e89e
runtime.doInit(0xaf792e0)
	/opt/ooce/go-1.17/src/runtime/proc.go:6475 +0x7e fp=0xc000f7fb98 sp=0xc000f7fa48 pc=0x128e89e
runtime.doInit(0xaf58060)
	/opt/ooce/go-1.17/src/runtime/proc.go:6475 +0x7e fp=0xc000f7fce8 sp=0xc000f7fb98 pc=0x128e89e
runtime.doInit(0xafb0ae0)
	/opt/ooce/go-1.17/src/runtime/proc.go:6475 +0x7e fp=0xc000f7fe38 sp=0xc000f7fce8 pc=0x128e89e
runtime.doInit(0xaf503a0)
	/opt/ooce/go-1.17/src/runtime/proc.go:6475 +0x7e fp=0xc000f7ff88 sp=0xc000f7fe38 pc=0x128e89e
runtime.main()
	/opt/ooce/go-1.17/src/runtime/proc.go:238 +0x205 fp=0xc000f7ffe0 sp=0xc000f7ff88 pc=0x12801c5
runtime.goexit()
	/opt/ooce/go-1.17/src/runtime/asm_amd64.s:1581 +0x1 fp=0xc000f7ffe8 sp=0xc000f7ffe0 pc=0x12b6d41

goroutine 18 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.flushDaemon()
	/home/dap/garbage-compactor/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/util/log/log_flush.go:75 +0x76
created by github.com/cockroachdb/cockroach/pkg/util/log.init.5
	/home/dap/garbage-compactor/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/util/log/log_flush.go:41 +0x35

goroutine 19 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.signalFlusher()
	/home/dap/garbage-compactor/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/util/log/log_flush.go:98 +0x145
created by github.com/cockroachdb/cockroach/pkg/util/log.init.5
	/home/dap/garbage-compactor/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/util/log/log_flush.go:42 +0x4d

goroutine 22 [syscall]:
os/signal.signal_recv()
	/opt/ooce/go-1.17/src/runtime/sigqueue.go:169 +0xab
os/signal.loop()
	/opt/ooce/go-1.17/src/os/signal/signal_unix.go:24 +0x25
created by os/signal.Notify.func1.1
	/opt/ooce/go-1.17/src/os/signal/signal.go:151 +0x3a

goroutine 8 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/goschedstats.init.0.func1()
	/home/dap/garbage-compactor/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/util/goschedstats/runnable.go:165 +0x1de
created by github.com/cockroachdb/cockroach/pkg/util/goschedstats.init.0
	/home/dap/garbage-compactor/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/util/goschedstats/runnable.go:157 +0x35

This happened on iteration 7 -- basically right away. Trying to repro, I ran it again, and it's been running several minutes without incident.

Separately, I ran the Omicron test suite twice. Each run of the test suite runs dozens of tests that each run cockroach at least once. Most tests passed, but I saw three failures across the two runs:

---- db::datastore::test::test_region_allocation stdout ----
log file: "/dangerzone/omicron_tmp/omicron_nexus-f7a77ee7f5dba530-test_region_allocation.23078.37.log"
note: configured to log to "/dangerzone/omicron_tmp/omicron_nexus-f7a77ee7f5dba530-test_region_allocation.23078.37.log"
thread 'db::datastore::test::test_region_allocation' panicked at 'called `Result::unwrap()` on an `Err` value: BadVersion { expected: "v22.1.9-dirty", found: Err(error Some(2) when checking CockroachDB version) }', /home/dap/omicron-cockroachdb/test-utils/src/dev/mod.rs:142:42
note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace
...
---- db::pagination::test::test_paginated_multicolumn_descending stdout ----
log file: "/dangerzone/omicron_tmp/omicron_nexus-f7a77ee7f5dba530-test_paginated_multicolumn_descending.23078.50.log"
note: configured to log to "/dangerzone/omicron_tmp/omicron_nexus-f7a77ee7f5dba530-test_paginated_multicolumn_descending.23078.50.log"
thread 'db::pagination::test::test_paginated_multicolumn_descending' panicked at 'called `Result::unwrap()` on an `Err` value: BadVersion { expected: "v22.1.9-dirty", found: Err(error Some(2) when checking CockroachDB version) }', /home/dap/omicron-cockroachdb/test-utils/src/dev/mod.rs:142:42
...
---- db::collection_detach::test::test_detach_missing_resource_fails stdout ----
log file: "/dangerzone/omicron_tmp/omicron_nexus-f7a77ee7f5dba530-test_detach_missing_resource_fails.24434.17.log"
note: configured to log to "/dangerzone/omicron_tmp/omicron_nexus-f7a77ee7f5dba530-test_detach_missing_resource_fails.24434.17.log"
thread 'db::collection_detach::test::test_detach_missing_resource_fails' panicked at 'called `Result::unwrap()` on an `Err` value: Exited { exit_code: 2 }', /home/dap/omicron-cockroachdb/test-utils/src/dev/mod.rs:142:42
note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace

For some reason, I only have stderr from the last of those failures. Following the log mentioned above:

$ bunyan /dangerzone/omicron_tmp/omicron_nexus-f7a77ee7f5dba530-test_detach_missing_resource_fails.24434.17.log
[2022-10-21T18:04:55.403226924Z]  INFO: test_detach_missing_resource_fails/24434 on ivanova: cockroach temporary directory: /dangerzone/omicron_tmp/.tmp8C2WQ7
[2022-10-21T18:04:55.403696163Z]  INFO: test_detach_missing_resource_fails/24434 on ivanova: cockroach: copying from seed directory (/home/dap/omicron-cockroachdb/target/debug/build/nexus-test-utils-110448a09861e607/out/crdb-base) to storage directory (/dangerzone/omicron_tmp/.tmp8C2WQ7/data)
[2022-10-21T18:04:55.434743557Z]  INFO: test_detach_missing_resource_fails/24434 on ivanova: cockroach command line: cockroach start-single-node --insecure --http-addr=:0 --store=path=/dangerzone/omicron_tmp/.tmp8C2WQ7/data,ballast-size=0 --listen-addr 127.0.0.1:0 --listening-url-file /dangerzone/omicron_tmp/.tmp8C2WQ7/listen-url

$ cat /dangerzone/omicron_tmp/.tmp8C2WQ7/cockroachdb_stderr
unexpected fault address 0x1252ba6
fatal error: fault
fatal error: releasep: invalid arg
[signal SIGSEGV: segmentation violation code=0x1 addr=0x1252ba6 pc=0x1252ba6]

goroutine 1 [running, locked to thread]:
runtime.throw({0x602c9f0, 0x5})
	/opt/ooce/go-1.17/src/runtime/panic.go:1198 +0x74 fp=0xc00117f800 sp=0xc00117f7d0 pc=0x127d6b4
runtime.sigpanic()
	/opt/ooce/go-1.17/src/runtime/signal_unix.go:742 +0x3b8 fp=0xc00117f860 sp=0xc00117f800 pc=0x1295c18
runtime.mapassign_fast32(0x5973b40, 0xc0003aa180, 0x0)
	/opt/ooce/go-1.17/src/runtime/map_fast32.go:126 +0x206 fp=0xc00117f8a0 sp=0xc00117f860 pc=0x1252ba6
github.com/cockroachdb/cockroach/pkg/ccl/sqlproxyccl/tenant.init()
	/home/dap/garbage-compactor/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/ccl/sqlproxyccl/tenant/directory.pb.go:63 +0x4c fp=0xc00117f8f8 sp=0xc00117f8a0 pc=0x53aeacc
runtime.doInit(0xaf6ebe0)
	/opt/ooce/go-1.17/src/runtime/proc.go:6498 +0x129 fp=0xc00117fa48 sp=0xc00117f8f8 pc=0x128e949
runtime.doInit(0xaf66440)
	/opt/ooce/go-1.17/src/runtime/proc.go:6475 +0x7e fp=0xc00117fb98 sp=0xc00117fa48 pc=0x128e89e
runtime.doInit(0xaf7e620)
	/opt/ooce/go-1.17/src/runtime/proc.go:6475 +0x7e fp=0xc00117fce8 sp=0xc00117fb98 pc=0x128e89e
runtime.doInit(0xafb0ae0)
	/opt/ooce/go-1.17/src/runtime/proc.go:6475 +0x7e fp=0xc00117fe38 sp=0xc00117fce8 pc=0x128e89e
runtime.doInit(0xaf503a0)
	/opt/ooce/go-1.17/src/runtime/proc.go:6475 +0x7e fp=0xc00117ff88 sp=0xc00117fe38 pc=0x128e89e
runtime.main()
	/opt/ooce/go-1.17/src/runtime/proc.go:238 +0x205 fp=0xc00117ffe0 sp=0xc00117ff88 pc=0x12801c5
runtime.goexit()
	/opt/ooce/go-1.17/src/runtime/asm_amd64.s:1581 +0x1 fp=0xc00117ffe8 sp=0xc00117ffe0 pc=0x12b6d41

goroutine 18 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.flushDaemon()
	/home/dap/garbage-compactor/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/util/log/log_flush.go:75 +0x76
created by github.com/cockroachdb/cockroach/pkg/util/log.init.5
	/home/dap/garbage-compactor/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/util/log/log_flush.go:41 +0x35

goroutine 19 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.signalFlusher()
	/home/dap/garbage-compactor/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/util/log/log_flush.go:98 +0x145
created by github.com/cockroachdb/cockroach/pkg/util/log.init.5
	/home/dap/garbage-compactor/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/util/log/log_flush.go:42 +0x4d

goroutine 22 [syscall]:
os/signal.signal_recv()
	/opt/ooce/go-1.17/src/runtime/sigqueue.go:169 +0xab
os/signal.loop()
	/opt/ooce/go-1.17/src/os/signal/signal_unix.go:24 +0x25
created by os/signal.Notify.func1.1
	/opt/ooce/go-1.17/src/os/signal/signal.go:151 +0x3a

goroutine 8 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/goschedstats.init.0.func1()
	/home/dap/garbage-compactor/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/util/goschedstats/runnable.go:165 +0x1de
created by github.com/cockroachdb/cockroach/pkg/util/goschedstats.init.0
	/home/dap/garbage-compactor/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/util/goschedstats/runnable.go:157 +0x35

goroutine 82 [select]:
go.opencensus.io/stats/view.(*worker).start(0xc0005a2c00)
	/home/dap/garbage-compactor/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/vendor/go.opencensus.io/stats/view/worker.go:276 +0xec
created by go.opencensus.io/stats/view.init.0
	/home/dap/garbage-compactor/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/vendor/go.opencensus.io/stats/view/worker.go:34 +0x76

runtime stack:
runtime.throw({0x60753a0, 0x15})
	/opt/ooce/go-1.17/src/runtime/panic.go:1198 +0x74
runtime.releasep()
	/opt/ooce/go-1.17/src/runtime/proc.go:5178 +0x190
runtime.gcstopm()
	/opt/ooce/go-1.17/src/runtime/proc.go:2650 +0x69
runtime.findrunnable()
	/opt/ooce/go-1.17/src/runtime/proc.go:2715 +0x59
runtime.schedule()
	/opt/ooce/go-1.17/src/runtime/proc.go:3367 +0x297
runtime.park_m(0xc0006029c0)
	/opt/ooce/go-1.17/src/runtime/proc.go:3516 +0x18e
runtime.mcall()
	/opt/ooce/go-1.17/src/runtime/asm_amd64.s:337 +0x63

As I wrote this up, I started a second bash loop to repro with just cockroach version, and that ran for a few minutes before bailing out too:

...
Build Tag:        v22.1.9-dirty
Build Time:       2022/10/21 16:56:49
Distribution:     OSS
Platform:         illumos amd64 (x86_64-pc-solaris2.11)
Go Version:       go1.17.13
C Compiler:       gcc 10.3.0
Build Commit ID:  e438c2f89282e607e0e6ca1d38b2e0a622f94493
Build Type:       release
runtime: s.allocCount= 133 s.nelems= 170
fatal error: s.allocCount != s.nelems && freeIndex == s.nelems

goroutine 1 [running, locked to thread]:
runtime.throw({0x612f6ce, 0x31})
        /opt/ooce/go-1.17/src/runtime/panic.go:1198 +0x74 fp=0xc000c03e28 sp=0xc000c03df8 pc=0x127d6b4
runtime.(*mcache).nextFree(0xfffffc7fef0c5688, 0xa)
        /opt/ooce/go-1.17/src/runtime/malloc.go:884 +0x228 fp=0xc000c03e68 sp=0xc000c03e28 pc=0x124e6e8
runtime.mallocgc(0x30, 0x5ec4500, 0x1)
        /opt/ooce/go-1.17/src/runtime/malloc.go:1077 +0x530 fp=0xc000c03ee8 sp=0xc000c03e68 pc=0x124ec30
runtime.newobject(0x5ec4500)
        /opt/ooce/go-1.17/src/runtime/malloc.go:1234 +0x38 fp=0xc000c03f18 sp=0xc000c03ee8 pc=0x124f1d8
runtime.makemap_small()
        /opt/ooce/go-1.17/src/runtime/map.go:293 +0x2d fp=0xc000c03f38 sp=0xc000c03f18 pc=0x124fdcd
github.com/aws/aws-sdk-go/aws/endpoints.init()
        /home/dap/garbage-compactor/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/vendor/github.com/aws/aws-sdk-go/aws/endpoints/defaults.go:6369 +0x8a21c fp=0xc000c177a8 sp=0xc000c03f38 pc=0x4e6c6bc
runtime.doInit(0xaf5a160)
        /opt/ooce/go-1.17/src/runtime/proc.go:6498 +0x129 fp=0xc000c178f8 sp=0xc000c177a8 pc=0x128e949
runtime.doInit(0xaf63380)
        /opt/ooce/go-1.17/src/runtime/proc.go:6475 +0x7e fp=0xc000c17a48 sp=0xc000c178f8 pc=0x128e89e
runtime.doInit(0xaf788c0)
        /opt/ooce/go-1.17/src/runtime/proc.go:6475 +0x7e fp=0xc000c17b98 sp=0xc000c17a48 pc=0x128e89e
runtime.doInit(0xaf58060)
        /opt/ooce/go-1.17/src/runtime/proc.go:6475 +0x7e fp=0xc000c17ce8 sp=0xc000c17b98 pc=0x128e89e
runtime.doInit(0xafb0ae0)
        /opt/ooce/go-1.17/src/runtime/proc.go:6475 +0x7e fp=0xc000c17e38 sp=0xc000c17ce8 pc=0x128e89e
runtime.doInit(0xaf503a0)
        /opt/ooce/go-1.17/src/runtime/proc.go:6475 +0x7e fp=0xc000c17f88 sp=0xc000c17e38 pc=0x128e89e
runtime.main()
        /opt/ooce/go-1.17/src/runtime/proc.go:238 +0x205 fp=0xc000c17fe0 sp=0xc000c17f88 pc=0x12801c5
runtime.goexit()
        /opt/ooce/go-1.17/src/runtime/asm_amd64.s:1581 +0x1 fp=0xc000c17fe8 sp=0xc000c17fe0 pc=0x12b6d41

goroutine 34 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.flushDaemon()
        /home/dap/garbage-compactor/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/util/log/log_flush.go:75 +0x76
created by github.com/cockroachdb/cockroach/pkg/util/log.init.5
        /home/dap/garbage-compactor/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/util/log/log_flush.go:41 +0x35

goroutine 35 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.signalFlusher()
        /home/dap/garbage-compactor/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/util/log/log_flush.go:98 +0x145
created by github.com/cockroachdb/cockroach/pkg/util/log.init.5
        /home/dap/garbage-compactor/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/util/log/log_flush.go:42 +0x4d

goroutine 38 [syscall]:
os/signal.signal_recv()
        /opt/ooce/go-1.17/src/runtime/sigqueue.go:169 +0xab
os/signal.loop()
        /opt/ooce/go-1.17/src/os/signal/signal_unix.go:24 +0x25
created by os/signal.Notify.func1.1
        /opt/ooce/go-1.17/src/os/signal/signal.go:151 +0x3a

goroutine 39 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/goschedstats.init.0.func1()
        /home/dap/garbage-compactor/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/util/goschedstats/runnable.go:165 +0x1de
created by github.com/cockroachdb/cockroach/pkg/util/goschedstats.init.0
        /home/dap/garbage-compactor/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/util/goschedstats/runnable.go:157 +0x35

@knz these feel pretty similar to the problems we were seeing before, don't they? They don't seem fixed in Go 1.17.

[knz]

Thanks. Can you compare the results of you build CRDB with stdmalloc as a go build tag? That will remove jemalloc from the picture.

[davepacheco]

@knz This should be a stdmalloc build already. I don't know how to confirm that, but I don't see jemalloc symbols in it:

$ nm ~/tools/cockroach-v22.1.9/bin/cockroach | grep -i jemalloc
[157157]	|           188214760|                   8|OBJT |LOCL |0    |35     |github.com/cockroachdb/cockroach/pkg/server/heapprofiler.jemallocHeapDump
[117026]	|            77790624|                 412|FUNC |LOCL |0    |15     |github.com/cockroachdb/cockroach/pkg/server/heapprofiler.takeJemallocProfile
[157166]	|           139753440|                  32|OBJT |LOCL |0    |18     |github.com/cockroachdb/cockroach/pkg/server/heapprofiler.takeJemallocProfile.stkobj

By comparison, here's one that was not built this way:

 $ nm ~/tools/cockroachdb-v21.1.10/bin/cockroach | grep -i jemalloc
[169366]	|            73621120|                  81|FUNC |GLOB |0    |15     |_cgo_a09ca8310212_C2func_jemalloc_get_stats
[169342]	|            73621232|                  59|FUNC |GLOB |0    |15     |_cgo_a09ca8310212_Cfunc_jemalloc_get_stats
[109464]	|            73453120|                 357|FUNC |LOCL |0    |15     |github.com/cockroachdb/cockroach/pkg/cli.writeJemallocProfile
[109680]	|            73555040|                 116|FUNC |LOCL |0    |15     |github.com/cockroachdb/cockroach/pkg/cli.writeJemallocProfile.func1
[109679]	|            73554912|                 103|FUNC |LOCL |0    |15     |github.com/cockroachdb/cockroach/pkg/cli.writeJemallocProfile.func1.1
[110387]	|           105208256|                  24|OBJT |LOCL |0    |18     |github.com/cockroachdb/cockroach/pkg/cli.writeJemallocProfile.stkobj
[125075]	|           148005312|                   8|OBJT |LOCL |0    |36     |github.com/cockroachdb/cockroach/pkg/server/heapprofiler.jemallocHeapDump
[93777]	|            65697728|                 367|FUNC |LOCL |0    |15     |github.com/cockroachdb/cockroach/pkg/server/heapprofiler.takeJemallocProfile
[125080]	|           105255456|                  24|OBJT |LOCL |0    |18     |github.com/cockroachdb/cockroach/pkg/server/heapprofiler.takeJemallocProfile.stkobj
[92781]	|            65212704|                 182|FUNC |LOCL |0    |15     |github.com/cockroachdb/cockroach/pkg/server/status._C2func_jemalloc_get_stats
[120183]	|           105624768|                  40|OBJT |LOCL |0    |18     |github.com/cockroachdb/cockroach/pkg/server/status._C2func_jemalloc_get_stats.stkobj
[120162]	|           137540264|                   8|OBJT |LOCL |0    |28     |github.com/cockroachdb/cockroach/pkg/server/status._cgo_a09ca8310212_C2func_jemalloc_get_stats
[92784]	|            65213248|                1757|FUNC |LOCL |0    |15     |github.com/cockroachdb/cockroach/pkg/server/status.getJemallocStats
[120185]	|           105945856|                  56|OBJT |LOCL |0    |18     |github.com/cockroachdb/cockroach/pkg/server/status.getJemallocStats.stkobj
[537]	|            74373536|                 722|FUNC |LOCL |2    |15     |je_jemalloc_postfork_child
[166019]	|            74438337|                   5|FUNC |LOCL |0    |15     |je_jemalloc_postfork_child.cold
[383]	|            74374272|                 722|FUNC |LOCL |2    |15     |je_jemalloc_postfork_parent
[166020]	|            74438342|                   5|FUNC |LOCL |0    |15     |je_jemalloc_postfork_parent.cold
[596]	|            74375008|                1570|FUNC |LOCL |2    |15     |je_jemalloc_prefork
[166021]	|            74438347|                   5|FUNC |LOCL |0    |15     |je_jemalloc_prefork.cold
[166043]	|            74438448|                 188|FUNC |LOCL |0    |15     |jemalloc_constructor
[166044]	|            74438437|                   5|FUNC |LOCL |0    |15     |jemalloc_constructor.cold
[170706]	|            73620816|                 297|FUNC |GLOB |0    |15     |jemalloc_get_stats
[165999]	|                   0|                   0|FILE |LOCL |0    |ABS    |jemalloc.c
[165218]	|                   0|                   0|FILE |LOCL |0    |ABS    |runtime_jemalloc.cgo2.c
[165200]	|                   0|                   0|FILE |LOCL |0    |ABS    |start_jemalloc.cgo2.c

(edit sorry jnz for the stray mention)

[davepacheco]

Here's a different failure mode, also from the cockroach version loop:

0xc000d8bec0 free  unmarked
0xc000d8bf20 free  unmarked
0xc000d8bf80 free  unmarked
fatal error: found pointer to free object

runtime stack:
runtime.throw({0x6097e1e, 0x1c})
        /opt/ooce/go-1.17/src/runtime/panic.go:1198 +0x74
runtime.(*mspan).reportZombies(0xfffffc7fee7d94e8)
        /opt/ooce/go-1.17/src/runtime/mgcsweep.go:691 +0x345
runtime.(*sweepLocked).sweep(0xfffffc7fe81ffe20, 0x0)
        /opt/ooce/go-1.17/src/runtime/mgcsweep.go:524 +0x9dd
runtime.(*mcentral).uncacheSpan(0xb3dc568, 0xfffffc7fee7d94e8)
        /opt/ooce/go-1.17/src/runtime/mcentral.go:223 +0xcf
runtime.(*mcache).releaseAll(0xfffffc7fef14f3c8)
        /opt/ooce/go-1.17/src/runtime/mcache.go:279 +0x134
runtime.(*mcache).prepareForSweep(0xfffffc7fef14f3c8)
        /opt/ooce/go-1.17/src/runtime/mcache.go:317 +0x46
runtime.gcMarkTermination.func4.1(0xc00008a000)
        /opt/ooce/go-1.17/src/runtime/mgc.go:1052 +0x2f
runtime.forEachP(0x747b708)
        /opt/ooce/go-1.17/src/runtime/proc.go:1602 +0x12a
runtime.gcMarkTermination.func4()
        /opt/ooce/go-1.17/src/runtime/mgc.go:1051 +0x2d
runtime.systemstack()
        /opt/ooce/go-1.17/src/runtime/asm_amd64.s:383 +0x52

goroutine 41 [running]:
runtime.systemstack_switch()
        /opt/ooce/go-1.17/src/runtime/asm_amd64.s:350 fp=0xc000509d50 sp=0xc000509d48 pc=0x12b4b40
runtime.gcMarkTermination(0x3ff0393dbb020f24)
        /opt/ooce/go-1.17/src/runtime/mgc.go:1050 +0x525 fp=0xc000509f08 sp=0xc000509d50 pc=0x125ea85
runtime.gcMarkDone()
        /opt/ooce/go-1.17/src/runtime/mgc.go:902 +0x2d5 fp=0xc000509f60 sp=0xc000509f08 pc=0x125e435
runtime.gcBgMarkWorker()
        /opt/ooce/go-1.17/src/runtime/mgc.go:1330 +0x30c fp=0xc000509fe0 sp=0xc000509f60 pc=0x125f7ac
runtime.goexit()
        /opt/ooce/go-1.17/src/runtime/asm_amd64.s:1581 +0x1 fp=0xc000509fe8 sp=0xc000509fe0 pc=0x12b6d41
created by runtime.gcBgMarkStartWorkers
        /opt/ooce/go-1.17/src/runtime/mgc.go:1124 +0x37

goroutine 1 [runnable, locked to thread]:
github.com/aws/aws-sdk-go/aws/endpoints.init()
        /home/dap/garbage-compactor/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/vendor/github.com/aws/aws-sdk-go/aws/endpoints/defaults.go:3799 +0x4fb5e

goroutine 18 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.flushDaemon()
        /home/dap/garbage-compactor/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/util/log/log_flush.go:75 +0x76
created by github.com/cockroachdb/cockroach/pkg/util/log.init.5
        /home/dap/garbage-compactor/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/util/log/log_flush.go:41 +0x35

goroutine 19 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.signalFlusher()
        /home/dap/garbage-compactor/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/util/log/log_flush.go:98 +0x145
created by github.com/cockroachdb/cockroach/pkg/util/log.init.5
        /home/dap/garbage-compactor/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/util/log/log_flush.go:42 +0x4d

goroutine 22 [syscall]:
os/signal.signal_recv()
        /opt/ooce/go-1.17/src/runtime/sigqueue.go:169 +0xab
os/signal.loop()
        /opt/ooce/go-1.17/src/os/signal/signal_unix.go:24 +0x25
created by os/signal.Notify.func1.1
        /opt/ooce/go-1.17/src/os/signal/signal.go:151 +0x3a

goroutine 8 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/goschedstats.init.0.func1()
        /home/dap/garbage-compactor/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/util/goschedstats/runnable.go:165 +0x1de
created by github.com/cockroachdb/cockroach/pkg/util/goschedstats.init.0
        /home/dap/garbage-compactor/cockroach/cache/gopath/src/github.com/cockroachdb/cockroach/pkg/util/goschedstats/runnable.go:157 +0x35