Test_HeadIteratorReverse, Test_Iterator failing in 2.2.0 release

[daurnimator]

=== RUN   Test_HeadIteratorReverse
--- FAIL: Test_HeadIteratorReverse (0.09s)
panic: runtime error: slice bounds out of range [:3] with capacity 1 [recovered]
	panic: runtime error: slice bounds out of range [:3] with capacity 1

goroutine 92005 [running]:
testing.tRunner.func1.2(0xc7bb00, 0xc00003e2e8)
	/usr/lib/go/src/testing/testing.go:1144 +0x332
testing.tRunner.func1(0xc000369500)
	/usr/lib/go/src/testing/testing.go:1147 +0x4b6
panic(0xc7bb00, 0xc00003e2e8)
	/usr/lib/go/src/runtime/panic.go:965 +0x1b9
github.com/grafana/loki/pkg/logql/log/logfmt.(*Decoder).ScanKeyval(0xc0004ac0c0, 0xc00158f700)
	/build/loki/src/loki-2.2.0/pkg/logql/log/logfmt/decode.go:62 +0xb27
github.com/grafana/loki/pkg/logql/log.(*LogfmtParser).Process(0xc035454618, 0xc00413c1f0, 0xc, 0x1, 0xc0127b0200, 0xc00413c200, 0xc00158f820, 0x7f6180369c80, 0x28)
	/build/loki/src/loki-2.2.0/pkg/logql/log/parser.go:251 +0xae
github.com/grafana/loki/pkg/logql/log.(*streamPipeline).Process(0xc00081e160, 0xc00413c1f0, 0xc, 0x1, 0xc00413aba0, 0x0, 0x0, 0xc00413aba0, 0x414abb, 0x1)
	/build/loki/src/loki-2.2.0/pkg/logql/log/pipeline.go:140 +0xe2
github.com/grafana/loki/pkg/logql/log.(*streamPipeline).ProcessString(0xc00081e160, 0xc00413c1f0, 0xc, 0x0, 0x1, 0xc00413aba0, 0x0, 0x1)
	/build/loki/src/loki-2.2.0/pkg/logql/log/pipeline.go:151 +0x65
github.com/grafana/loki/pkg/chunkenc.(*headBlock).iterator.func1(0x32167, 0xc00413c1f0, 0xc)
	/build/loki/src/loki-2.2.0/pkg/chunkenc/memchunk.go:818 +0x92
github.com/grafana/loki/pkg/chunkenc.(*headBlock).iterator(0xc0344aaea0, 0xe0a350, 0xc000124018, 0x1, 0x0, 0x32169, 0xe00c88, 0xc00081e160, 0x40008, 0xe0a350)
	/build/loki/src/loki-2.2.0/pkg/chunkenc/memchunk.go:842 +0x4fd
github.com/grafana/loki/pkg/chunkenc.(*MemChunk).Iterator(0xc00158ff30, 0xe0a350, 0xc000124018, 0x0, 0xe7791f700, 0x12983c0, 0x32169, 0xe7791f700, 0x12983c0, 0x1, ...)
	/build/loki/src/loki-2.2.0/pkg/chunkenc/memchunk.go:685 +0xaff
github.com/grafana/loki/pkg/chunkenc.Test_HeadIteratorReverse.func2(0xc000369500, 0x32169)
	/build/loki/src/loki-2.2.0/pkg/chunkenc/memchunk_test.go:1045 +0x2bb
github.com/grafana/loki/pkg/chunkenc.Test_HeadIteratorReverse(0xc000369500)
	/build/loki/src/loki-2.2.0/pkg/chunkenc/memchunk_test.go:1053 +0x1b1
testing.tRunner(0xc000369500, 0xd56b48)
	/usr/lib/go/src/testing/testing.go:1194 +0xef
created by testing.(*T).Run
	/usr/lib/go/src/testing/testing.go:1239 +0x2b3
FAIL	github.com/grafana/loki/pkg/chunkenc	29.650s

=== RUN   Test_Iterator
--- FAIL: Test_Iterator (0.00s)
panic: runtime error: slice bounds out of range [:3] with capacity 1 [recovered]
	panic: runtime error: slice bounds out of range [:3] with capacity 1

goroutine 1134 [running]:
testing.tRunner.func1.2(0x1938ce0, 0xc00408f1d0)
	/usr/lib/go/src/testing/testing.go:1144 +0x332
testing.tRunner.func1(0xc0052be300)
	/usr/lib/go/src/testing/testing.go:1147 +0x4b6
panic(0x1938ce0, 0xc00408f1d0)
	/usr/lib/go/src/runtime/panic.go:965 +0x1b9
github.com/grafana/loki/pkg/logql/log/logfmt.(*Decoder).ScanKeyval(0xc000c0dc20, 0xc000dd0c00)
	/build/loki/src/loki-2.2.0/pkg/logql/log/logfmt/decode.go:62 +0xb27
github.com/grafana/loki/pkg/logql/log.(*LogfmtParser).Process(0xc00023c0d0, 0xc00408f188, 0x12, 0x1, 0xc0052c8080, 0xc00408f1a0, 0xc000dd0d40, 0x7fdcfd66bc10, 0x28)
	/build/loki/src/loki-2.2.0/pkg/logql/log/parser.go:251 +0xae
github.com/grafana/loki/pkg/logql/log.(*streamPipeline).Process(0xc005292340, 0xc00408f188, 0x12, 0x1, 0xc0052bd3e0, 0x0, 0x0, 0xc0052bd3e0, 0x414d1b, 0x1)
	/build/loki/src/loki-2.2.0/pkg/logql/log/pipeline.go:140 +0xe2
github.com/grafana/loki/pkg/logql/log.(*streamPipeline).ProcessString(0xc005292340, 0xc00408f188, 0x12, 0x0, 0x1, 0xc0052bd3e0, 0x0, 0x1)
	/build/loki/src/loki-2.2.0/pkg/logql/log/pipeline.go:151 +0x65
github.com/grafana/loki/pkg/chunkenc.(*headBlock).iterator.func1(0x7, 0xc00408f188, 0x12)
	/build/loki/src/loki-2.2.0/pkg/chunkenc/memchunk.go:818 +0x92
github.com/grafana/loki/pkg/chunkenc.(*headBlock).iterator(0xc0052bccf0, 0x1ceb360, 0xc0000400d0, 0xc000000001, 0x0, 0x5f5e100, 0x1cd3e78, 0xc005292340, 0xc000dd1130, 0x452fe5)
	/build/loki/src/loki-2.2.0/pkg/chunkenc/memchunk.go:842 +0x4fd
github.com/grafana/loki/pkg/chunkenc.(*MemChunk).Iterator(0xc0052c8000, 0x1ceb360, 0xc0000400d0, 0x0, 0xe7791f700, 0x27c4300, 0x5f5e100, 0xe7791f700, 0x27c4300, 0xc000000001, ...)
	/build/loki/src/loki-2.2.0/pkg/chunkenc/memchunk.go:685 +0xaff
github.com/grafana/loki/pkg/ingester.(*stream).Iterator(0xc003db2d80, 0x1ceb360, 0xc0000400d0, 0xc000dd15c0, 0x0, 0xe7791f700, 0x27c4300, 0x5f5e100, 0xe7791f700, 0x27c4300, ...)
	/build/loki/src/loki-2.2.0/pkg/ingester/stream.go:327 +0x3ad
github.com/grafana/loki/pkg/ingester.(*instance).Query.func1(0xc003db2d80, 0xc0052bc930, 0x493dae6fce6b5197)
	/build/loki/src/loki-2.2.0/pkg/ingester/instance.go:301 +0x133
github.com/grafana/loki/pkg/ingester.(*instance).forMatchingStreams(0xc0051f41c0, 0xc00023c0c8, 0x1, 0x1, 0xc000dd1610, 0x0, 0x0)
	/build/loki/src/loki-2.2.0/pkg/ingester/instance.go:466 +0x22a
github.com/grafana/loki/pkg/ingester.(*instance).Query(0xc0051f41c0, 0x1ceb360, 0xc0000400d0, 0xc000dd1a68, 0x0, 0x0, 0x0, 0xc00058d440, 0x0)
	/build/loki/src/loki-2.2.0/pkg/ingester/instance.go:298 +0x1f8
github.com/grafana/loki/pkg/ingester.Test_Iterator(0xc0052be300)
	/build/loki/src/loki-2.2.0/pkg/ingester/instance_test.go:395 +0x7e5
testing.tRunner(0xc0052be300, 0x1ab25d0)
	/usr/lib/go/src/testing/testing.go:1194 +0xef
created by testing.(*T).Run
	/usr/lib/go/src/testing/testing.go:1239 +0x2b3
FAIL	github.com/grafana/loki/pkg/ingester	85.390s

Environment:

• OS: ArchLinux
• Architecture: x86-64
• Go: 1.16.2

[daurnimator]

This appears to have been broken by #3383 .... apparently your CI doesn't actually run the test suite?

[cyriltovena]

This appears to have been broken by #3383 .... apparently your CI doesn't actually run the test suite?

No this is from golang 1.16.

I'll take a look, and probably work on #3481 with @slim-bean

[daurnimator]

Are we likely to get a bugfix release soon? (with e.g. this and #3502)
Or should we cherry-pick the fixes into Arch?

[cyriltovena]

2.2 uses go 1.15 so it should be fine we don't plan to hotfix for this so if you want 2.2 with go 1.16 I suggest you cherry pick the commit.