Panic opening boltdb file

[slim-bean]

2020-11-20 16:24:40.361814 I | proto: duplicate proto type registered: ingester.Series
panic: invalid freelist page: 781, page type is leaf

goroutine 1 [running]:
go.etcd.io/bbolt.(*freelist).read(0xc0001c0500, 0x7f16a64ff000)
        /src/loki/vendor/go.etcd.io/bbolt/freelist.go:266 +0x30b
go.etcd.io/bbolt.(*DB).loadFreelist.func1()
        /src/loki/vendor/go.etcd.io/bbolt/db.go:316 +0xd4
sync.(*Once).doSlow(0xc000fe4568, 0xc000a27e68)
        /usr/local/go/src/sync/once.go:66 +0xec
sync.(*Once).Do(...)
        /usr/local/go/src/sync/once.go:57
go.etcd.io/bbolt.(*DB).loadFreelist(0xc000fe4400)
        /src/loki/vendor/go.etcd.io/bbolt/db.go:309 +0x6a
go.etcd.io/bbolt.Open(0xc000eda230, 0x4b, 0x1b6, 0xc000a28170, 0x4b, 0x0, 0x3c)
        /src/loki/vendor/go.etcd.io/bbolt/db.go:286 +0x36f
github.com/cortexproject/cortex/pkg/chunk/local.OpenBoltdbFile(...)
        /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/local/boltdb_index_client.go:366
github.com/grafana/loki/pkg/storage/stores/shipper/downloads.LoadTable(0x2655060, 0xc000522380, 0xc000e463a3, 0x14, 0x7ffecf06ee76, 0x12, 0x7f16e62d7dc8, 0xc000f2c380, 0x7f16e62d7da8, 0xc00158c410, ...)
        /src/loki/pkg/storage/stores/shipper/downloads/table.go:145 +0xb6d
github.com/grafana/loki/pkg/storage/stores/shipper/downloads.(*TableManager).loadLocalTables(0xc000ebe320, 0xc000ebe320, 0xc000f2c5a0)
        /src/loki/pkg/storage/stores/shipper/downloads/table_manager.go:345 +0x372
github.com/grafana/loki/pkg/storage/stores/shipper/downloads.NewTableManager(0x7ffecf06ee76, 0x12, 0x45d964b800, 0x4e94914f0000, 0x0, 0x7f16e62d7da8, 0xc00158c410, 0x7f16e62d7dc8, 0xc000f2c380, 0x26435e0, ...)
        /src/loki/pkg/storage/stores/shipper/downloads/table_manager.go:68 +0x198
github.com/grafana/loki/pkg/storage/stores/shipper.(*Shipper).init(0xc000ebe280, 0x265af60, 0xc0005c46c0, 0x26435e0, 0xc000180780, 0x0, 0x226b6ac)
        /src/loki/pkg/storage/stores/shipper/shipper_index_client.go:147 +0x2a5
github.com/grafana/loki/pkg/storage/stores/shipper.NewShipper(0x0, 0x0, 0xc00005c240, 0x3, 0x7ffecf06ee76, 0x12, 0x4e94914f0000, 0x45d964b800, 0x0, 0xc0003ce964, ...)
        /src/loki/pkg/storage/stores/shipper/shipper_index_client.go:94 +0x157
github.com/grafana/loki/pkg/storage.RegisterCustomIndexClients.func1(0x1ee8d00, 0xc000506f90, 0xc0003cfde0, 0xe)
        /src/loki/pkg/storage/store.go:356 +0x17c
github.com/cortexproject/cortex/pkg/chunk/storage.NewIndexClient(0xc0003cfde0, 0xe, 0x2250a0d, 0x6, 0x0, 0x4000000000000000, 0x4024000000000000, 0x0, 0x0, 0x186a0, ...)
        /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/storage/factory.go:208 +0xbb6
github.com/cortexproject/cortex/pkg/chunk/storage.NewStore(0x2250a0d, 0x6, 0x0, 0x4000000000000000, 0x4024000000000000, 0x0, 0x0, 0x186a0, 0x3ff4cccccccccccd, 0x3ff0000000000000, ...)
        /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/storage/factory.go:174 +0x730
github.com/grafana/loki/pkg/loki.(*Loki).initStore(0xc000272000, 0xc000bf00e0, 0x0, 0x0, 0xc0006167d0)
        /src/loki/pkg/loki/modules.go:291 +0x272
github.com/cortexproject/cortex/pkg/util/modules.(*Manager).initModule(0xc000120bd0, 0x7ffecf06ef2f, 0x7, 0xc000a2c0b0, 0xc0009c70e0, 0x0, 0x0)
        /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/modules/modules.go:103 +0x2c8
github.com/cortexproject/cortex/pkg/util/modules.(*Manager).InitModuleServices(0xc000120bd0, 0xc000a2c258, 0x1, 0x1, 0xc0000afd40, 0x2668da0, 0xc0000afd40)
        /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/modules/modules.go:75 +0x105
github.com/grafana/loki/pkg/loki.(*Loki).Run(0xc000272000, 0xc0005c6340, 0x4)
        /src/loki/pkg/loki/loki.go:204 +0x8b
main.main()
        /src/loki/cmd/loki/main.go:132 +0x59c

This was found on a querier and put it into a tight crashloop, might be nice to catch this and just delete the problem file, or consider logging the error and sleeping for 5min to let someone exec into the pod?