Skip to content

Commit

Permalink
compact: recover from panics (thanos-io#7318)
Browse files Browse the repository at this point in the history
For thanos-io#6775, it would be useful
to know the exact block IDs to aid debugging.

Signed-off-by: Giedrius Statkevičius <giedrius.statkevicius@vinted.com>
  • Loading branch information
GiedriusS authored and saswatamcode committed May 28, 2024
1 parent d9a0efa commit 8e0321b
Showing 1 changed file with 16 additions and 0 deletions.
16 changes: 16 additions & 0 deletions pkg/compact/compact.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"os"
"path/filepath"
"sort"
"strings"
"sync"
"time"

Expand Down Expand Up @@ -871,6 +872,21 @@ func (cg *Group) Compact(ctx context.Context, dir string, planner Planner, comp
return false, ulid.ULID{}, errors.Wrap(err, "create compaction group dir")
}

defer func() {
if p := recover(); p != nil {
var sb strings.Builder

cgIDs := cg.IDs()
for i, blid := range cgIDs {
_, _ = sb.WriteString(blid.String())
if i < len(cgIDs)-1 {
_, _ = sb.WriteString(",")
}
}
rerr = fmt.Errorf("paniced while compacting %s: %v", sb.String(), p)
}
}()

errChan := make(chan error, 1)
err := tracing.DoInSpanWithErr(ctx, "compaction_group", func(ctx context.Context) (err error) {
shouldRerun, compID, err = cg.compact(ctx, subDir, planner, comp, blockDeletableChecker, compactionLifecycleCallback, errChan)
Expand Down

0 comments on commit 8e0321b

Please sign in to comment.