Compact: Display Planned and Running Compactions

.bingo/gotesplit.sum

@@ -1,4 +1,6 @@
 github.com/Songmu/gotesplit v0.2.1 h1:qJFvR75nJpeKyMQFwyDtFrcc6zDWhrHAkks7DvM8oLo=
 github.com/Songmu/gotesplit v0.2.1/go.mod h1:sVBfmLT26b1H5VhUpq8cRhCVK75GAmW9c8r2NiK0gzk=
 github.com/jstemmer/go-junit-report v1.0.0 h1:8X1gzZpR+nVQLAht+L/foqOeX2l9DTZoaIPbEQHxsds=
+github.com/jstemmer/go-junit-report v1.0.0/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
 golang.org/x/sync v0.0.0-20220513210516-0976fa681c29 h1:w8s32wxx3sY+OjLlv9qltkLU5yvJzxjjgiHWLjdIcw4=
+golang.org/x/sync v0.0.0-20220513210516-0976fa681c29/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=

cmd/thanos/compact.go

@@ -239,6 +239,13 @@ func runCompact(
 	consistencyDelayMetaFilter := block.NewConsistencyDelayMetaFilter(logger, conf.consistencyDelay, extprom.WrapRegistererWithPrefix("thanos_", reg))
 	timePartitionMetaFilter := block.NewTimePartitionMetaFilter(conf.filterConf.MinTime, conf.filterConf.MaxTime)
 
+	levels, err := compactions.levels(conf.maxCompactionLevel)
+	if err != nil {
+		return errors.Wrap(err, "get compaction levels")
+	}
+
+	tsdbPlanner := compact.NewPlanner(logger, levels, noCompactMarkerFilter)
+
 	var blockLister block.Lister
 	switch syncStrategy(conf.blockListStrategy) {
 	case concurrentDiscovery:
@@ -288,6 +295,7 @@ func runCompact(
 		cf.UpdateOnChange(func(blocks []metadata.Meta, err error) {
 			api.SetLoaded(blocks, err)
 		})
+
 		sy, err = compact.NewMetaSyncer(
 			logger,
 			reg,
@@ -303,11 +311,6 @@ func runCompact(
 		}
 	}
 
-	levels, err := compactions.levels(conf.maxCompactionLevel)
-	if err != nil {
-		return errors.Wrap(err, "get compaction levels")
-	}
-
 	if conf.maxCompactionLevel < compactions.maxLevel() {
 		level.Warn(logger).Log("msg", "Max compaction level is lower than should be", "current", conf.maxCompactionLevel, "default", compactions.maxLevel())
 	}
@@ -371,7 +374,6 @@ func runCompact(
 	)
 	var planner compact.Planner
 
-	tsdbPlanner := compact.NewPlanner(logger, levels, noCompactMarkerFilter)
 	largeIndexFilterPlanner := compact.WithLargeTotalIndexSizeFilter(
 		tsdbPlanner,
 		insBkt,

pkg/api/blocks/v1.go

@@ -26,15 +26,16 @@ import (
 
 // BlocksAPI is a very simple API used by Thanos Block Viewer.
 type BlocksAPI struct {
-	baseAPI          *api.BaseAPI
-	logger           log.Logger
-	globalBlocksInfo *BlocksInfo
-	loadedBlocksInfo *BlocksInfo
+	baseAPI           *api.BaseAPI
+	logger            log.Logger
+	globalBlocksInfo  *BlocksInfo
+	loadedBlocksInfo  *BlocksInfo
+	plannedBlocksInfo *BlocksInfo
 
 	globalLock, loadedLock sync.Mutex
-	disableCORS            bool
-	bkt                    objstore.Bucket
-	disableAdminOperations bool
+	disableCORS                         bool
+	bkt                                 objstore.Bucket
+	disableAdminOperations              bool
 }
 
 type BlocksInfo struct {
@@ -90,6 +91,7 @@ func (bapi *BlocksAPI) Register(r *route.Router, tracer opentracing.Tracer, logg
 
 	r.Get("/blocks", instr("blocks", bapi.blocks))
 	r.Post("/blocks/mark", instr("blocks_mark", bapi.markBlock))
+	r.Get("/blocks/plan", instr("blocks_plan", bapi.plannedBlocks))
 }
 
 func (bapi *BlocksAPI) markBlock(r *http.Request) (interface{}, []error, *api.ApiError, func()) {
@@ -146,6 +148,19 @@ func (bapi *BlocksAPI) blocks(r *http.Request) (interface{}, []error, *api.ApiEr
 	return bapi.globalBlocksInfo, nil, nil, func() {}
 }
 
+func (bapi *BlocksAPI) plannedBlocks(r *http.Request) (interface{}, []error, *api.ApiError, func()) {
+	ctx := r.Context()
+
+
+	select {
+	case <-ctx.Done():
+		return nil, []error{ctx.Err()}, nil, func() {}
+	default:
+	}
+
+	return bapi.plannedBlocksInfo, nil, nil, func() {}
+}
+
 func (b *BlocksInfo) set(blocks []metadata.Meta, err error) {
 	if err != nil {
 		// Last view is maintained.
@@ -174,3 +189,9 @@ func (bapi *BlocksAPI) SetLoaded(blocks []metadata.Meta, err error) {
 
 	bapi.loadedBlocksInfo.set(blocks, err)
 }
+
+// SetPlanned updates the plan blocks' metadata in the API.
+func (bapi *BlocksAPI) SetPlanned(blocks []metadata.Meta, err error) {
+
+	bapi.plannedBlocksInfo.set(blocks, err)
+}

pkg/compact/compact.go

@@ -795,6 +795,11 @@ type Planner interface {
 	// Plan returns a list of blocks that should be compacted into single one.
 	// The blocks can be overlapping. The provided metadata has to be ordered by minTime.
 	Plan(ctx context.Context, metasByMinTime []*metadata.Meta, errChan chan error, extensions any) ([]*metadata.Meta, error)
+	UpdateOnPlanned(f func([]metadata.Meta, error))
+}
+
+func (p *tsdbBasedPlanner) UpdateOnPlanned(f func([]metadata.Meta, error)) {
+	p.updateOnPlanned = f
 }
 
 type BlockDeletableChecker interface {
@@ -1135,6 +1140,9 @@ func (cg *Group) compact(ctx context.Context, dir string, planner Planner, comp
 	var toCompact []*metadata.Meta
 	if err := tracing.DoInSpanWithErr(ctx, "compaction_planning", func(ctx context.Context) (e error) {
 		toCompact, e = planner.Plan(ctx, cg.metasByMinTime, errChan, cg.extensions)
+		planner.UpdateOnPlanned(func(metas []metadata.Meta, err error) {
+			level.Info(cg.logger).Log("msg", "planner updated", "metas", len(metas), "err", err)
+		})
 		return e
 	}); err != nil {
 		return false, nil, errors.Wrap(err, "plan compaction")

pkg/compact/planner.go

@@ -25,6 +25,7 @@ type tsdbBasedPlanner struct {
 	ranges []int64
 
 	noCompBlocksFunc func() map[ulid.ULID]*metadata.NoCompactMark
+	updateOnPlanned  func([]metadata.Meta, error)
 }
 
 var _ Planner = &tsdbBasedPlanner{}

pkg/ui/react-app/src/App.tsx

@@ -5,6 +5,7 @@ import { QueryParamProvider } from 'use-query-params';
 import useMedia from 'use-media';
 
 import { Alerts, Config, Flags, Rules, ServiceDiscovery, Status, Targets, TSDBStatus, PanelList, NotFound } from './pages';
+import { GlobalBlocks, PlannedBlocks } from './thanos/pages/blocks/Blocks';
 import PathPrefixProps from './types/PathPrefixProps';
 import ThanosComponentProps from './thanos/types/ThanosComponentProps';
 import Navigation from './thanos/Navbar';
@@ -17,7 +18,7 @@ const defaultRouteConfig: { [component: string]: string } = {
   query: '/graph',
   rule: '/alerts',
   bucket: '/blocks',
-  compact: '/loaded',
+  compact: '/global-blocks',
   store: '/loaded',
 };
 
@@ -61,6 +62,8 @@ const App: FC<PathPrefixProps & ThanosComponentProps> = ({ pathPrefix, thanosCom
               <Stores path="/stores" pathPrefix={pathPrefix} />
               <Blocks path="/blocks" pathPrefix={pathPrefix} />
               <Blocks path="/loaded" pathPrefix={pathPrefix} view="loaded" />
+              <GlobalBlocks path="/global-blocks" pathPrefix={pathPrefix} />
+              <PlannedBlocks path="/planned-blocks" pathPrefix={pathPrefix} />
               <NotFound pathPrefix={pathPrefix} default defaultRoute={defaultRouteConfig[thanosComponent]} />
             </Router>
           </QueryParamProvider>

pkg/ui/react-app/src/thanos/Navbar.tsx

@@ -63,7 +63,8 @@ const navConfig: { [component: string]: (NavConfig | NavDropDown)[] } = {
     },
   ],
   compact: [
-    { name: 'Global Blocks', uri: '/blocks' },
+    { name: 'Global Blocks', uri: '/global-blocks' },
+    { name: 'Planned Blocks', uri: '/planned-blocks' },
     { name: 'Loaded Blocks', uri: '/loaded' },
     {
       name: 'Status',

pkg/ui/react-app/src/thanos/pages/blocks/Blocks.tsx

@@ -207,7 +207,7 @@ interface BlocksProps {
 
 export const Blocks: FC<RouteComponentProps & PathPrefixProps & BlocksProps> = ({ pathPrefix = '', view = 'global' }) => {
   const { response, error, isLoading } = useFetch<BlockListProps>(
-    `${pathPrefix}/api/v1/blocks${view ? '?view=' + view : ''}`
+    `${pathPrefix}/api/v1/blocks${view === 'planned' ? '/plan' : view ? '?view=' + view : ''}`
   );
   const { status: responseStatus } = response;
   const badResponse = responseStatus !== 'success' && responseStatus !== 'start fetching';
@@ -221,5 +221,12 @@ export const Blocks: FC<RouteComponentProps & PathPrefixProps & BlocksProps> = (
     />
   );
 };
+export const GlobalBlocks: React.FC<RouteComponentProps & PathPrefixProps> = ({ pathPrefix }) => {
+  return <Blocks view="global" pathPrefix={pathPrefix} />;
+};
+
+export const PlannedBlocks: React.FC<RouteComponentProps & PathPrefixProps> = ({ pathPrefix }) => {
+  return <Blocks view="planned" pathPrefix={pathPrefix} />;
+};
 
 export default Blocks;

pkg/ui/static/react/asset-manifest.json

@@ -1,15 +1,15 @@
 {
   "files": {
-    "main.css": "./static/css/main.17d667f4.css",
-    "main.js": "./static/js/main.20f9b436.js",
+    "main.css": "./static/css/main.0cc3dff0.css",
+    "main.js": "./static/js/main.39be79f4.js",
     "static/media/codicon.ttf": "./static/media/codicon.b3726f0165bf67ac6849.ttf",
     "index.html": "./index.html",
     "static/media/index.cjs": "./static/media/index.cd351d7c31d0d3fccf96.cjs",
-    "main.17d667f4.css.map": "./static/css/main.17d667f4.css.map",
-    "main.20f9b436.js.map": "./static/js/main.20f9b436.js.map"
+    "main.0cc3dff0.css.map": "./static/css/main.0cc3dff0.css.map",
+    "main.39be79f4.js.map": "./static/js/main.39be79f4.js.map"
   },
   "entrypoints": [
-    "static/css/main.17d667f4.css",
-    "static/js/main.20f9b436.js"
+    "static/css/main.0cc3dff0.css",
+    "static/js/main.39be79f4.js"
   ]
 }

pkg/ui/static/react/index.html

@@ -1 +1 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="shortcut icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,shrink-to-fit=no"/><meta name="theme-color" content="#000000"/><script>const GLOBAL_PATH_PREFIX="{{ pathPrefix }}"</script><script>const THANOS_COMPONENT="{{ .Component }}",THANOS_QUERY_URL="{{ .queryURL }}",THANOS_TENANT_HEADER="{{ .tenantHeader }}",THANOS_DEFAULT_TENANT="{{ .defaultTenant }}",THANOS_DISPLAY_TENANT_BOX="{{ .displayTenantBox }}"</script><link rel="manifest" href="./manifest.json"/><title>Thanos | Highly available Prometheus setup</title><script defer="defer" src="./static/js/main.20f9b436.js"></script><link href="./static/css/main.17d667f4.css" rel="stylesheet"></head><body class="bootstrap"><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="shortcut icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,shrink-to-fit=no"/><meta name="theme-color" content="#000000"/><script>const GLOBAL_PATH_PREFIX="{{ pathPrefix }}"</script><script>const THANOS_COMPONENT="{{ .Component }}",THANOS_QUERY_URL="{{ .queryURL }}",THANOS_TENANT_HEADER="{{ .tenantHeader }}",THANOS_DEFAULT_TENANT="{{ .defaultTenant }}",THANOS_DISPLAY_TENANT_BOX="{{ .displayTenantBox }}"</script><link rel="manifest" href="./manifest.json"/><title>Thanos | Highly available Prometheus setup</title><script defer="defer" src="./static/js/main.39be79f4.js"></script><link href="./static/css/main.0cc3dff0.css" rel="stylesheet"></head><body class="bootstrap"><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

test/e2e/compact_test.go

@@ -18,6 +18,7 @@ import (
 
 	"github.com/efficientgo/e2e"
 	e2edb "github.com/efficientgo/e2e/db"
+	e2einteractive "github.com/efficientgo/e2e/interactive"
 	e2emon "github.com/efficientgo/e2e/monitoring"
 	"github.com/efficientgo/e2e/monitoring/matchers"
 	"github.com/go-kit/log"
@@ -692,6 +693,9 @@ func testCompactWithStoreGateway(t *testing.T, penaltyDedup bool) {
 		c := cFuture.Init(bktConfig, nil, extArgs...)
 		testutil.Ok(t, e2e.StartAndWaitReady(c))
 
+		testutil.Ok(t, e2einteractive.OpenInBrowser("http://"+c.Endpoint("http")))
+		time.Sleep(5 * time.Minute)
+
 		// NOTE: We cannot assert on intermediate `thanos_blocks_meta_` metrics as those are gauge and change dynamically due to many
 		// compaction groups. Wait for at least first compaction iteration (next is in 5m).
 		testutil.Ok(t, c.WaitSumMetrics(e2emon.Greater(0), "thanos_compact_iterations_total"))