Skip to content

Commit

Permalink
Adds sleep command at the end of compaction. (#660) (#667)
Browse files Browse the repository at this point in the history
  • Loading branch information
abdasgupta authored Oct 4, 2023
1 parent 686fbfb commit 582d985
Show file tree
Hide file tree
Showing 3 changed files with 34 additions and 10 deletions.
17 changes: 17 additions & 0 deletions pkg/compactor/compactor.go
Original file line number Diff line number Diff line change
Expand Up @@ -181,5 +181,22 @@ func (cp *Compactor) Compact(ctx context.Context, opts *brtypes.CompactOptions)
cancel()
}

// Add a sleep command so that prometheus can collect necessary metrics related to the uploading of snapshots. see https://github.com/gardener/etcd-druid/issues/648
err = sleepWithContext(ctx, opts.MetricsScrapeWaitDuration.Duration)
if err != nil {
cp.logger.Warnf("Could not sleep for specified duration: %v", err)
}

return snapshot, nil
}

func sleepWithContext(ctx context.Context, sleepFor time.Duration) error {
for {
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(sleepFor):
return nil
}
}
}
9 changes: 5 additions & 4 deletions pkg/compactor/compactor_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -89,10 +89,11 @@ var _ = Describe("Running Compactor", func() {
PeerURLs: peerUrls,
}
compactorConfig = &brtypes.CompactorConfig{
NeedDefragmentation: needDefragmentation,
SnapshotTimeout: wrappers.Duration{Duration: snapshotTimeout},
DefragTimeout: wrappers.Duration{Duration: defragTimeout},
EnabledLeaseRenewal: false,
NeedDefragmentation: needDefragmentation,
SnapshotTimeout: wrappers.Duration{Duration: snapshotTimeout},
DefragTimeout: wrappers.Duration{Duration: defragTimeout},
EnabledLeaseRenewal: false,
MetricsScrapeWaitDuration: wrappers.Duration{Duration: 0},
}
compactOptions = &brtypes.CompactOptions{
RestoreOptions: restoreOpts,
Expand Down
18 changes: 12 additions & 6 deletions pkg/types/compactor.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@ const (
defaultDefragTimeout time.Duration = 8 * time.Minute
// defaultSnapshotTimeout defines default timeout duration for taking compacted FullSnapshot.
defaultSnapshotTimeout time.Duration = 30 * time.Minute
//defaultMetricsScrapeWaitDuration defines default duration to wait for after compaction is completed, to allow Prometheus metrics to be scraped
defaultMetricsScrapeWaitDuration time.Duration = 0 * time.Second
)

// CompactOptions holds all configurable options of compact.
Expand All @@ -43,17 +45,20 @@ type CompactorConfig struct {
FullSnapshotLeaseName string `json:"fullSnapshotLeaseName,omitempty"`
DeltaSnapshotLeaseName string `json:"deltaSnapshotLeaseName,omitempty"`
EnabledLeaseRenewal bool `json:"enabledLeaseRenewal"`
// see https://github.com/gardener/etcd-druid/issues/648
MetricsScrapeWaitDuration wrappers.Duration `json:"metricsScrapeWaitDuration,omitempty"`
}

// NewCompactorConfig returns the CompactorConfig.
func NewCompactorConfig() *CompactorConfig {
return &CompactorConfig{
NeedDefragmentation: true,
SnapshotTimeout: wrappers.Duration{Duration: defaultSnapshotTimeout},
DefragTimeout: wrappers.Duration{Duration: defaultDefragTimeout},
FullSnapshotLeaseName: DefaultFullSnapshotLeaseName,
DeltaSnapshotLeaseName: DefaultDeltaSnapshotLeaseName,
EnabledLeaseRenewal: DefaultSnapshotLeaseRenewalEnabled,
NeedDefragmentation: true,
SnapshotTimeout: wrappers.Duration{Duration: defaultSnapshotTimeout},
DefragTimeout: wrappers.Duration{Duration: defaultDefragTimeout},
FullSnapshotLeaseName: DefaultFullSnapshotLeaseName,
DeltaSnapshotLeaseName: DefaultDeltaSnapshotLeaseName,
EnabledLeaseRenewal: DefaultSnapshotLeaseRenewalEnabled,
MetricsScrapeWaitDuration: wrappers.Duration{Duration: defaultMetricsScrapeWaitDuration},
}
}

Expand All @@ -65,6 +70,7 @@ func (c *CompactorConfig) AddFlags(fs *flag.FlagSet) {
fs.StringVar(&c.FullSnapshotLeaseName, "full-snapshot-lease-name", c.FullSnapshotLeaseName, "full snapshot lease name")
fs.StringVar(&c.DeltaSnapshotLeaseName, "delta-snapshot-lease-name", c.DeltaSnapshotLeaseName, "delta snapshot lease name")
fs.BoolVar(&c.EnabledLeaseRenewal, "enable-snapshot-lease-renewal", c.EnabledLeaseRenewal, "Allows compactor to renew the full snapshot lease when successfully compacted snapshot is uploaded")
fs.DurationVar(&c.MetricsScrapeWaitDuration.Duration, "metrics-scrape-wait-duration", c.MetricsScrapeWaitDuration.Duration, "The duration to wait for after compaction is completed, to allow Prometheus metrics to be scraped")
}

// Validate validates the config.
Expand Down

0 comments on commit 582d985

Please sign in to comment.