From 590c0f75dd58a6d3c87a8beb53e71d72c25495ef Mon Sep 17 00:00:00 2001 From: Pablo Balbi Date: Tue, 31 Jan 2023 10:07:33 -0300 Subject: [PATCH] post rebase fixes --- CHANGELOG.md | 2 -- clients/pkg/promtail/client/manager.go | 3 ++- clients/pkg/promtail/client/manager_test.go | 1 - clients/pkg/promtail/config/config.go | 1 - clients/pkg/promtail/promtail.go | 10 +++----- clients/pkg/promtail/utils/entries.go | 27 +++++++++++---------- clients/pkg/promtail/utils/entries_test.go | 7 +++--- clients/pkg/promtail/wal/config.go | 6 ++--- clients/pkg/promtail/wal/reader.go | 7 +++--- clients/pkg/promtail/wal/wal.go | 3 ++- clients/pkg/promtail/wal/writer.go | 12 +++++---- clients/pkg/promtail/wal/writer_test.go | 3 +-- pkg/ingester/recovery.go | 2 +- 13 files changed, 42 insertions(+), 42 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 648296aad566..46c7aaf7e6b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,8 +47,6 @@ * [8315](https://github.com/grafana/loki/pull/8315) **thepalbi** Relicense and export `pkg/ingester` WAL code to be used in Promtail's WAL. -* [8120](https://github.com/grafana/loki/pull/8232) **TaehyunHwang** Fix version info issue that shows wrong version. -* #### Promtail ##### Enhancements diff --git a/clients/pkg/promtail/client/manager.go b/clients/pkg/promtail/client/manager.go index f1a5616982b0..c6df8391b636 100644 --- a/clients/pkg/promtail/client/manager.go +++ b/clients/pkg/promtail/client/manager.go @@ -6,9 +6,10 @@ import ( "sync" "github.com/go-kit/log" + "github.com/prometheus/client_golang/prometheus" + "github.com/grafana/loki/clients/pkg/promtail/api" "github.com/grafana/loki/clients/pkg/promtail/wal" - "github.com/prometheus/client_golang/prometheus" ) // Manager manages remote write client instantiation, and connects the related components to orchestrate the flow of api.Entry diff --git a/clients/pkg/promtail/client/manager_test.go b/clients/pkg/promtail/client/manager_test.go index 5ca105a29f5d..4c56f04b1c06 100644 --- a/clients/pkg/promtail/client/manager_test.go +++ b/clients/pkg/promtail/client/manager_test.go @@ -16,7 +16,6 @@ import ( "github.com/grafana/loki/clients/pkg/promtail/api" "github.com/grafana/loki/clients/pkg/promtail/wal" - "github.com/grafana/loki/pkg/logproto" ) diff --git a/clients/pkg/promtail/config/config.go b/clients/pkg/promtail/config/config.go index 3676864c416f..65437f58e5ad 100644 --- a/clients/pkg/promtail/config/config.go +++ b/clients/pkg/promtail/config/config.go @@ -16,7 +16,6 @@ import ( "github.com/grafana/loki/clients/pkg/promtail/server" "github.com/grafana/loki/clients/pkg/promtail/targets/file" "github.com/grafana/loki/clients/pkg/promtail/wal" - "github.com/grafana/loki/pkg/tracing" "github.com/grafana/loki/pkg/util/flagext" ) diff --git a/clients/pkg/promtail/promtail.go b/clients/pkg/promtail/promtail.go index fa0162e02895..5265552b7ae8 100644 --- a/clients/pkg/promtail/promtail.go +++ b/clients/pkg/promtail/promtail.go @@ -4,26 +4,24 @@ import ( "crypto/md5" "errors" "fmt" - "github.com/grafana/loki/clients/pkg/promtail/utils" "os" "os/signal" "sync" "syscall" - "github.com/grafana/loki/clients/pkg/promtail/api" - "github.com/grafana/loki/clients/pkg/promtail/wal" - "github.com/go-kit/log" "github.com/go-kit/log/level" "github.com/prometheus/client_golang/prometheus" "github.com/grafana/loki/clients/pkg/logentry/stages" + "github.com/grafana/loki/clients/pkg/promtail/api" "github.com/grafana/loki/clients/pkg/promtail/client" "github.com/grafana/loki/clients/pkg/promtail/config" "github.com/grafana/loki/clients/pkg/promtail/server" "github.com/grafana/loki/clients/pkg/promtail/targets" "github.com/grafana/loki/clients/pkg/promtail/targets/target" - + "github.com/grafana/loki/clients/pkg/promtail/utils" + "github.com/grafana/loki/clients/pkg/promtail/wal" util_log "github.com/grafana/loki/pkg/util/log" ) @@ -242,7 +240,7 @@ func (p *Promtail) Shutdown() { p.client.Stop() } -// ActiveTargets returns active handlers per jobs from the target manager +// ActiveTargets returns active targets per jobs from the target manager func (p *Promtail) ActiveTargets() map[string][]target.Target { return p.targetManagers.ActiveTargets() } diff --git a/clients/pkg/promtail/utils/entries.go b/clients/pkg/promtail/utils/entries.go index de8b8fdf90d7..065cdf288b32 100644 --- a/clients/pkg/promtail/utils/entries.go +++ b/clients/pkg/promtail/utils/entries.go @@ -1,8 +1,9 @@ package utils import ( - "github.com/grafana/loki/clients/pkg/promtail/api" "sync" + + "github.com/grafana/loki/clients/pkg/promtail/api" ) // EntryHandlerFanouter implements api.EntryHandler, fanning out received entries to one or multiple channels. @@ -14,18 +15,6 @@ type EntryHandlerFanouter struct { wg sync.WaitGroup } -func (x *EntryHandlerFanouter) Chan() chan<- api.Entry { - return x.entries -} - -// Stop only stops the channel EntryHandlerFanouter exposes, not the ones it fans out to. -func (x *EntryHandlerFanouter) Stop() { - x.once.Do(func() { - close(x.entries) - }) - x.wg.Wait() -} - func NewEntryHandlerFanouter(handlers ...api.EntryHandler) *EntryHandlerFanouter { multiplex := &EntryHandlerFanouter{ entries: make(chan api.Entry), @@ -42,3 +31,15 @@ func NewEntryHandlerFanouter(handlers ...api.EntryHandler) *EntryHandlerFanouter }() return multiplex } + +func (eh *EntryHandlerFanouter) Chan() chan<- api.Entry { + return eh.entries +} + +// Stop only stops the channel EntryHandlerFanouter exposes, not the ones it fans out to. +func (eh *EntryHandlerFanouter) Stop() { + eh.once.Do(func() { + close(eh.entries) + }) + eh.wg.Wait() +} diff --git a/clients/pkg/promtail/utils/entries_test.go b/clients/pkg/promtail/utils/entries_test.go index 0fce263ad58a..df1f34b574f2 100644 --- a/clients/pkg/promtail/utils/entries_test.go +++ b/clients/pkg/promtail/utils/entries_test.go @@ -1,14 +1,15 @@ package utils import ( - "github.com/grafana/loki/pkg/logproto" - "github.com/prometheus/common/model" - "github.com/stretchr/testify/require" "sync" "testing" "time" + "github.com/prometheus/common/model" + "github.com/stretchr/testify/require" + "github.com/grafana/loki/clients/pkg/promtail/api" + "github.com/grafana/loki/pkg/logproto" ) func TestEntryHandlerFanouter(t *testing.T) { diff --git a/clients/pkg/promtail/wal/config.go b/clients/pkg/promtail/wal/config.go index ed4b5790f54d..be158f66a4b6 100644 --- a/clients/pkg/promtail/wal/config.go +++ b/clients/pkg/promtail/wal/config.go @@ -10,15 +10,15 @@ const ( // Config contains all WAL-related settings. type Config struct { - // Path where the WAL is written to. - Dir string `yaml:"dir"` - // Whether WAL-support should be enabled. // // WAL support is a WIP. Do not enable in production setups until https://github.com/grafana/loki/issues/8197 // is finished. Enabled bool `yaml:"enabled"` + // Path where the WAL is written to. + Dir string `yaml:"dir"` + // MaxSegmentAge is threshold at which a WAL segment is considered old enough to be cleaned up. Default: 1h. MaxSegmentAge time.Duration `yaml:"cleanSegmentsOlderThan"` } diff --git a/clients/pkg/promtail/wal/reader.go b/clients/pkg/promtail/wal/reader.go index 3276f0694de2..59555f2347e8 100644 --- a/clients/pkg/promtail/wal/reader.go +++ b/clients/pkg/promtail/wal/reader.go @@ -2,12 +2,13 @@ package wal import ( "fmt" - "github.com/grafana/loki/clients/pkg/promtail/api" - "github.com/grafana/loki/pkg/ingester/wal" - walUtils "github.com/grafana/loki/pkg/util/wal" + "github.com/prometheus/common/model" + "github.com/grafana/loki/clients/pkg/promtail/api" + "github.com/grafana/loki/pkg/ingester/wal" "github.com/grafana/loki/pkg/util" + walUtils "github.com/grafana/loki/pkg/util/wal" ) // ReadWAL will read all entries in the WAL located under dir. Mainly used for testing diff --git a/clients/pkg/promtail/wal/wal.go b/clients/pkg/promtail/wal/wal.go index 5d516419de20..3d54925cedb1 100644 --- a/clients/pkg/promtail/wal/wal.go +++ b/clients/pkg/promtail/wal/wal.go @@ -2,13 +2,14 @@ package wal import ( "fmt" - "github.com/grafana/loki/pkg/ingester/wal" "os" "github.com/go-kit/log" "github.com/go-kit/log/level" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/prometheus/tsdb/wlog" + + "github.com/grafana/loki/pkg/ingester/wal" ) var ( diff --git a/clients/pkg/promtail/wal/writer.go b/clients/pkg/promtail/wal/writer.go index 3a9dd5e2a1de..7ade2c4801f5 100644 --- a/clients/pkg/promtail/wal/writer.go +++ b/clients/pkg/promtail/wal/writer.go @@ -2,8 +2,6 @@ package wal import ( "fmt" - "github.com/grafana/loki/pkg/ingester/wal" - "github.com/prometheus/client_golang/prometheus" "os" "path/filepath" "sort" @@ -13,11 +11,13 @@ import ( "github.com/go-kit/log" "github.com/go-kit/log/level" - "github.com/grafana/loki/clients/pkg/promtail/api" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/tsdb/chunks" "github.com/prometheus/prometheus/tsdb/record" + "github.com/grafana/loki/clients/pkg/promtail/api" + "github.com/grafana/loki/pkg/ingester/wal" "github.com/grafana/loki/pkg/logproto" "github.com/grafana/loki/pkg/util" ) @@ -28,7 +28,8 @@ const ( // Writer implements api.EntryHandler, exposing a channel were scraping targets can write to. Reading from there, it // writes incoming entries to a WAL. -// Also, since Writer is responsible for all changing operations over the WAL, a routine is run for cleaning old segments. +// Also, since Writer is responsible for all changing operations over the WAL, therefore a routine is run for cleaning +// old segments. type Writer struct { entries chan api.Entry log log.Logger @@ -220,12 +221,13 @@ type segmentRef struct { lastModified time.Time } +// listSegments list wal segments under the given directory, alongside with some file system information for each. func listSegments(dir string) (refs []segmentRef, err error) { files, err := os.ReadDir(dir) if err != nil { return nil, err } - // the following will attempt to get segments info in a best effort manner + // the following will attempt to get segments info in a best effort manner, omitting file if error for _, f := range files { fn := f.Name() k, err := strconv.Atoi(fn) diff --git a/clients/pkg/promtail/wal/writer_test.go b/clients/pkg/promtail/wal/writer_test.go index b71dc72cbfb1..f09a0ffba223 100644 --- a/clients/pkg/promtail/wal/writer_test.go +++ b/clients/pkg/promtail/wal/writer_test.go @@ -1,19 +1,18 @@ package wal import ( - "github.com/go-kit/log/level" "os" "path/filepath" "testing" "time" "github.com/go-kit/log" + "github.com/go-kit/log/level" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" "github.com/stretchr/testify/require" "github.com/grafana/loki/clients/pkg/promtail/api" - "github.com/grafana/loki/pkg/logproto" ) diff --git a/pkg/ingester/recovery.go b/pkg/ingester/recovery.go index 8bdc85804042..6b99b61f9724 100644 --- a/pkg/ingester/recovery.go +++ b/pkg/ingester/recovery.go @@ -1,7 +1,7 @@ package ingester import ( - io "io" + "io" "runtime" "sync"