Merge pull request grafana#188 from grafana/main

Update from upstream repository

.drone/drone.jsonnet

@@ -363,7 +363,6 @@ local logql_analyzer() = pipeline('logql-analyzer') + arch_image('amd64') {
   depends_on: ['check'],
 };
 
-
 local multiarch_image(arch) = pipeline('docker-' + arch) + arch_image(arch) {
   steps+: [
     // dry run for everything that is not tag or main
@@ -917,6 +916,7 @@ local manifest_ecr(apps, archs) = pipeline('manifest-ecr') {
           DOCKER_PASSWORD: { from_secret: docker_password_secret.name },
         },
         commands: [
+          'git fetch origin --tags',
           'make docker-driver-push',
         ],
         volumes: [

.drone/drone.yml

@@ -1743,6 +1743,7 @@ kind: pipeline
 name: docker-driver
 steps:
 - commands:
+  - git fetch origin --tags
   - make docker-driver-push
   depends_on:
   - clone
@@ -2016,6 +2017,6 @@ kind: secret
 name: gpg_private_key
 ---
 kind: signature
-hmac: 6aff5ae73eeaed171dafbc08b599abaf9a4ef4c1d68212da1a1442f36f5c7d8b
+hmac: 93e04a0de03da46eb44ad99732caf328ff578d627473c6ccf3bed8401d03f666
 
 ...

.github/workflows/dependabot_reviewer.yml

@@ -31,4 +31,4 @@ jobs:
           --approve -b "Auto approve dependencies bump PR"
         env:
           PR_URL: ${{github.event.pull_request.html_url}}
-          GITHUB_TOKEN: ${{secrets.GH_BOT_ACCESS_TOKEN}}
+          GITHUB_TOKEN: "${{ secrets.GH_BOT_ACCESS_TOKEN }}"

CHANGELOG.md

@@ -17,6 +17,9 @@
 * [10395](https://github.com/grafana/loki/pull/10395/) **shantanualshi** Remove deprecated `split_queries_by_interval` and `forward_headers_list` configuration options in the `query_range` section
 * [10456](https://github.com/grafana/loki/pull/10456) **dannykopping** Add `loki_distributor_ingester_append_timeouts_total` metric, remove `loki_distributor_ingester_append_failures_total` metric
 * [10534](https://github.com/grafana/loki/pull/10534) **chaudum** Remove configuration `use_boltdb_shipper_as_backup`
+* [10620](https://github.com/grafana/loki/pull/10620) **ashwanthgoli** Enable embedded cache if no other cache is explicitly enabled.
+* [10655](https://github.com/grafana/loki/pull/10655) **chaudum** Remove legacy ingester shutdown handler `/ingester/flush_shutdown`.
+* [10693](https://github.com/grafana/loki/pull/10693) **ashwanthgoli** Embedded cache: Updates the metric prefix from `querier_cache_` to `loki_embeddedcache_` and removes duplicate metrics.
 
 ##### Fixes
 
@@ -32,8 +35,15 @@
 * [10416](https://github.com/grafana/loki/pull/10416) **lpugoy**: Lambda-Promtail: Add support for WAF logs in S3
 * [10301](https://github.com/grafana/loki/pull/10301) **wildum**: users can now define `additional_fields` in cloudflare configuration.
 
+##### Changes
+
+* [10677](https://github.com/grafana/loki/pull/10677) **chaudum** Remove deprecated `stream_lag_labels` setting from both the `options` and `client` configuration sections.
+* [10689](https://github.com/grafana/loki/pull/10689) **dylanguedes**: Ingester: Make jitter to be 20% of flush check period instead of 1%.
+
 ##### Fixes
 
+* [10631](https://github.com/grafana/loki/pull/10631) **thampiotr**: Fix race condition in cleaning up metrics when stopping to tail files.
+
 #### LogCLI
 
 ##### Fixes
@@ -80,10 +90,13 @@
 #### Loki
 
 ##### Enhancements
-* [10322](https://github.com/grafana/loki/pull/10322) **chaudum**: Deprecate misleading setting `-ruler.evaluation-delay-duration`.
-* [10324](https://github.com/grafana/loki/pull/10324) **ashwanthgoli**: Deprecate ingester.unordered-writes and a few unused configs(log.use-buffered, log.use-sync, frontend.forward-headers-list)
 
+* [10101](https://github.com/grafana/loki/pull/10101) **owen-d**: Sharding optimizations and fix bug on `<aggr> by|without ()` groupings which removed the grouping while downstreaming
+* [10324](https://github.com/grafana/loki/pull/10324) **ashwanthgoli**: Deprecate ingester.unordered-writes and a few unused configs(log.use-buffered, log.use-sync, frontend.forward-headers-list)
+* [10322](https://github.com/grafana/loki/pull/10322) **chaudum**: Deprecate misleading setting `-ruler.evaluation-delay-duration`.
+* [10295](https://github.com/grafana/loki/pull/10295) **changhyuni**: Storage: remove signatureversionv2 from s3.
 * [10109](https://github.com/grafana/loki/pull/10109) **vardhaman-surana**: Ruler: add limit parameter in rulegroup
+* [10187](https://github.com/grafana/loki/pull/10187) **roelarents**: Add connection-string option for Azure Blob Storage.
 * [9621](https://github.com/grafana/loki/pull/9621) **DylanGuedes**: Introduce TSDB postings cache.
 * [10010](https://github.com/grafana/loki/pull/10010) **rasta-rocket**: feat(promtail): retrieve BotTags field from cloudflare
 * [9995](https://github.com/grafana/loki/pull/9995) **chaudum**: Add jitter to the flush interval to prevent multiple ingesters to flush at the same time.

Makefile

@@ -10,7 +10,7 @@ help:
 .PHONY: docker-driver docker-driver-clean docker-driver-enable docker-driver-push
 .PHONY: fluent-bit-image, fluent-bit-push, fluent-bit-test
 .PHONY: fluentd-image, fluentd-push, fluentd-test
-.PHONY: push-images push-latest save-images load-images promtail-image loki-image build-image
+.PHONY: push-images push-latest save-images load-images promtail-image loki-image build-image build-image-push
 .PHONY: bigtable-backup, push-bigtable-backup
 .PHONY: benchmark-store, drone, check-drone-drift, check-mod
 .PHONY: migrate migrate-image lint-markdown ragel

clients/cmd/fluentd/README.md

@@ -2,7 +2,7 @@
 
 [Fluentd](https://fluentd.org/) is a data collector for unified logging layer, it can be configured with the Loki output plugin, provided in this folder, to ship logs to Loki.
 
-See [docs/client/fluentd/README.md](../../docs/sources/clients/fluentd/_index.md) for detailed information.
+See the [Fluentd documentation](../../../docs/sources/send-data/fluentd/_index.md) for detailed information.
 
 ## Development
 

clients/cmd/promtail/main.go

@@ -153,9 +153,6 @@ func main() {
 	}
 
 	clientMetrics := client.NewMetrics(prometheus.DefaultRegisterer)
-	if config.Options.StreamLagLabels.String() != "" {
-		level.Warn(util_log.Logger).Log("msg", "the stream_lag_labels setting is deprecated and the associated metric has been removed", "stream_lag_labels", config.Options.StreamLagLabels.String())
-	}
 	newConfigFunc := func() (*promtail_config.Config, error) {
 		mtx.Lock()
 		defer mtx.Unlock()

clients/pkg/promtail/client/client.go

@@ -186,9 +186,6 @@ type Tripperware func(http.RoundTripper) http.RoundTripper
 
 // New makes a new Client.
 func New(metrics *Metrics, cfg Config, maxStreams, maxLineSize int, maxLineSizeTruncate bool, logger log.Logger) (Client, error) {
-	if cfg.StreamLagLabels.String() != "" {
-		return nil, fmt.Errorf("client config stream_lag_labels is deprecated and the associated metric has been removed, stream_lag_labels: %+v", cfg.StreamLagLabels.String())
-	}
 	return newClient(metrics, cfg, maxStreams, maxLineSize, maxLineSizeTruncate, logger)
 }
 
@@ -197,6 +194,9 @@ func newClient(metrics *Metrics, cfg Config, maxStreams, maxLineSize int, maxLin
 	if cfg.URL.URL == nil {
 		return nil, errors.New("client needs target URL")
 	}
+	if metrics == nil {
+		return nil, errors.New("metrics must be instantiated")
+	}
 
 	ctx, cancel := context.WithCancel(context.Background())
 
@@ -442,11 +442,10 @@ func (c *client) sendBatch(tenantID string, batch *batch) {
 func (c *client) send(ctx context.Context, tenantID string, buf []byte) (int, error) {
 	ctx, cancel := context.WithTimeout(ctx, c.cfg.Timeout)
 	defer cancel()
-	req, err := http.NewRequest("POST", c.cfg.URL.String(), bytes.NewReader(buf))
+	req, err := http.NewRequestWithContext(ctx, "POST", c.cfg.URL.String(), bytes.NewReader(buf))
 	if err != nil {
 		return -1, err
 	}
-	req = req.WithContext(ctx)
 	req.Header.Set("Content-Type", contentType)
 	req.Header.Set("User-Agent", UserAgent)
 

clients/pkg/promtail/client/config.go

@@ -44,8 +44,6 @@ type Config struct {
 	// 429 'Too Many Requests' response from the distributor. Helps
 	// prevent HOL blocking in multitenant deployments.
 	DropRateLimitedBatches bool `yaml:"drop_rate_limited_batches"`
-
-	StreamLagLabels flagext.StringSliceCSV `yaml:"stream_lag_labels" doc:"deprecated"`
 }
 
 // RegisterFlags with prefix registers flags where every name is prefixed by

clients/pkg/promtail/config/config.go

@@ -6,7 +6,6 @@ import (
 
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
-	dskit_flagext "github.com/grafana/dskit/flagext"
 	"gopkg.in/yaml.v2"
 
 	"github.com/grafana/loki/clients/pkg/promtail/client"
@@ -23,7 +22,6 @@ import (
 
 // Options contains cross-cutting promtail configurations
 type Options struct {
-	StreamLagLabels dskit_flagext.StringSliceCSV `mapstructure:"stream_lag_labels,omitempty" yaml:"stream_lag_labels,omitempty" doc:"deprecated"`
 }
 
 // Config for promtail, describing what files to watch.