Merge branch 'main' into patch-1

.github/workflows/backport.yml

@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout Actions
-        uses: actions/checkout@v3.3.0
+        uses: actions/checkout@v3
         with:
           repository: "grafana/grafana-github-actions"
           path: ./actions

.github/workflows/helm-ci.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout Code
-        uses: actions/checkout@v3.3.0
+        uses: actions/checkout@v3
 
       - name: Check Docs
         run: |
@@ -43,7 +43,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3.3.0
+        uses: actions/checkout@v3
         with:
           fetch-depth: 0
 

.github/workflows/issue_commands.yml

@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout Actions
-        uses: actions/checkout@v3.3.0
+        uses: actions/checkout@v3
         with:
           repository: "grafana/grafana-github-actions"
           path: ./actions

.github/workflows/metrics-collector.yml

@@ -8,7 +8,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout Actions
-        uses: actions/checkout@v3.3.0
+        uses: actions/checkout@v3
         with:
           repository: "grafana/grafana-github-actions"
           path: ./actions

.github/workflows/nix-ci.yaml

@@ -9,7 +9,7 @@ jobs:
   tests:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3.3.0
+      - uses: actions/checkout@v3
       - uses: cachix/install-nix-action@v18
         with:
           nix_path: nixpkgs=channel:nixos-unstable

.github/workflows/operator-bundle.yaml

@@ -23,7 +23,7 @@ jobs:
       with:
         go-version: ${{ matrix.go }}
       id: go
-    - uses: actions/checkout@v3.3.0
+    - uses: actions/checkout@v3
     - name: Install make
       run: sudo apt-get install make
     - name: make bundle

.github/workflows/operator-images.yaml

@@ -18,7 +18,7 @@ jobs:
  publish-manager:
    runs-on: ubuntu-latest
    steps:
-     - uses: actions/checkout@v3.3.0
+     - uses: actions/checkout@v3
 
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v1
@@ -56,7 +56,7 @@ jobs:
  publish-bundle:
    runs-on: ubuntu-latest
    steps:
-     - uses: actions/checkout@v3.3.0
+     - uses: actions/checkout@v3
 
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v1
@@ -95,7 +95,7 @@ jobs:
  publish-size-calculator:
    runs-on: ubuntu-latest
    steps:
-     - uses: actions/checkout@v3.3.0
+     - uses: actions/checkout@v3
 
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v1

.github/workflows/operator-scorecard.yaml

@@ -26,7 +26,7 @@ jobs:
     - uses: engineerd/setup-kind@v0.5.0
       with:
         version: "v0.17.0"
-    - uses: actions/checkout@v3.3.0
+    - uses: actions/checkout@v3
     - name: Install make
       run: sudo apt-get install make
     - name: Run scorecard

.github/workflows/operator.yaml

@@ -25,7 +25,7 @@ jobs:
       with:
         go-version: ${{ matrix.go }}
       id: go
-    - uses: actions/checkout@v3.3.0
+    - uses: actions/checkout@v3
     - name: Lint
       uses: golangci/golangci-lint-action@v3.4.0
       with:
@@ -51,7 +51,7 @@ jobs:
       with:
         go-version: ${{ matrix.go }}
       id: go
-    - uses: actions/checkout@v3.3.0
+    - uses: actions/checkout@v3
     - name: Build Manager
       working-directory: ./operator
       run: |-
@@ -72,7 +72,7 @@ jobs:
       with:
         go-version: ${{ matrix.go }}
       id: go
-    - uses: actions/checkout@v3.3.0
+    - uses: actions/checkout@v3
     - name: Build Broker
       working-directory: ./operator
       run: |-
@@ -93,7 +93,7 @@ jobs:
       with:
         go-version: ${{ matrix.go }}
       id: go
-    - uses: actions/checkout@v3.3.0
+    - uses: actions/checkout@v3
     - name: Run tests
       working-directory: ./operator
       run: go test -coverprofile=profile.cov ./...

.github/workflows/publish-technical-documentation-next.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: "ubuntu-latest"
     steps:
       - name: "Check out code"
-        uses: "actions/checkout@v3.3.0"
+        uses: "actions/checkout@v3"
       - name: "Build website"
         # -e HUGO_REFLINKSERRORLEVEL=ERROR prevents merging broken refs with the downside
         # that no refs to external content can be used as these refs will not resolve in the
@@ -25,7 +25,7 @@ jobs:
     needs: "test"
     steps:
       - name: "Check out code"
-        uses: "actions/checkout@v3.3.0"
+        uses: "actions/checkout@v3"
 
       - name: "Clone website-sync Action"
         # WEBSITE_SYNC_TOKEN is a fine-grained GitHub Personal Access Token that expires.

.github/workflows/publish-technical-documentation-release.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: "ubuntu-latest"
     steps:
       - name: "Check out code"
-        uses: "actions/checkout@v3.3.0"
+        uses: "actions/checkout@v3"
       - name:
           "Build website"
           # -e HUGO_REFLINKSERRORLEVEL=ERROR prevents merging broken refs with the downside
@@ -28,12 +28,12 @@ jobs:
     needs: "test"
     steps:
       - name: "Checkout code and tags"
-        uses: "actions/checkout@v3.3.0"
+        uses: "actions/checkout@v3"
         with:
           fetch-depth: 0
 
       - name: "Checkout Actions library"
-        uses: "actions/checkout@v3.3.0"
+        uses: "actions/checkout@v3"
         with:
           repository: "grafana/grafana-github-actions"
           path: "./actions"

.github/workflows/syft-sbom-ci.yml

@@ -11,10 +11,10 @@ jobs:
 
     steps:
     - name: Checkout
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
 
     - name: Anchore SBOM Action
-      uses: anchore/sbom-action@v0.12.0
+      uses: anchore/sbom-action@v0.14.1
       with:
          artifact-name: ${{ github.event.repository.name }}-spdx.json
 

.gitignore

@@ -51,3 +51,4 @@ pkg/loki/wal
 
 # nix
 result
+/docs/make-docs

Makefile

@@ -131,13 +131,16 @@ check-generated-files: yacc ragel fmt-proto protos clients/pkg/promtail/server/u
 ##########
 .PHONY: cmd/logcli/logcli
 logcli: cmd/logcli/logcli
+logcli-debug: cmd/logcli/logcli-debug
 
 logcli-image:
 	$(SUDO) docker build -t $(IMAGE_PREFIX)/logcli:$(IMAGE_TAG) -f cmd/logcli/Dockerfile .
 
 cmd/logcli/logcli:
 	CGO_ENABLED=0 go build $(GO_FLAGS) -o $@ ./cmd/logcli
 
+cmd/logcli/logcli-debug:
+	CGO_ENABLED=0 go build $(DEBUG_GO_FLAGS) -o ./cmd/logcli/logcli-debug ./cmd/logcli
 ########
 # Loki #
 ########

clients/pkg/promtail/scrapeconfig/scrapeconfig.go

@@ -407,6 +407,10 @@ type GcplogTargetConfig struct {
 
 	// Server is the weaveworks server config for listening connections. Used just for `push` subscription type.
 	Server server.Config `yaml:"server"`
+
+	// UseFullLine force Promtail to send the full line from Cloud Logging even if `textPayload` is available.
+	// By default, if `textPayload` is present in the line, then it's used as log line.
+	UseFullLine bool `yaml:"use_full_line"`
 }
 
 // HerokuDrainTargetConfig describes a scrape config to listen and consume heroku logs, in the HTTPS drain manner.

clients/pkg/promtail/targets/gcplog/formatter.go

@@ -13,10 +13,9 @@ import (
 	"github.com/grafana/loki/clients/pkg/promtail/api"
 
 	"github.com/grafana/loki/pkg/logproto"
-	"github.com/grafana/loki/pkg/util"
 )
 
-// LogEntry that will be written to the pubsub topic.
+// GCPLogEntry that will be written to the pubsub topic.
 // According to the following spec.
 // https://cloud.google.com/logging/docs/reference/v2/rest/v2/LogEntry
 // nolint:revive
@@ -32,13 +31,22 @@ type GCPLogEntry struct {
 	// Its important that `Timestamp` is optional in GCE log entry.
 	ReceiveTimestamp string `json:"receiveTimestamp"`
 
+	// Optional. The severity of the log entry. The default value is DEFAULT.
+	// DEFAULT, DEBUG, INFO, NOTICE, WARNING, ERROR, CRITICAL, ALERT, EMERGENCY
+	// https://cloud.google.com/logging/docs/reference/v2/rest/v2/LogEntry#LogSeverity
+	Severity string `json:"severity"`
+
+	// Optional. A map of key, value pairs that provides additional information about the log entry.
+	// The labels can be user-defined or system-defined.
+	Labels map[string]string `json:"labels"`
+
 	TextPayload string `json:"textPayload"`
 
 	// NOTE(kavi): There are other fields on GCPLogEntry. but we need only need above fields for now
 	// anyway we will be sending the entire entry to Loki.
 }
 
-func parseGCPLogsEntry(data []byte, other model.LabelSet, otherInternal labels.Labels, useIncomingTimestamp bool, relabelConfig []*relabel.Config) (api.Entry, error) {
+func parseGCPLogsEntry(data []byte, other model.LabelSet, otherInternal labels.Labels, useIncomingTimestamp, useFullLine bool, relabelConfig []*relabel.Config) (api.Entry, error) {
 	var ge GCPLogEntry
 
 	if err := json.Unmarshal(data, &ge); err != nil {
@@ -50,10 +58,16 @@ func parseGCPLogsEntry(data []byte, other model.LabelSet, otherInternal labels.L
 	lbs := labels.NewBuilder(otherInternal)
 	lbs.Set("__gcp_logname", ge.LogName)
 	lbs.Set("__gcp_resource_type", ge.Resource.Type)
+	lbs.Set("__gcp_severity", ge.Severity)
 
-	// labels from gcp log entry. Add it as internal labels
+	// resource labels from gcp log entry. Add it as internal labels
 	for k, v := range ge.Resource.Labels {
-		lbs.Set("__gcp_resource_labels_"+util.SnakeCase(k), v)
+		lbs.Set("__gcp_resource_labels_"+convertToLokiCompatibleLabel(k), v)
+	}
+
+	// labels from gcp log entry. Add it as internal labels
+	for k, v := range ge.Labels {
+		lbs.Set("__gcp_labels_"+convertToLokiCompatibleLabel(k), v)
 	}
 
 	var processed labels.Labels
@@ -101,8 +115,8 @@ func parseGCPLogsEntry(data []byte, other model.LabelSet, otherInternal labels.L
 		}
 	}
 
-	// Send only `ge.textPaylload` as log line if its present.
-	if strings.TrimSpace(ge.TextPayload) != "" {
+	// Send only `ge.textPayload` as log line if its present and user don't explicitly ask for the whole log.
+	if !useFullLine && strings.TrimSpace(ge.TextPayload) != "" {
 		line = ge.TextPayload
 	}
 

clients/pkg/promtail/targets/gcplog/formatter_test.go

@@ -22,6 +22,7 @@ func TestFormat(t *testing.T) {
 		labels               model.LabelSet
 		relabel              []*relabel.Config
 		useIncomingTimestamp bool
+		useFullLine          bool
 		expected             api.Entry
 	}{
 		{
@@ -49,13 +50,31 @@ func TestFormat(t *testing.T) {
 					Action:       "replace",
 					Replacement:  "$1",
 				},
+				{
+					SourceLabels: model.LabelNames{"__gcp_severity"},
+					Separator:    ";",
+					Regex:        relabel.MustNewRegexp("(.*)"),
+					TargetLabel:  "severity",
+					Action:       "replace",
+					Replacement:  "$1",
+				},
+				{
+					SourceLabels: model.LabelNames{"__gcp_labels_dataflow_googleapis_com_region"},
+					Separator:    ";",
+					Regex:        relabel.MustNewRegexp("(.*)"),
+					TargetLabel:  "region",
+					Action:       "replace",
+					Replacement:  "$1",
+				},
 			},
 			useIncomingTimestamp: true,
 			expected: api.Entry{
 				Labels: model.LabelSet{
 					"jobname":              "pubsub-test",
 					"backend_service_name": "http-loki",
 					"bucket_name":          "loki-bucket",
+					"severity":             "INFO",
+					"region":               "europe-west1",
 				},
 				Entry: logproto.Entry{
 					Timestamp: mustTime(t, "2020-12-22T15:01:23.045123456Z"),
@@ -100,11 +119,48 @@ func TestFormat(t *testing.T) {
 				},
 			},
 		},
+		{
+			name:        "use-full-line",
+			useFullLine: true,
+			msg: &pubsub.Message{
+				Data: []byte(withTextPayload),
+			},
+			labels: model.LabelSet{
+				"jobname": "pubsub-test",
+			},
+			expected: api.Entry{
+				Labels: model.LabelSet{
+					"jobname": "pubsub-test",
+				},
+				Entry: logproto.Entry{
+					Timestamp: time.Now(),
+					Line:      withTextPayload,
+				},
+			},
+		},
+		{
+			name: "use-text-payload",
+			msg: &pubsub.Message{
+				Data: []byte(withTextPayload),
+			},
+			labels: model.LabelSet{
+				"jobname": "pubsub-test",
+			},
+			expected: api.Entry{
+				Labels: model.LabelSet{
+					"jobname": "pubsub-test",
+				},
+				Entry: logproto.Entry{
+					Timestamp: time.Now(),
+					Line:      logTextPayload,
+				},
+			},
+		},
 	}
 
 	for _, c := range cases {
 		t.Run(c.name, func(t *testing.T) {
-			got, err := parseGCPLogsEntry(c.msg.Data, c.labels, nil, c.useIncomingTimestamp, c.relabel)
+			got, err := parseGCPLogsEntry(c.msg.Data, c.labels, nil, c.useIncomingTimestamp, c.useFullLine, c.relabel)
 
 			require.NoError(t, err)
 
@@ -130,5 +186,7 @@ func mustTime(t *testing.T, v string) time.Time {
 }
 
 const (
-	withAllFields = `{"logName": "https://project/gcs", "resource": {"type": "gcs", "labels": {"backendServiceName": "http-loki", "bucketName": "loki-bucket", "instanceId": "344555"}}, "timestamp": "2020-12-22T15:01:23.045123456Z"}`
+	withAllFields   = `{"logName": "https://project/gcs", "severity": "INFO", "resource": {"type": "gcs", "labels": {"backendServiceName": "http-loki", "bucketName": "loki-bucket", "instanceId": "344555"}}, "timestamp": "2020-12-22T15:01:23.045123456Z", "labels": {"dataflow.googleapis.com/region": "europe-west1"}}`
+	logTextPayload  = "text-payload-log"
+	withTextPayload = `{"logName": "https://project/gcs", "severity": "INFO", "textPayload": "` + logTextPayload + `", "resource": {"type": "gcs", "labels": {"backendServiceName": "http-loki", "bucketName": "loki-bucket", "instanceId": "344555"}}, "timestamp": "2020-12-22T15:01:23.045123456Z", "labels": {"dataflow.googleapis.com/region": "europe-west1"}}`
 )

clients/pkg/promtail/targets/gcplog/pull_target.go

@@ -108,7 +108,7 @@ func (t *pullTarget) run() error {
 		case <-t.ctx.Done():
 			return t.ctx.Err()
 		case m := <-t.msgs:
-			entry, err := parseGCPLogsEntry(m.Data, t.config.Labels, nil, t.config.UseIncomingTimestamp, t.relabelConfig)
+			entry, err := parseGCPLogsEntry(m.Data, t.config.Labels, nil, t.config.UseIncomingTimestamp, t.config.UseFullLine, t.relabelConfig)
 			if err != nil {
 				level.Error(t.logger).Log("event", "error formating log entry", "cause", err)
 				m.Ack()