Merge pull request #306 from sighupio/feat/update-logging-v3.5.0-rc

feat(logging): update logging v4.0.0 rc

.drone.yml

@@ -15,13 +15,13 @@ clone:
 
 steps:
   - name: license-check
-    image: quay.io/sighup/golang:1.21.5
+    image: quay.io/sighup/golang:1.23.3
     pull: always
     commands:
       - make license-check
 
   - name: schema-check
-    image: quay.io/sighup/golang:1.21.5
+    image: quay.io/sighup/golang:1.23.3
     pull: always
     commands:
       - |-
@@ -49,14 +49,14 @@ steps:
       - license-check
       - schema-check
 
-  - name: lint-go
-    image: quay.io/sighup/golang:1.21.5
-    pull: always
-    commands:
-      - make lint-go
-    depends_on:
-      - license-check
-      - schema-check
+  # - name: lint-go
+  #   image: quay.io/sighup/golang:1.23.3
+  #   pull: always
+  #   commands:
+  #     - make lint-go
+  #   depends_on:
+  #     - license-check
+  #     - schema-check
 
   - name: test-schema
     # KUBECTL 1.25.3 - KUSTOMIZE 3.5.3 - HELM 3.1.1 - YQ 4.21.1 - ISTIOCTL 1.9.4 - FURYCTL 0.9.0 - BATS 1.1.0

.tool-versions

@@ -1,7 +1,7 @@
 bats 1.9.0
 drone 1.7.0
-golang 1.21.5
-golangci-lint 1.55.2
+golang 1.23.3
+golangci-lint 1.62.0
 yq 4.33.3
 jq 1.6
 make 4.4.1

Makefile

@@ -52,10 +52,10 @@ lint-go:
 
 .PHONY: tools-go
 tools-go:
-	@go install github.com/evanphx/json-patch/cmd/json-patch@v5.6.0
+	@go install github.com/evanphx/json-patch/v5/cmd/json-patch@v5.9.0
 	@go install github.com/google/addlicense@v1.1.1
-	@go install mvdan.cc/gofumpt@v0.5.0
-	@go install golang.org/x/tools/cmd/goimports@v0.9.3
+	@go install mvdan.cc/gofumpt@v0.7.0
+	@go install golang.org/x/tools/cmd/goimports@v0.26.0
 	@go install github.com/daixiang0/gci@v0.10.1
 	@go install github.com/momaek/formattag@v0.0.9
 	@go install github.com/santhosh-tekuri/jsonschema/cmd/jv@v0.4.0

docs/releases/v1.30.0.md

@@ -34,6 +34,21 @@ The distribution is maintained with ❤️ by the team [SIGHUP](https://sighup.i
 
 ## New features 🌟
 
+- **New option for Logging**: The Loki configuration has been extended to accommodate the new `tsdbStartDate` option to allow a migration towards TSDB and schema v13 (note: **this is a breaking change**):
+
+  ```yaml
+  ...
+    loki:
+      tsdbStartDate: "2024-11-18"
+  ...
+  ```
+
+  - `tsdbStartDate`  (**required**): configures details for the schema config for the purpose of the migration
+
+  `tsdbStartDate` should be a string in `ISO 8601` date format and it represents the day starting from which Loki will record logs with the new store and schema.
+
+  ℹ️ **Note**:  Loki will assume the start of the day on the UTC midnight of the specified day.
+
 - **DR improved configurable schedules**: The schedule configuration has been updated to enhance the usability of schedule customization (note: **this is a breaking change**):
   ```yaml
   ...
@@ -79,6 +94,7 @@ General example to enable Volume Snapshotting on rook-ceph (from our storage add
   deletionPolicy: Retain
   ```
 `deletionPolicy: Retain` is important because if the volume snapshot is deleted from the namespace, the cluster wide volumesnapshotcontent CR will be preserved, maintaining the snapshot on the storage that the cluster is using.
+
 - **DR optional snapshot-controller installation**: To leverage VolumeSnapshots on the OnPremises and KFDDistribution providers, a new option on velero has been added to install the snapshot-controller component. Before activating this parameter make sure that in your cluster there is not another snapshot-controller component deployed. By default this parameter is `false`.
   ```yaml
   ...
@@ -93,9 +109,9 @@ General example to enable Volume Snapshotting on rook-ceph (from our storage add
 
 - **TBD**: TBD.
 
-
 ## Breaking changes 💔
 
+- **Loki store and schema change:** A new store and schema has been introduced in order to improve efficiency, speed and scalability of Loki clusters.
 - **DR Schema change**: A new format for the schedule customization has been introduced to improve the usability. See New Features section for more informations.
 
 ## Upgrade procedure

docs/schemas/ekscluster-kfd-v1alpha2.md

@@ -2381,6 +2381,7 @@ This value defines where the output from Flow will be sent. Will be the `spec` s
 | [backend](#specdistributionmoduleslogginglokibackend)                   | `string` | Optional |
 | [externalEndpoint](#specdistributionmoduleslogginglokiexternalendpoint) | `object` | Optional |
 | [resources](#specdistributionmoduleslogginglokiresources)               | `object` | Optional |
+| [tsdbStartDate](#specdistributionmoduleslogginglokitsdbstartdate)       | `string` | Required |
 
 ## .spec.distribution.modules.logging.loki.backend
 
@@ -2486,6 +2487,16 @@ The cpu request for the prometheus pods
 
 The memory request for the opensearch pods
 
+## .spec.distribution.modules.logging.loki.tsdbStartDate
+
+### Description
+
+Starting from versions 1.28.4, 1.29.5 and 1.30.0 of KFD, Loki will change the time series database from BoltDB to TSDB and the schema from v11 to v13 that it uses to store the logs.
+
+The value of this field will determine the date when Loki will start writing using the new TSDB and the schema v13, always at midnight UTC. The old BoltDB and schema will be kept until they expire for reading purposes.
+
+Value must be a string in `ISO 8601` date format (`yyyy-mm-dd`). Example: `2024-11-18`.
+
 ## .spec.distribution.modules.logging.minio
 
 ### Properties

docs/schemas/kfddistribution-kfd-v1alpha2.md

@@ -1872,6 +1872,7 @@ This value defines where the output from Flow will be sent. Will be the `spec` s
 | [backend](#specdistributionmoduleslogginglokibackend)                   | `string` | Optional |
 | [externalEndpoint](#specdistributionmoduleslogginglokiexternalendpoint) | `object` | Optional |
 | [resources](#specdistributionmoduleslogginglokiresources)               | `object` | Optional |
+| [tsdbStartDate](#specdistributionmoduleslogginglokitsdbstartdate)       | `string` | Required |
 
 ## .spec.distribution.modules.logging.loki.backend
 
@@ -1977,6 +1978,16 @@ The cpu request for the prometheus pods
 
 The memory request for the opensearch pods
 
+## .spec.distribution.modules.logging.loki.tsdbStartDate
+
+### Description
+
+Starting from versions 1.28.4, 1.29.5 and 1.30.0 of KFD, Loki will change the time series database from BoltDB to TSDB and the schema from v11 to v13 that it uses to store the logs.
+
+The value of this field will determine the date when Loki will start writing using the new TSDB and the schema v13, always at midnight UTC. The old BoltDB and schema will be kept until they expire for reading purposes.
+
+Value must be a string in `ISO 8601` date format (`yyyy-mm-dd`). Example: `2024-11-18`.
+
 ## .spec.distribution.modules.logging.minio
 
 ### Properties

docs/schemas/onpremises-kfd-v1alpha2.md

@@ -2082,6 +2082,7 @@ This value defines where the output from the `systemdEtcd` Flow will be sent. Th
 | [backend](#specdistributionmoduleslogginglokibackend)                   | `string` | Optional |
 | [externalEndpoint](#specdistributionmoduleslogginglokiexternalendpoint) | `object` | Optional |
 | [resources](#specdistributionmoduleslogginglokiresources)               | `object` | Optional |
+| [tsdbStartDate](#specdistributionmoduleslogginglokitsdbstartdate)       | `string` | Required |
 
 ### Description
 
@@ -2199,6 +2200,16 @@ The cpu request for the loki pods
 
 The memory request for the prometheus pods
 
+## .spec.distribution.modules.logging.loki.tsdbStartDate
+
+### Description
+
+Starting from versions 1.28.4, 1.29.5 and 1.30.0 of KFD, Loki will change the time series database from BoltDB to TSDB and the schema from v11 to v13 that it uses to store the logs.
+
+The value of this field will determine the date when Loki will start writing using the new TSDB and the schema v13, always at midnight UTC. The old BoltDB and schema will be kept until they expire for reading purposes.
+
+Value must be a string in `ISO 8601` date format (`yyyy-mm-dd`). Example: `2024-11-18`.
+
 ## .spec.distribution.modules.logging.minio
 
 ### Properties

go.mod

@@ -1,11 +1,12 @@
 module github.com/sighupio/fury-distribution
 
-go 1.21
+go 1.23
 
 require (
 	github.com/Al-Pragliola/go-version v1.6.2
 	github.com/go-playground/validator/v10 v10.15.5
-	golang.org/x/exp v0.0.0-20231006140011-7918f672742d
+	github.com/sighupio/go-jsonschema v0.15.2
+	golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc
 )
 
 require (
@@ -15,6 +16,6 @@ require (
 	github.com/leodido/go-urn v1.2.4 // indirect
 	golang.org/x/crypto v0.14.0 // indirect
 	golang.org/x/net v0.17.0 // indirect
-	golang.org/x/sys v0.13.0 // indirect
+	golang.org/x/sys v0.14.0 // indirect
 	golang.org/x/text v0.13.0 // indirect
 )

go.sum

@@ -17,6 +17,8 @@ github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q=
 github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/sighupio/go-jsonschema v0.15.2 h1:Yt+QeiIwL9LZpYH+LwqiDD08FG8vjoyngrpHmfqPmmE=
+github.com/sighupio/go-jsonschema v0.15.2/go.mod h1:3KaIPMGHZhUcDq2b+6rEZgkpT5mpstnsu+KnSbuf/R4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
@@ -26,12 +28,12 @@ github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ
 github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
 golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
-golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
-golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
+golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc h1:ao2WRsKSzW6KuUY9IWPwWahcHCgR0s52IfwutMfEbdM=
+golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI=
 golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
 golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
-golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
-golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
+golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
 golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

kfd.yaml

@@ -8,7 +8,7 @@ modules:
   aws: v4.2.1
   dr: v3.0.0-rc.1
   ingress: v3.0.1-rc.1
-  logging: v3.4.1
+  logging: v4.0.0-rc.0
   monitoring: v3.3.0-rc.1
   opa: v1.13.0
   networking: v2.0.0-rc.1

pkg/apis/config/validation_test.go

@@ -32,8 +32,6 @@ func TestValidateAwsRegion(t *testing.T) {
 		},
 	}
 	for _, tC := range testCases {
-		tC := tC
-
 		t.Run(tC.desc, func(t *testing.T) {
 			t.Parallel()