From 4f3ed77cb92c2ffd605743237e609c28f7841728 Mon Sep 17 00:00:00 2001 From: Robert Jacob Date: Mon, 3 Jun 2024 11:02:15 +0200 Subject: [PATCH] fix(operator): Use a minimum value for replay memory ceiling (#13066) --- .../manifests/internal/config/build_test.go | 68 +++++++++---------- .../manifests/internal/config/options.go | 10 +++ 2 files changed, 44 insertions(+), 34 deletions(-) diff --git a/operator/internal/manifests/internal/config/build_test.go b/operator/internal/manifests/internal/config/build_test.go index fa20c45a0226..1d9a0287b398 100644 --- a/operator/internal/manifests/internal/config/build_test.go +++ b/operator/internal/manifests/internal/config/build_test.go @@ -69,7 +69,7 @@ ingester: wal: enabled: true dir: /tmp/wal - replay_memory_ceiling: 2500 + replay_memory_ceiling: 536870912 ingester_client: grpc_client_config: max_recv_msg_size: 67108864 @@ -240,7 +240,7 @@ overrides: }, WriteAheadLog: WriteAheadLog{ Directory: "/tmp/wal", - IngesterMemoryRequest: 5000, + IngesterMemoryRequest: 0, }, ObjectStorage: storage.Options{ SharedStore: lokiv1.ObjectStorageSecretS3, @@ -327,7 +327,7 @@ ingester: wal: enabled: true dir: /tmp/wal - replay_memory_ceiling: 2500 + replay_memory_ceiling: 2147483648 ingester_client: grpc_client_config: max_recv_msg_size: 67108864 @@ -588,7 +588,7 @@ overrides: }, WriteAheadLog: WriteAheadLog{ Directory: "/tmp/wal", - IngesterMemoryRequest: 5000, + IngesterMemoryRequest: 4 * 1024 * 1024 * 1024, }, ObjectStorage: storage.Options{ SharedStore: lokiv1.ObjectStorageSecretS3, @@ -673,7 +673,7 @@ func TestBuild_ConfigAndRuntimeConfig_CreateLokiConfigFailed(t *testing.T) { }, WriteAheadLog: WriteAheadLog{ Directory: "/tmp/wal", - IngesterMemoryRequest: 5000, + IngesterMemoryRequest: 4 * 1024 * 1024 * 1024, }, ObjectStorage: storage.Options{ SharedStore: lokiv1.ObjectStorageSecretS3, @@ -754,7 +754,7 @@ ingester: wal: enabled: true dir: /tmp/wal - replay_memory_ceiling: 2500 + replay_memory_ceiling: 2147483648 ingester_client: grpc_client_config: max_recv_msg_size: 67108864 @@ -1026,7 +1026,7 @@ overrides: }, WriteAheadLog: WriteAheadLog{ Directory: "/tmp/wal", - IngesterMemoryRequest: 5000, + IngesterMemoryRequest: 4 * 1024 * 1024 * 1024, }, ObjectStorage: storage.Options{ SharedStore: lokiv1.ObjectStorageSecretS3, @@ -1113,7 +1113,7 @@ ingester: wal: enabled: true dir: /tmp/wal - replay_memory_ceiling: 2500 + replay_memory_ceiling: 2147483648 ingester_client: grpc_client_config: max_recv_msg_size: 67108864 @@ -1386,7 +1386,7 @@ overrides: }, WriteAheadLog: WriteAheadLog{ Directory: "/tmp/wal", - IngesterMemoryRequest: 5000, + IngesterMemoryRequest: 4 * 1024 * 1024 * 1024, }, ObjectStorage: storage.Options{ SharedStore: lokiv1.ObjectStorageSecretS3, @@ -1473,7 +1473,7 @@ ingester: wal: enabled: true dir: /tmp/wal - replay_memory_ceiling: 2500 + replay_memory_ceiling: 2147483648 ingester_client: grpc_client_config: max_recv_msg_size: 67108864 @@ -1776,7 +1776,7 @@ overrides: }, WriteAheadLog: WriteAheadLog{ Directory: "/tmp/wal", - IngesterMemoryRequest: 5000, + IngesterMemoryRequest: 4 * 1024 * 1024 * 1024, }, ObjectStorage: storage.Options{ SharedStore: lokiv1.ObjectStorageSecretS3, @@ -1866,7 +1866,7 @@ ingester: wal: enabled: true dir: /tmp/wal - replay_memory_ceiling: 2500 + replay_memory_ceiling: 2147483648 ingester_client: grpc_client_config: max_recv_msg_size: 67108864 @@ -2112,7 +2112,7 @@ overrides: }, WriteAheadLog: WriteAheadLog{ Directory: "/tmp/wal", - IngesterMemoryRequest: 5000, + IngesterMemoryRequest: 4 * 1024 * 1024 * 1024, }, ObjectStorage: storage.Options{ SharedStore: lokiv1.ObjectStorageSecretS3, @@ -2202,7 +2202,7 @@ ingester: wal: enabled: true dir: /tmp/wal - replay_memory_ceiling: 2500 + replay_memory_ceiling: 2147483648 ingester_client: grpc_client_config: max_recv_msg_size: 67108864 @@ -2535,7 +2535,7 @@ overrides: }, WriteAheadLog: WriteAheadLog{ Directory: "/tmp/wal", - IngesterMemoryRequest: 5000, + IngesterMemoryRequest: 4 * 1024 * 1024 * 1024, }, ObjectStorage: storage.Options{ SharedStore: lokiv1.ObjectStorageSecretS3, @@ -2635,7 +2635,7 @@ ingester: wal: enabled: true dir: /tmp/wal - replay_memory_ceiling: 2500 + replay_memory_ceiling: 2147483648 ingester_client: grpc_client_config: max_recv_msg_size: 67108864 @@ -2881,7 +2881,7 @@ overrides: }, WriteAheadLog: WriteAheadLog{ Directory: "/tmp/wal", - IngesterMemoryRequest: 5000, + IngesterMemoryRequest: 4 * 1024 * 1024 * 1024, }, ObjectStorage: storage.Options{ SharedStore: lokiv1.ObjectStorageSecretS3, @@ -2967,7 +2967,7 @@ ingester: wal: enabled: true dir: /tmp/wal - replay_memory_ceiling: 2500 + replay_memory_ceiling: 2147483648 ingester_client: grpc_client_config: max_recv_msg_size: 67108864 @@ -3377,7 +3377,7 @@ overrides: }, WriteAheadLog: WriteAheadLog{ Directory: "/tmp/wal", - IngesterMemoryRequest: 5000, + IngesterMemoryRequest: 4 * 1024 * 1024 * 1024, }, ObjectStorage: storage.Options{ SharedStore: lokiv1.ObjectStorageSecretS3, @@ -3465,7 +3465,7 @@ ingester: wal: enabled: true dir: /tmp/wal - replay_memory_ceiling: 2500 + replay_memory_ceiling: 2147483648 ingester_client: grpc_client_config: max_recv_msg_size: 67108864 @@ -3638,7 +3638,7 @@ overrides: }, WriteAheadLog: WriteAheadLog{ Directory: "/tmp/wal", - IngesterMemoryRequest: 5000, + IngesterMemoryRequest: 4 * 1024 * 1024 * 1024, }, ObjectStorage: storage.Options{ SharedStore: lokiv1.ObjectStorageSecretS3, @@ -3727,7 +3727,7 @@ ingester: wal: enabled: true dir: /tmp/wal - replay_memory_ceiling: 2500 + replay_memory_ceiling: 2147483648 ingester_client: grpc_client_config: max_recv_msg_size: 67108864 @@ -3901,7 +3901,7 @@ overrides: }, WriteAheadLog: WriteAheadLog{ Directory: "/tmp/wal", - IngesterMemoryRequest: 5000, + IngesterMemoryRequest: 4 * 1024 * 1024 * 1024, }, ObjectStorage: storage.Options{ SharedStore: lokiv1.ObjectStorageSecretS3, @@ -3990,7 +3990,7 @@ ingester: wal: enabled: true dir: /tmp/wal - replay_memory_ceiling: 2500 + replay_memory_ceiling: 2147483648 ingester_client: grpc_client_config: max_recv_msg_size: 67108864 @@ -4162,7 +4162,7 @@ overrides: }, WriteAheadLog: WriteAheadLog{ Directory: "/tmp/wal", - IngesterMemoryRequest: 5000, + IngesterMemoryRequest: 4 * 1024 * 1024 * 1024, }, ObjectStorage: storage.Options{ SharedStore: lokiv1.ObjectStorageSecretS3, @@ -4254,7 +4254,7 @@ ingester: wal: enabled: true dir: /tmp/wal - replay_memory_ceiling: 2500 + replay_memory_ceiling: 2147483648 ingester_client: grpc_client_config: max_recv_msg_size: 67108864 @@ -4460,7 +4460,7 @@ overrides: }, WriteAheadLog: WriteAheadLog{ Directory: "/tmp/wal", - IngesterMemoryRequest: 5000, + IngesterMemoryRequest: 4 * 1024 * 1024 * 1024, }, ObjectStorage: storage.Options{ SharedStore: lokiv1.ObjectStorageSecretS3, @@ -4554,7 +4554,7 @@ ingester: wal: enabled: true dir: /tmp/wal - replay_memory_ceiling: 2500 + replay_memory_ceiling: 2147483648 ingester_client: grpc_client_config: max_recv_msg_size: 67108864 @@ -4760,7 +4760,7 @@ overrides: }, WriteAheadLog: WriteAheadLog{ Directory: "/tmp/wal", - IngesterMemoryRequest: 5000, + IngesterMemoryRequest: 4 * 1024 * 1024 * 1024, }, ObjectStorage: storage.Options{ SharedStore: lokiv1.ObjectStorageSecretS3, @@ -4852,7 +4852,7 @@ ingester: wal: enabled: true dir: /tmp/wal - replay_memory_ceiling: 2500 + replay_memory_ceiling: 2147483648 ingester_client: grpc_client_config: max_recv_msg_size: 67108864 @@ -5019,7 +5019,7 @@ overrides: }, WriteAheadLog: WriteAheadLog{ Directory: "/tmp/wal", - IngesterMemoryRequest: 5000, + IngesterMemoryRequest: 4 * 1024 * 1024 * 1024, }, ObjectStorage: storage.Options{ SharedStore: lokiv1.ObjectStorageSecretS3, @@ -5109,7 +5109,7 @@ func defaultOptions() Options { }, WriteAheadLog: WriteAheadLog{ Directory: "/tmp/wal", - IngesterMemoryRequest: 5000, + IngesterMemoryRequest: 4 * 1024 * 1024 * 1024, }, ObjectStorage: storage.Options{ SharedStore: lokiv1.ObjectStorageSecretS3, @@ -5343,7 +5343,7 @@ ingester: wal: enabled: true dir: /tmp/wal - replay_memory_ceiling: 2500 + replay_memory_ceiling: 2147483648 ingester_client: grpc_client_config: max_recv_msg_size: 67108864 @@ -5518,7 +5518,7 @@ ingester: wal: enabled: true dir: /tmp/wal - replay_memory_ceiling: 2500 + replay_memory_ceiling: 2147483648 ingester_client: grpc_client_config: max_recv_msg_size: 67108864 diff --git a/operator/internal/manifests/internal/config/options.go b/operator/internal/manifests/internal/config/options.go index 110bce9ee8df..e19d4c34f326 100644 --- a/operator/internal/manifests/internal/config/options.go +++ b/operator/internal/manifests/internal/config/options.go @@ -223,10 +223,20 @@ type WriteAheadLog struct { IngesterMemoryRequest int64 } +const ( + // minimumReplayCeiling contains the minimum value that will be used for the replay_memory_ceiling. + // It is set, so that even when the ingester has a low memory request, the replay will not flush each block + // on its own. + minimumReplayCeiling = 512 * 1024 * 1024 +) + // ReplayMemoryCeiling calculates 50% of the ingester memory // for the ingester to use for the write-ahead-log capbability. func (w WriteAheadLog) ReplayMemoryCeiling() string { value := int64(math.Ceil(float64(w.IngesterMemoryRequest) * float64(0.5))) + if value < minimumReplayCeiling { + value = minimumReplayCeiling + } return fmt.Sprintf("%d", value) }