Jsonnet: use dedicated configmaps for distributors and ingesters (#775)

* Jsonnet: make dedicated configmap for distributor and ingester * Update CHANGELOG.md * Move distributor config into its own configmap * Update kube-manifests Signed-off-by: Annanay <annanayagarwal@gmail.com> Co-authored-by: Annanay <annanayagarwal@gmail.com>
grafana · Jun 21, 2021 · e446520 · e446520
1 parent b22ebdc
commit e446520
Show file tree

Hide file tree

Showing 15 changed files with 139 additions and 47 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,6 @@
 ## main / unreleased
 
+* [CHANGE] Jsonnet: use dedicated configmaps for distributors and ingesters [#775](https://github.com/grafana/tempo/pull/775) (@kvrhdn)
 * [FEATURE] Added the ability to hedge requests with all backends [#750](https://github.com/grafana/tempo/pull/750) (@joe-elliott)
 * [ENHANCEMENT] Performance: improve compaction speed with concurrent reads and writes [#754](https://github.com/grafana/tempo/pull/754) (@mdisibio)
 * [ENHANCEMENT] Improve readability of cpu and memory metrics on operational dashboard [#764](https://github.com/grafana/tempo/pull/764) (@bboreham)

diff --git a/operations/jsonnet/microservices/configmap.libsonnet b/operations/jsonnet/microservices/configmap.libsonnet
@@ -5,9 +5,7 @@
     server: {
       http_listen_port: $._config.port,
     },
-    distributor: {
-      receivers: $._config.distributor.receivers,
-    },
+    distributor: null,
     ingester: {
       lifecycler: {
         ring: {
@@ -52,6 +50,14 @@
     },
   },
 
+  tempo_distributor_config:: $.tempo_config {
+    distributor: {
+      receivers: $._config.distributor.receivers,
+    },
+  },
+
+  tempo_ingester_config:: $.tempo_config{},
+
   tempo_compactor_config:: $.tempo_config {
     compactor: {
       compaction: {
@@ -90,10 +96,23 @@
     }
   },
 
-  tempo_configmap:
-    configMap.new('tempo') +
+  tempo_query_frontend_config:: $.tempo_config{},
+
+  tempo_distributor_configmap:
+    configMap.new('tempo-distributor') +
     configMap.withData({
-      'tempo.yaml': $.util.manifestYaml($.tempo_config),
+      'tempo.yaml': $.util.manifestYaml($.tempo_distributor_config),
+    }) +
+    configMap.withDataMixin({
+      'overrides.yaml': $.util.manifestYaml({
+        overrides: $._config.overrides,
+      }),
+    }),
+
+  tempo_ingester_configmap:
+    configMap.new('tempo-ingester') +
+    configMap.withData({
+      'tempo.yaml': $.util.manifestYaml($.tempo_ingester_config),
     }) +
     configMap.withDataMixin({
       'overrides.yaml': $.util.manifestYaml({
@@ -112,8 +131,6 @@
       }),
     }),
 
-  tempo_query_frontend_config:: $.tempo_config{},
-
   tempo_querier_configmap:
     configMap.new('tempo-querier') +
     configMap.withData({

diff --git a/operations/jsonnet/microservices/distributor.libsonnet b/operations/jsonnet/microservices/distributor.libsonnet
@@ -40,10 +40,10 @@
     deployment.mixin.spec.strategy.rollingUpdate.withMaxUnavailable(1) +
     deployment.mixin.spec.template.spec.withTerminationGracePeriodSeconds(60) +
     deployment.mixin.spec.template.metadata.withAnnotations({
-      config_hash: std.md5(std.toString($.tempo_configmap.data['tempo.yaml'])),
+      config_hash: std.md5(std.toString($.tempo_distributor_configmap.data['tempo.yaml'])),
     }) +
     deployment.mixin.spec.template.spec.withVolumes([
-      volume.fromConfigMap(tempo_config_volume, $.tempo_configmap.metadata.name),
+      volume.fromConfigMap(tempo_config_volume, $.tempo_distributor_configmap.metadata.name),
     ]),
 
   tempo_distributor_service:

diff --git a/operations/jsonnet/microservices/ingester.libsonnet b/operations/jsonnet/microservices/ingester.libsonnet
@@ -50,8 +50,11 @@
     )
     + $.util.antiAffinityStatefulSet
     + statefulset.mixin.spec.withServiceName(target_name)
+    + statefulset.mixin.spec.template.metadata.withAnnotations({
+      config_hash: std.md5(std.toString($.tempo_ingester_configmap.data['tempo.yaml'])),
+    })
     + statefulset.mixin.spec.template.spec.withVolumes([
-      volume.fromConfigMap(tempo_config_volume, $.tempo_configmap.metadata.name),
+      volume.fromConfigMap(tempo_config_volume, $.tempo_ingester_configmap.metadata.name),
     ]),
 
   tempo_ingester_service:

diff --git a/operations/kube-manifests/ConfigMap-tempo-compactor.yaml b/operations/kube-manifests/ConfigMap-tempo-compactor.yaml
@@ -10,16 +10,7 @@ data:
         ring:
             kvstore:
                 store: memberlist
-    distributor:
-        receivers:
-            jaeger:
-                protocols:
-                    grpc:
-                        endpoint: 0.0.0.0:14250
-            otlp:
-                protocols:
-                    grpc:
-                        endpoint: 0.0.0.0:55680
+    distributor: null
     ingester:
         lifecycler:
             ring:

diff --git a/operations/kube-manifests/ConfigMap-tempo-distributor.yaml b/operations/kube-manifests/ConfigMap-tempo-distributor.yaml
@@ -0,0 +1,52 @@
+apiVersion: v1
+data:
+  overrides.yaml: |
+    overrides: {}
+  tempo.yaml: |
+    compactor: null
+    distributor:
+        receivers:
+            jaeger:
+                protocols:
+                    grpc:
+                        endpoint: 0.0.0.0:14250
+            otlp:
+                protocols:
+                    grpc:
+                        endpoint: 0.0.0.0:55680
+    ingester:
+        lifecycler:
+            ring:
+                replication_factor: 3
+    memberlist:
+        abort_if_cluster_join_fails: false
+        bind_port: 7946
+        join_members:
+          - gossip-ring.tracing.svc.cluster.local:7946
+    overrides:
+        per_tenant_override_config: /conf/overrides.yaml
+    server:
+        http_listen_port: 3100
+    storage:
+        trace:
+            backend: gcs
+            blocklist_poll: "0"
+            cache: memcached
+            gcs:
+                bucket_name: tempo
+                chunk_buffer_size: 1.048576e+07
+            memcached:
+                consistent_hash: true
+                host: memcached
+                service: memcached-client
+                timeout: 500ms
+            pool:
+                queue_depth: 2000
+            s3:
+                bucket: tempo
+            wal:
+                path: /var/tempo/wal
+kind: ConfigMap
+metadata:
+  name: tempo-distributor
+  namespace: tracing
diff --git a/operations/kube-manifests/ConfigMap-tempo-ingester.yaml b/operations/kube-manifests/ConfigMap-tempo-ingester.yaml
@@ -0,0 +1,43 @@
+apiVersion: v1
+data:
+  overrides.yaml: |
+    overrides: {}
+  tempo.yaml: |
+    compactor: null
+    distributor: null
+    ingester:
+        lifecycler:
+            ring:
+                replication_factor: 3
+    memberlist:
+        abort_if_cluster_join_fails: false
+        bind_port: 7946
+        join_members:
+          - gossip-ring.tracing.svc.cluster.local:7946
+    overrides:
+        per_tenant_override_config: /conf/overrides.yaml
+    server:
+        http_listen_port: 3100
+    storage:
+        trace:
+            backend: gcs
+            blocklist_poll: "0"
+            cache: memcached
+            gcs:
+                bucket_name: tempo
+                chunk_buffer_size: 1.048576e+07
+            memcached:
+                consistent_hash: true
+                host: memcached
+                service: memcached-client
+                timeout: 500ms
+            pool:
+                queue_depth: 2000
+            s3:
+                bucket: tempo
+            wal:
+                path: /var/tempo/wal
+kind: ConfigMap
+metadata:
+  name: tempo-ingester
+  namespace: tracing
diff --git a/operations/kube-manifests/ConfigMap-tempo-querier.yaml b/operations/kube-manifests/ConfigMap-tempo-querier.yaml
@@ -2,16 +2,7 @@ apiVersion: v1
 data:
   tempo.yaml: |
     compactor: null
-    distributor:
-        receivers:
-            jaeger:
-                protocols:
-                    grpc:
-                        endpoint: 0.0.0.0:14250
-            otlp:
-                protocols:
-                    grpc:
-                        endpoint: 0.0.0.0:55680
+    distributor: null
     ingester:
         lifecycler:
             ring:

diff --git a/operations/kube-manifests/ConfigMap-tempo-query-frontend.yaml b/operations/kube-manifests/ConfigMap-tempo-query-frontend.yaml
@@ -2,16 +2,7 @@ apiVersion: v1
 data:
   tempo.yaml: |
     compactor: null
-    distributor:
-        receivers:
-            jaeger:
-                protocols:
-                    grpc:
-                        endpoint: 0.0.0.0:14250
-            otlp:
-                protocols:
-                    grpc:
-                        endpoint: 0.0.0.0:55680
+    distributor: null
     ingester:
         lifecycler:
             ring:

diff --git a/operations/kube-manifests/Deployment-compactor.yaml b/operations/kube-manifests/Deployment-compactor.yaml
@@ -18,7 +18,7 @@ spec:
   template:
     metadata:
       annotations:
-        config_hash: 3bc06ce225fc97d13d89fbee2dd6ea25
+        config_hash: 391906d5363b2172f5ef468fcc2fca50
       labels:
         app: compactor
         name: compactor

diff --git a/operations/kube-manifests/Deployment-distributor.yaml b/operations/kube-manifests/Deployment-distributor.yaml
@@ -48,5 +48,5 @@ spec:
       terminationGracePeriodSeconds: 60
       volumes:
       - configMap:
-          name: tempo
+          name: tempo-distributor
         name: tempo-conf
diff --git a/operations/kube-manifests/Deployment-querier.yaml b/operations/kube-manifests/Deployment-querier.yaml
@@ -19,7 +19,7 @@ spec:
   template:
     metadata:
       annotations:
-        config_hash: af35e647b8504eed98436ddd2f35360c
+        config_hash: 5ae9881ddd5abc1d3609268198a0aacd
       labels:
         app: querier
         name: querier

diff --git a/operations/kube-manifests/Deployment-query-frontend.yaml b/operations/kube-manifests/Deployment-query-frontend.yaml
@@ -18,7 +18,7 @@ spec:
   template:
     metadata:
       annotations:
-        config_hash: 76b3caf721a80349e206778e56d41a66
+        config_hash: 215474a01327c5068b4236ec4a327838
       labels:
         app: query-frontend
         name: query-frontend

diff --git a/operations/kube-manifests/StatefulSet-ingester.yaml b/operations/kube-manifests/StatefulSet-ingester.yaml
@@ -14,6 +14,8 @@ spec:
   serviceName: ingester
   template:
     metadata:
+      annotations:
+        config_hash: 215474a01327c5068b4236ec4a327838
       labels:
         app: ingester
         name: ingester
@@ -50,7 +52,7 @@ spec:
           name: ingester-data
       volumes:
       - configMap:
-          name: tempo
+          name: tempo-ingester
         name: tempo-conf
   updateStrategy:
     type: RollingUpdate

diff --git a/operations/kube-manifests/StatefulSet-memcached.yaml b/operations/kube-manifests/StatefulSet-memcached.yaml
@@ -25,7 +25,7 @@ spec:
       - args:
         - -m 1024
         - -I 5m
-        - -c 1024
+        - -c 4096
         - -v
         image: memcached:1.5.17-alpine
         imagePullPolicy: IfNotPresent
@@ -50,3 +50,4 @@ spec:
           name: http-metrics
   updateStrategy:
     type: RollingUpdate
+  volumeClaimTemplates: []