Memberlist related jsonnet (#6253)

* Add memberlist.libsonnet and modify loki services so we can set the port when using memberlist for the ring. Signed-off-by: Callum Styan <callumstyan@gmail.com> * Missed the + for the ruler ring config here. Signed-off-by: Callum Styan <callumstyan@gmail.com> * Fix linting. Signed-off-by: Callum Styan <callumstyan@gmail.com>
grafana · May 30, 2022 · 0cb3994 · 0cb3994
1 parent 51dabcf
commit 0cb3994
Show file tree

Hide file tree

Showing 7 changed files with 162 additions and 15 deletions.
diff --git a/production/ksonnet/loki/boltdb_shipper.libsonnet b/production/ksonnet/loki/boltdb_shipper.libsonnet
@@ -49,14 +49,13 @@
     target: 'compactor',
   } else {},
 
-  local compactor_ports =
-    [
-      containerPort.new(name='http-metrics', port=$._config.http_listen_port),
-    ],
+  compactor_ports: [
+    containerPort.new(name='http-metrics', port=$._config.http_listen_port),
+  ],
 
   compactor_container:: if $._config.using_boltdb_shipper then
     container.new('compactor', $._images.compactor) +
-    container.withPorts(compactor_ports) +
+    container.withPorts($.compactor_ports) +
     container.withArgsMixin(k.util.mapToFlags($.compactor_args)) +
     container.withVolumeMountsMixin([volumeMount.new('compactor-data', '/data')]) +
     container.mixin.readinessProbe.httpGet.withPath('/ready') +

diff --git a/production/ksonnet/loki/distributor.libsonnet b/production/ksonnet/loki/distributor.libsonnet
@@ -8,9 +8,11 @@ local k = import 'ksonnet-util/kausal.libsonnet';
       target: 'distributor',
     },
 
+  distributor_ports:: $.util.defaultPorts,
+
   distributor_container::
     container.new('distributor', $._images.distributor) +
-    container.withPorts($.util.defaultPorts) +
+    container.withPorts($.distributor_ports) +
     container.withArgsMixin(k.util.mapToFlags($.distributor_args)) +
     container.mixin.readinessProbe.httpGet.withPath('/ready') +
     container.mixin.readinessProbe.httpGet.withPort($._config.http_listen_port) +

diff --git a/production/ksonnet/loki/index-gateway.libsonnet b/production/ksonnet/loki/index-gateway.libsonnet
@@ -34,15 +34,14 @@
     target: 'index-gateway',
   } else {},
 
-  local index_gateway_ports =
-    [
-      containerPort.new(name='http-metrics', port=$._config.http_listen_port),
-      containerPort.new(name='grpc', port=9095),
-    ],
+  index_gateway_ports: [
+    containerPort.new(name='http-metrics', port=$._config.http_listen_port),
+    containerPort.new(name='grpc', port=9095),
+  ],
 
   index_gateway_container:: if $._config.use_index_gateway then
     container.new('index-gateway', $._images.index_gateway) +
-    container.withPorts(index_gateway_ports) +
+    container.withPorts($.index_gateway_ports) +
     container.withArgsMixin(k.util.mapToFlags($.index_gateway_args)) +
     container.withVolumeMountsMixin([volumeMount.new('index-gateway-data', '/data')]) +
     container.mixin.readinessProbe.httpGet.withPath('/ready') +

diff --git a/production/ksonnet/loki/ingester.libsonnet b/production/ksonnet/loki/ingester.libsonnet
@@ -15,9 +15,11 @@ local k = import 'ksonnet-util/kausal.libsonnet';
         'ingester.max-transfer-retries': 0,
       } else {},
 
+  ingester_ports: $.util.defaultPorts,
+
   ingester_container::
     container.new('ingester', $._images.ingester) +
-    container.withPorts($.util.defaultPorts) +
+    container.withPorts($.ingester_ports) +
     container.withArgsMixin(k.util.mapToFlags($.ingester_args)) +
     container.mixin.readinessProbe.httpGet.withPath('/ready') +
     container.mixin.readinessProbe.httpGet.withPort($._config.http_listen_port) +

diff --git a/production/ksonnet/loki/memberlist.libsonnet b/production/ksonnet/loki/memberlist.libsonnet
@@ -0,0 +1,141 @@
+{
+  local memberlistConfig = {
+    'memberlist.abort-if-join-fails': false,
+    'memberlist.bind-port': gossipRingPort,
+    'memberlist.join': 'gossip-ring.%s.svc.cluster.local:%d' % [$._config.namespace, gossipRingPort],
+  },
+
+  local setupGossipRing(storeOption, consulHostnameOption, multiStoreOptionsPrefix) = if $._config.multikv_migration_enabled then {
+    [storeOption]: 'multi',
+    [multiStoreOptionsPrefix + '.primary']: $._config.multikv_primary,
+    [multiStoreOptionsPrefix + '.secondary']: $._config.multikv_secondary,
+    // don't remove consul.hostname, it may still be needed.
+  } else {
+    [storeOption]: 'memberlist',
+    [consulHostnameOption]: null,
+  },
+
+  _config+:: {
+    gossip_member_label: 'loki_gossip_member',
+    // Enables use of memberlist for all rings, instead of consul. If multikv_migration_enabled is true, consul hostname is still configured,
+    // but "primary" KV depends on value of multikv_primary.
+    memberlist_ring_enabled: false,
+
+    // Migrating from consul to memberlist is a multi-step process:
+    // 1) Enable multikv_migration_enabled, with primary=consul, secondary=memberlist, and multikv_mirror_enabled=false, restart components.
+    // 2) Set multikv_mirror_enabled=true. This doesn't require restart.
+    // 3) Swap multikv_primary and multikv_secondary, ie. multikv_primary=memberlist, multikv_secondary=consul. This doesn't require restart.
+    // 4) Set multikv_migration_enabled=false and multikv_migration_teardown=true. This requires a restart, but components will now use only memberlist.
+    // 5) Set multikv_migration_teardown=false. This doesn't require a restart.
+    multikv_migration_enabled: false,
+    multikv_migration_teardown: false,
+    multikv_primary: 'consul',
+    multikv_secondary: 'memberlist',
+    multikv_switch_primary_secondary: false,
+    multikv_mirror_enabled: false,
+
+    multi_kv_obj: if $._config.multikv_migration_enabled then {
+      store: 'multi',
+      multi: {
+        primary: $._config.multikv_primary,
+        secondary: $._config.multikv_secondary,
+      },
+    } else {
+      store: 'memberlist',
+      consul: {
+        host: null,
+      },
+    },
+
+    loki+: {
+      ingester+: {
+        lifecycler+: {
+          ring+: {
+            kvstore+: $._config.multi_kv_obj,
+          },
+        },
+      },
+
+      distributor+: {
+        ring+: {
+          kvstore+: $._config.multi_kv_obj,
+        },
+      },
+
+      ruler+: if !$._config.ruler_enabled then {} else {
+        ring+: {
+          kvstore+: $._config.multi_kv_obj,
+        },
+      },
+
+      // other components pick this up from common
+      common+: {
+        ring+: {
+          kvstore+: $._config.multi_kv_obj,
+        },
+      },
+
+      memberlist: {
+        abort_if_cluster_join_fails: false,
+
+        // Expose this port on all distributor, ingester
+        // and querier replicas.
+        bind_port: gossipRingPort,
+
+        // You can use a headless k8s service for all distributor,
+        // ingester and querier components.
+        join_members: ['gossip-ring.%s.svc.cluster.local:%d' % [$._config.namespace, gossipRingPort]],
+
+        max_join_backoff: '1m',
+        max_join_retries: '10',
+        min_join_backoff: '1s',
+      },
+    },
+
+    // When doing migration via multi KV store, this section can be used
+    // to configure runtime parameters of multi KV store
+    multi_kv_config: if !$._config.multikv_migration_enabled && !$._config.multikv_migration_teardown then {} else {
+      primary: if $._config.multikv_switch_primary_secondary then $._config.multikv_secondary else $._config.multikv_primary,
+      mirror_enabled: $._config.multikv_mirror_enabled,
+    },
+  },
+
+  local gossipRingPort = 7946,
+
+  local containerPort = $.core.v1.containerPort,
+  local gossipPort = containerPort.newNamed(name='gossip-ring', containerPort=gossipRingPort),
+
+  compactor_ports+:: if !$._config.memberlist_ring_enabled then [] else [gossipPort],
+  distributor_ports+:: if !$._config.memberlist_ring_enabled then [] else [gossipPort],
+  index_gateway_ports+:: if !$._config.memberlist_ring_enabled then [] else [gossipPort],
+  ingester_ports+:: if !$._config.memberlist_ring_enabled then [] else [gossipPort],
+  query_scheduler_ports+:: if !$._config.memberlist_ring_enabled then [] else [gossipPort],
+  ruler_ports+:: if !$._config.memberlist_ring_enabled then [] else [gossipPort],
+
+  // Don't add label to matcher, only to pod labels.
+  local gossipLabel = $.apps.v1.statefulSet.spec.template.metadata.withLabelsMixin({ [$._config.gossip_member_label]: 'true' }),
+
+  compactor_statefulset+: if !$._config.memberlist_ring_enabled then {} else gossipLabel,
+  distributor_deployment+: if !$._config.memberlist_ring_enabled then {} else gossipLabel,
+  index_gateway_statefulset+: if !$._config.memberlist_ring_enabled then {} else gossipLabel,
+  ingester_statefulset+: if !$._config.memberlist_ring_enabled then {} else gossipLabel,
+  query_scheduler_deployment+: if !$._config.memberlist_ring_enabled then {} else gossipLabel,
+  ruler_deployment+: if !$._config.memberlist_ring_enabled || !$._config.ruler_enabled then {} else gossipLabel,
+
+  // Headless service (= no assigned IP, DNS returns all targets instead) pointing to gossip network members.
+  gossip_ring_service:
+    if !$._config.memberlist_ring_enabled then null
+    else
+      local service = $.core.v1.service;
+      local servicePort = $.core.v1.servicePort;
+
+      local ports = [
+        servicePort.newNamed('gossip-ring', gossipRingPort, gossipRingPort) +
+        servicePort.withProtocol('TCP'),
+      ];
+      service.new(
+        'gossip-ring',  // name
+        { [$._config.gossip_member_label]: 'true' },  // point to all gossip members
+        ports,
+      ) + service.mixin.spec.withClusterIp('None'),  // headless service
+}
diff --git a/production/ksonnet/loki/query-scheduler.libsonnet b/production/ksonnet/loki/query-scheduler.libsonnet
@@ -32,10 +32,12 @@ local k = import 'ksonnet-util/kausal.libsonnet';
     }
   else {},
 
+  query_scheduler_ports:: $.util.grpclbDefaultPorts,
+
   local container = k.core.v1.container,
   query_scheduler_container:: if $._config.query_scheduler_enabled then
     container.new('query-scheduler', $._images.query_scheduler) +
-    container.withPorts($.util.grpclbDefaultPorts) +
+    container.withPorts($.query_scheduler_ports) +
     container.withArgsMixin(k.util.mapToFlags($.query_scheduler_args)) +
     $.jaeger_mixin +
     k.util.resourcesRequests('2', '600Mi') +

diff --git a/production/ksonnet/loki/ruler.libsonnet b/production/ksonnet/loki/ruler.libsonnet
@@ -19,10 +19,12 @@ local k = import 'ksonnet-util/kausal.libsonnet';
     stateful_rulers: if self.using_boltdb_shipper && !self.use_index_gateway then true else super.stateful_rulers,
   },
 
+  ruler_ports:: $.util.defaultPorts,
+
   ruler_container::
     if $._config.ruler_enabled then
       container.new('ruler', $._images.ruler) +
-      container.withPorts($.util.defaultPorts) +
+      container.withPorts($.ruler_ports) +
       container.withArgsMixin(k.util.mapToFlags($.ruler_args)) +
       k.util.resourcesRequests('1', '6Gi') +
       k.util.resourcesLimits('16', '16Gi') +