Merge pull request #1650 from ArthurSens/as/network-policies

Adds NetworkPolicies to all components of Kube-prometheus
prometheus-operator · Apr 5, 2022 · 01004de · 01004de
2 parents a182d78 + 3ad0867
commit 01004de
Show file tree

Hide file tree

Showing 24 changed files with 577 additions and 7 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -97,7 +97,25 @@ jobs:
       with:
         version: ${{ env.kind-version }}
         image: ${{ matrix.kind-image }}
-        wait: 300s
+        wait: 10s # Without default CNI, control-plane doesn't get ready until Cilium is installed
+        config: .github/workflows/kind/config.yml
+    - name: Setup Helm
+      uses: azure/setup-helm@v1
+    - name: Install Cilium
+      run: |
+        helm repo add cilium https://helm.cilium.io/
+        helm install cilium cilium/cilium --version 1.9.13 \
+        --namespace kube-system \
+        --set nodeinit.enabled=true \
+        --set kubeProxyReplacement=partial \
+        --set hostServices.enabled=false \
+        --set externalIPs.enabled=true \
+        --set nodePort.enabled=true \
+        --set hostPort.enabled=true \
+        --set bpf.masquerade=false \
+        --set image.pullPolicy=IfNotPresent \
+        --set ipam.mode=kubernetes \
+        --set operator.replicas=1
     - name: Wait for cluster to finish bootstraping
       run: kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout=300s
     - name: Create kube-prometheus stack

diff --git a/.github/workflows/kind/config.yml b/.github/workflows/kind/config.yml
@@ -0,0 +1,6 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+networking:
+  disableDefaultCNI: true
+  podSubnet: "10.10.0.0/16"
+  serviceSubnet: "10.11.0.0/16"
diff --git a/.gitignore b/.gitignore
@@ -6,4 +6,4 @@ vendor/
 crdschemas/
 
 developer-workspace/gitpod/_output
-kind
+developer-workspace/codespaces/kind 
diff --git a/Makefile b/Makefile
@@ -17,7 +17,7 @@ JSONNETFMT_ARGS=-n 2 --max-blank-lines 2 --string-style s --comment-style s
 MDOX_VALIDATE_CONFIG?=.mdox.validate.yaml
 MD_FILES_TO_FORMAT=$(shell find docs developer-workspace examples experimental jsonnet manifests -name "*.md") $(shell ls *.md)
 
-KUBESCAPE_THRESHOLD=9
+KUBESCAPE_THRESHOLD=1
 
 all: generate fmt test docs
 

diff --git a/developer-workspace/codespaces/prepare-kind.sh b/developer-workspace/codespaces/prepare-kind.sh
@@ -9,12 +9,27 @@ if [[ $? != 0 ]]; then
     | cut -d : -f 2,3 \
     | tr -d \" \
     | wget -qi -
-    mv kind-linux-amd64 kind && chmod +x kind
+    mv kind-linux-amd64 developer-workspace/codespaces/kind && chmod +x developer-workspace/codespaces/kind
+    export PATH=$PATH:$PWD/developer-workspace/codespaces
 fi
 
-cluster_created=$($PWD/kind get clusters 2>&1)
+cluster_created=$($PWD/developer-workspace/codespaces/kind get clusters 2>&1)
 if [[ "$cluster_created" == "No kind clusters found." ]]; then 
-    $PWD/kind create cluster
+    $PWD/developer-workspace/codespaces/kind create cluster --config $PWD/.github/workflows/kind/config.yml
 else
     echo "Cluster '$cluster_created' already present" 
-fi
+fi
+
+helm repo add --force-update cilium https://helm.cilium.io/ 
+helm install cilium cilium/cilium --version 1.9.13 \
+  --namespace kube-system \
+  --set nodeinit.enabled=true \
+  --set kubeProxyReplacement=partial \
+  --set hostServices.enabled=false \
+  --set externalIPs.enabled=true \
+  --set nodePort.enabled=true \
+  --set hostPort.enabled=true \
+  --set bpf.masquerade=false \
+  --set image.pullPolicy=IfNotPresent \
+  --set ipam.mode=kubernetes \
+  --set operator.replicas=1
diff --git a/examples/networkpolicies-disabled.jsonnet b/examples/networkpolicies-disabled.jsonnet
@@ -0,0 +1,25 @@
+local kp = (import 'kube-prometheus/main.libsonnet') +
+           (import 'kube-prometheus/addons/networkpolicies-disabled.libsonnet') + {
+  values+:: {
+    common+: {
+      namespace: 'monitoring',
+    },
+  },
+};
+
+{
+  ['setup/' + resource]: kp[component][resource]
+  for component in std.objectFields(kp)
+  for resource in std.filter(
+    function(resource)
+      kp[component][resource].kind == 'CustomResourceDefinition' || kp[component][resource].kind == 'Namespace', std.objectFields(kp[component])
+  )
+} +
+{
+  [component + '-' + resource]: kp[component][resource]
+  for component in std.objectFields(kp)
+  for resource in std.filter(
+    function(resource)
+      kp[component][resource].kind != 'CustomResourceDefinition' && kp[component][resource].kind != 'Namespace', std.objectFields(kp[component])
+  )
+}
diff --git a/jsonnet/kube-prometheus/addons/networkpolicies-disabled.libsonnet b/jsonnet/kube-prometheus/addons/networkpolicies-disabled.libsonnet
@@ -0,0 +1,35 @@
+// Disables creation of NetworkPolicies
+
+{
+  blackboxExporter+: {
+    networkPolicies:: {},
+  },
+
+  kubeStateMetrics+: {
+    networkPolicies:: {},
+  },
+
+  nodeExporter+: {
+    networkPolicies:: {},
+  },
+
+  prometheusAdapter+: {
+    networkPolicies:: {},
+  },
+
+  alertmanager+: {
+    networkPolicies:: {},
+  },
+
+  grafana+: {
+    networkPolicies:: {},
+  },
+
+  prometheus+: {
+    networkPolicies:: {},
+  },
+
+  prometheusOperator+: {
+    networkPolicies:: {},
+  },
+}
diff --git a/jsonnet/kube-prometheus/components/alertmanager.libsonnet b/jsonnet/kube-prometheus/components/alertmanager.libsonnet
@@ -103,6 +103,51 @@ function(params) {
     },
   },
 
+  networkPolicy: {
+    apiVersion: 'networking.k8s.io/v1',
+    kind: 'NetworkPolicy',
+    metadata: am.service.metadata,
+    spec: {
+      podSelector: {
+        matchLabels: am._config.selectorLabels,
+      },
+      policyTypes: ['Egress', 'Ingress'],
+      egress: [{}],
+      ingress: [
+        {
+          from: [{
+            podSelector: {
+              matchLabels: {
+                'app.kubernetes.io/name': 'prometheus',
+              },
+            },
+          }],
+          ports: std.map(function(o) {
+            port: o.port,
+            protocol: 'TCP',
+          }, am.service.spec.ports),
+        },
+        // Alertmanager cluster peer-to-peer communication
+        {
+          from: [{
+            podSelector: {
+              matchLabels: {
+                'app.kubernetes.io/name': 'alertmanager',
+              },
+            },
+          }],
+          ports: [{
+            port: 9094,
+            protocol: 'TCP',
+          }, {
+            port: 9094,
+            protocol: 'UDP',
+          }],
+        },
+      ],
+    },
+  },
+
   secret: {
     apiVersion: 'v1',
     kind: 'Secret',

diff --git a/jsonnet/kube-prometheus/components/blackbox-exporter.libsonnet b/jsonnet/kube-prometheus/components/blackbox-exporter.libsonnet
@@ -250,6 +250,32 @@ function(params) {
       },
     },
 
+  networkPolicy: {
+    apiVersion: 'networking.k8s.io/v1',
+    kind: 'NetworkPolicy',
+    metadata: bb.service.metadata,
+    spec: {
+      podSelector: {
+        matchLabels: bb._config.selectorLabels,
+      },
+      policyTypes: ['Egress', 'Ingress'],
+      egress: [{}],
+      ingress: [{
+        from: [{
+          podSelector: {
+            matchLabels: {
+              'app.kubernetes.io/name': 'prometheus',
+            },
+          },
+        }],
+        ports: std.map(function(o) {
+          port: o.port,
+          protocol: 'TCP',
+        }, bb.service.spec.ports),
+      }],
+    },
+  },
+
   service: {
     apiVersion: 'v1',
     kind: 'Service',

diff --git a/jsonnet/kube-prometheus/components/grafana.libsonnet b/jsonnet/kube-prometheus/components/grafana.libsonnet
@@ -84,6 +84,32 @@ function(params)
       },
     },
 
+    networkPolicy: {
+      apiVersion: 'networking.k8s.io/v1',
+      kind: 'NetworkPolicy',
+      metadata: g.service.metadata,
+      spec: {
+        podSelector: {
+          matchLabels: g._config.selectorLabels,
+        },
+        policyTypes: ['Egress', 'Ingress'],
+        egress: [{}],
+        ingress: [{
+          from: [{
+            podSelector: {
+              matchLabels: {
+                'app.kubernetes.io/name': 'prometheus',
+              },
+            },
+          }],
+          ports: std.map(function(o) {
+            port: o.port,
+            protocol: 'TCP',
+          }, g.service.spec.ports),
+        }],
+      },
+    },
+
     // FIXME(ArthurSens): The securityContext overrides can be removed after some PRs get merged
     // 'allowPrivilegeEscalation: false' can be deleted when https://github.com/brancz/kubernetes-grafana/pull/128 gets merged.
     // 'readOnlyRootFilesystem: true' and extra volumeMounts can be deleted when https://github.com/brancz/kubernetes-grafana/pull/129 gets merged.

diff --git a/jsonnet/kube-prometheus/components/kube-state-metrics.libsonnet b/jsonnet/kube-prometheus/components/kube-state-metrics.libsonnet
@@ -124,6 +124,32 @@ function(params) (import 'github.com/kubernetes/kube-state-metrics/jsonnet/kube-
     image: ksm._config.kubeRbacProxyImage,
   }),
 
+  networkPolicy: {
+    apiVersion: 'networking.k8s.io/v1',
+    kind: 'NetworkPolicy',
+    metadata: ksm.service.metadata,
+    spec: {
+      podSelector: {
+        matchLabels: ksm._config.selectorLabels,
+      },
+      policyTypes: ['Egress', 'Ingress'],
+      egress: [{}],
+      ingress: [{
+        from: [{
+          podSelector: {
+            matchLabels: {
+              'app.kubernetes.io/name': 'prometheus',
+            },
+          },
+        }],
+        ports: std.map(function(o) {
+          port: o.port,
+          protocol: 'TCP',
+        }, ksm.service.spec.ports),
+      }],
+    },
+  },
+
   deployment+: {
     spec+: {
       template+: {

diff --git a/jsonnet/kube-prometheus/components/node-exporter.libsonnet b/jsonnet/kube-prometheus/components/node-exporter.libsonnet
@@ -160,6 +160,32 @@ function(params) {
     },
   },
 
+  networkPolicy: {
+    apiVersion: 'networking.k8s.io/v1',
+    kind: 'NetworkPolicy',
+    metadata: ne.service.metadata,
+    spec: {
+      podSelector: {
+        matchLabels: ne._config.selectorLabels,
+      },
+      policyTypes: ['Egress', 'Ingress'],
+      egress: [{}],
+      ingress: [{
+        from: [{
+          podSelector: {
+            matchLabels: {
+              'app.kubernetes.io/name': 'prometheus',
+            },
+          },
+        }],
+        ports: std.map(function(o) {
+          port: o.port,
+          protocol: 'TCP',
+        }, ne.service.spec.ports),
+      }],
+    },
+  },
+
   daemonset:
     local nodeExporter = {
       name: ne._config.name,

diff --git a/jsonnet/kube-prometheus/components/prometheus-adapter.libsonnet b/jsonnet/kube-prometheus/components/prometheus-adapter.libsonnet
@@ -206,6 +206,21 @@ function(params) {
     },
   },
 
+  networkPolicy: {
+    apiVersion: 'networking.k8s.io/v1',
+    kind: 'NetworkPolicy',
+    metadata: pa.service.metadata,
+    spec: {
+      podSelector: {
+        matchLabels: pa._config.selectorLabels,
+      },
+      policyTypes: ['Egress', 'Ingress'],
+      egress: [{}],
+      // Prometheus-adapter needs ingress allowed so HPAs can request metrics from it.
+      ingress: [{}],
+    },
+  },
+
   deployment:
     local c = {
       name: pa._config.name,

diff --git a/jsonnet/kube-prometheus/components/prometheus-operator.libsonnet b/jsonnet/kube-prometheus/components/prometheus-operator.libsonnet
@@ -72,6 +72,32 @@ function(params)
       },
     },
 
+    networkPolicy: {
+      apiVersion: 'networking.k8s.io/v1',
+      kind: 'NetworkPolicy',
+      metadata: po.service.metadata,
+      spec: {
+        podSelector: {
+          matchLabels: po._config.selectorLabels,
+        },
+        policyTypes: ['Egress', 'Ingress'],
+        egress: [{}],
+        ingress: [{
+          from: [{
+            podSelector: {
+              matchLabels: {
+                'app.kubernetes.io/name': 'prometheus',
+              },
+            },
+          }],
+          ports: std.map(function(o) {
+            port: o.port,
+            protocol: 'TCP',
+          }, po.service.spec.ports),
+        }],
+      },
+    },
+
     service+: {
       spec+: {
         ports: [