Merge pull request kubernetes-sigs#332 from matrus2/cni

Cni

examples/cni/cilium/README.md

@@ -0,0 +1,33 @@
+# Using Cilium
+
+This directory demonstrates how to use Cilium as a CNI in coordination with the test framework on Kind.
+
+### How it works?
+
+#### CNI Installation (main_test.go)
+
+1. Create the cluster with `disableDefaultCNI` parameter. To do so `CreateClusterWithConfig` is invoked with custom
+   configuration provided in `kind-config.yaml`.
+2. Create a namespace for workloads.
+3. Install Cilium as a Helm chart. First add necessary chart repository and later install the chart in `kube-system`
+   namespace.
+4. The cluster without CNI is non-functional as nodes status is set to `NotReady`, so that the setup is waiting for Cilium
+   deamonset to properly configure network interface and mark nodes as `Ready`, so the tests may proceed.
+5. At the end all components are being deleted.
+
+#### Tests (np_test.go)
+
+1. Upload basic Cilium configuration from `templates` folder to:  
+   a. set `CiliumClusterwideNetworkPolicy`s to allow connections
+   within a cluster to `kube-dns` and from `kube-dns` to `api-server` and externally.  It means that ingress and egress is denied and to enable any other traffic it is required to explicitly declare it (whitelist). (`templates/allow-dns.yaml`)  
+   b. allow egress traffic to `api.github.com` on `443` port in `cilium-test` namespace for any nginx
+   pod. (`templates/allow-github.yaml`)
+2. Create nginx deployment in the specified namespace.
+3. Ensure that nginx pod can connect to `api.github.com`.
+4. Ensure that nginx pod can't connect to `www.wikipedia.org`.
+
+### How to run?
+
+```bash
+go test -c -o cilium.test . && ./cilium.test --v 4
+```

examples/cni/cilium/kind-config.yaml

@@ -0,0 +1,7 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+  - role: control-plane
+  - role: worker
+networking:
+  disableDefaultCNI: true

examples/cni/cilium/main_test.go

@@ -0,0 +1,109 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cilium
+
+import (
+	"context"
+	"os"
+	"testing"
+	"time"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"sigs.k8s.io/e2e-framework/klient/k8s"
+	"sigs.k8s.io/e2e-framework/klient/wait"
+	"sigs.k8s.io/e2e-framework/klient/wait/conditions"
+	"sigs.k8s.io/e2e-framework/third_party/helm"
+
+	"sigs.k8s.io/e2e-framework/pkg/env"
+	"sigs.k8s.io/e2e-framework/pkg/envconf"
+	"sigs.k8s.io/e2e-framework/pkg/envfuncs"
+	"sigs.k8s.io/e2e-framework/support/kind"
+)
+
+var testEnv env.Environment
+
+func TestMain(m *testing.M) {
+	cfg, _ := envconf.NewFromFlags()
+	testEnv = env.NewWithConfig(cfg)
+	kindClusterName := "kind-with-cni"
+	namespace := "cilium-test"
+	testEnv.Setup(
+		// Create kind cluster with custom config
+		envfuncs.CreateClusterWithConfig(
+			kind.NewProvider(),
+			kindClusterName,
+			"kind-config.yaml",
+			kind.WithImage("kindest/node:v1.22.2")),
+		// Create random namespace
+		envfuncs.CreateNamespace(namespace),
+		// Install Cilium via Helm
+		func(ctx context.Context, cfg *envconf.Config) (context.Context, error) {
+			manager := helm.New(cfg.KubeconfigFile())
+			err := manager.RunRepo(helm.WithArgs("add", "cilium", "https://helm.cilium.io/"))
+			if err != nil {
+				return nil, err
+			}
+
+			err = manager.RunInstall(
+				helm.WithChart("cilium/cilium"),
+				helm.WithNamespace("kube-system"),
+				helm.WithArgs("--generate-name", "--set", "image.pullPolicy=IfNotPresent", "--set", "ipam.mode=kubernetes", "--wait"))
+			if err != nil {
+				return nil, err
+			}
+
+			// Wait for a worker node to be ready
+			client, err := cfg.NewClient()
+			if err != nil {
+				return nil, err
+			}
+
+			node := &corev1.Node{
+				ObjectMeta: metav1.ObjectMeta{Name: kindClusterName + "-worker"},
+			}
+
+			wait.For(conditions.New(client.Resources()).ResourceMatch(node, func(object k8s.Object) bool {
+				d := object.(*corev1.Node)
+				status := false
+				for _, v := range d.Status.Conditions {
+					if v.Type == "Ready" && v.Status == "True" {
+						status = true
+					}
+				}
+				return status
+			}), wait.WithTimeout(time.Minute*2))
+			return ctx, nil
+		})
+
+	testEnv.Finish(
+		// Uninstall Cilium
+		func(ctx context.Context, cfg *envconf.Config) (context.Context, error) {
+			manager := helm.New(cfg.KubeconfigFile())
+			err := manager.RunRepo(helm.WithArgs("remove", "cilium"))
+			if err != nil {
+				return nil, err
+			}
+			return ctx, nil
+		},
+		envfuncs.DeleteNamespace(namespace),
+		envfuncs.ExportClusterLogs(kindClusterName, "./logs"),
+		envfuncs.DestroyCluster(kindClusterName),
+	)
+	os.Exit(testEnv.Run(m))
+}

examples/cni/cilium/np_test.go

@@ -0,0 +1,140 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cilium
+
+import (
+	"bytes"
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	"sigs.k8s.io/e2e-framework/klient/decoder"
+	"sigs.k8s.io/e2e-framework/klient/k8s/resources"
+
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/e2e-framework/klient/wait"
+	"sigs.k8s.io/e2e-framework/klient/wait/conditions"
+
+	"sigs.k8s.io/e2e-framework/pkg/envconf"
+	"sigs.k8s.io/e2e-framework/pkg/features"
+)
+
+func TestNetworkPolicies(t *testing.T) {
+	containerName := "nginx"
+	podName := ""
+	feature := features.New("FQDN whitelisting").
+		Setup(func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context {
+			// Setup cluster network policies to only allow whitelisted traffic
+			r, err := resources.New(config.Client().RESTConfig())
+			if err != nil {
+				t.Fatal(err)
+			}
+			err = decoder.ApplyWithManifestDir(ctx, r, "./templates", "*", []resources.CreateOption{})
+			if err != nil {
+				t.Fatal(err)
+			}
+			// Create deployment
+			deploymentName := "test-deployment"
+
+			deployment := newDeployment(config.Namespace(), deploymentName, 1, containerName)
+			client, err := config.NewClient()
+			if err != nil {
+				t.Fatal(err)
+			}
+			if err = client.Resources().Create(ctx, deployment); err != nil {
+				t.Fatal(err)
+			}
+			err = wait.For(conditions.New(client.Resources()).DeploymentConditionMatch(deployment, appsv1.DeploymentAvailable, corev1.ConditionTrue), wait.WithTimeout(time.Minute*5))
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			pods := &corev1.PodList{}
+			err = client.Resources(config.Namespace()).List(context.TODO(), pods)
+			if err != nil || pods.Items == nil {
+				t.Error("error while getting pods", err)
+			}
+			podName = pods.Items[0].Name
+
+			return ctx
+		}).
+		Assess("Nginx pod can call github api", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context {
+			client, err := c.NewClient()
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			var stdout, stderr bytes.Buffer
+
+			// Curl Github API
+			command := []string{"curl", "-I", "https://api.github.com"}
+			if err := client.Resources().ExecInPod(context.TODO(), c.Namespace(), podName, containerName, command, &stdout, &stderr); err != nil {
+				t.Log(stderr.String())
+				t.Fatal(err)
+			}
+
+			httpStatus := strings.Split(stdout.String(), "\n")[0]
+			if !strings.Contains(httpStatus, "200") {
+				t.Fatal("Couldn't connect to api.github.com")
+			}
+
+			return ctx
+		}).
+		Assess("Nginx pod can call github api", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context {
+			client, err := c.NewClient()
+			if err != nil {
+				t.Fatal(err)
+			}
+			var stdout, stderr bytes.Buffer
+
+			// Curl Wikipedia
+			command := []string{"curl", "-I", "-m", "1", "https://www.wikipedia.org"}
+			if err := client.Resources().ExecInPod(context.TODO(), c.Namespace(), podName, containerName, command, &stdout, &stderr); err == nil {
+				t.Log(stderr.String())
+				t.Fatal(err)
+			}
+
+			httpStatus := strings.Split(stdout.String(), "\n")[0]
+			if strings.Contains(httpStatus, "200") {
+				t.Fatal("It should not connect to wikipedia")
+			}
+
+			return ctx
+		}).Feature()
+
+	_ = testEnv.Test(t, feature)
+}
+
+func newDeployment(namespace string, name string, replicas int32, containerName string) *appsv1.Deployment {
+	labels := map[string]string{"app": "nginx"}
+	return &appsv1.Deployment{
+		ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace},
+		Spec: appsv1.DeploymentSpec{
+			Replicas: &replicas,
+			Selector: &metav1.LabelSelector{
+				MatchLabels: labels,
+			},
+			Template: corev1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{Labels: labels},
+				Spec:       corev1.PodSpec{Containers: []corev1.Container{{Name: containerName, Image: "nginx"}}},
+			},
+		},
+	}
+}

examples/cni/cilium/templates/allow-dns.yaml

@@ -0,0 +1,53 @@
+# Allow all to connect to DNS
+apiVersion: cilium.io/v2
+kind: CiliumClusterwideNetworkPolicy
+metadata:
+  name: core-dns-ingress
+spec:
+  endpointSelector:
+    matchLabels:
+      "k8s-app": kube-dns
+  ingress:
+    - fromEndpoints:
+        - {}
+      toPorts:
+        - ports:
+            - port: "53"
+              protocol: ANY
+          rules:
+            dns:
+              - matchPattern: "*"
+---
+apiVersion: cilium.io/v2
+kind: CiliumClusterwideNetworkPolicy
+metadata:
+  name: core-dns-egress
+spec:
+  endpointSelector: {}
+  egress:
+    - toEndpoints:
+        - matchLabels:
+            "k8s-app": kube-dns
+      toPorts:
+        - ports:
+            - port: "53"
+              protocol: ANY
+          rules:
+            dns:
+              - matchPattern: "*"
+---
+# Allow core-dns to connect to kube-apiserver and world
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: core-dns
+  namespace: kube-system
+spec:
+  endpointSelector:
+    matchLabels:
+      io.cilium.k8s.policy.serviceaccount: coredns
+  egress:
+    - toEntities:
+        - kube-apiserver
+        - world
+---

examples/cni/cilium/templates/allow-github.yaml

@@ -0,0 +1,17 @@
+# Allow app:nginx pod to call api.github.com
+apiVersion: "cilium.io/v2"
+kind: CiliumNetworkPolicy
+metadata:
+  name: fqdn
+  namespace: cilium-test
+spec:
+  endpointSelector:
+    matchLabels:
+      app: nginx
+  egress:
+    - toFQDNs:
+        - matchName: "api.github.com"
+      toPorts:
+        - ports:
+            - port: "443"
+---

pkg/internal/types/types.go

@@ -25,21 +25,21 @@ import (
 )
 
 // EnvFunc represents a user-defined operation that
-// can be used to customized the behavior of the
+// can be used to customize the behavior of the
 // environment. Changes to context are expected to surface
 // to caller.
 type EnvFunc func(context.Context, *envconf.Config) (context.Context, error)
 
 // FeatureEnvFunc represents a user-defined operation that
-// can be used to customized the behavior of the
+// can be used to customize the behavior of the
 // environment. Changes to context are expected to surface
 // to caller. Meant for use with before/after feature hooks.
 // *testing.T is provided in order to provide pass/fail context to
 // features.
 type FeatureEnvFunc func(context.Context, *envconf.Config, *testing.T, Feature) (context.Context, error)
 
 // TestEnvFunc represents a user-defined operation that
-// can be used to customized the behavior of the
+// can be used to customize the behavior of the
 // environment. Changes to context are expected to surface
 // to caller. Meant for use with before/after test hooks.
 type TestEnvFunc func(context.Context, *envconf.Config, *testing.T) (context.Context, error)