From b5a58034142576f7a8339a1bdc560856b7aed94e Mon Sep 17 00:00:00 2001 From: Bogusz Przybyslawski Date: Thu, 12 Oct 2023 16:42:25 +0200 Subject: [PATCH 1/2] Add custom cni (Cilium) test example --- examples/cni/cilium/README.md | 33 +++++ examples/cni/cilium/kind-config.yaml | 7 + examples/cni/cilium/main_test.go | 109 ++++++++++++++ examples/cni/cilium/np_test.go | 140 ++++++++++++++++++ examples/cni/cilium/templates/allow-dns.yaml | 53 +++++++ .../cni/cilium/templates/allow-github.yaml | 17 +++ pkg/internal/types/types.go | 6 +- 7 files changed, 362 insertions(+), 3 deletions(-) create mode 100644 examples/cni/cilium/README.md create mode 100644 examples/cni/cilium/kind-config.yaml create mode 100644 examples/cni/cilium/main_test.go create mode 100644 examples/cni/cilium/np_test.go create mode 100644 examples/cni/cilium/templates/allow-dns.yaml create mode 100644 examples/cni/cilium/templates/allow-github.yaml diff --git a/examples/cni/cilium/README.md b/examples/cni/cilium/README.md new file mode 100644 index 00000000..1993e1ab --- /dev/null +++ b/examples/cni/cilium/README.md @@ -0,0 +1,33 @@ +# Using Cilium + +This directory demonstrates how to use Cilium as a CNI in coordination with the test framework on Kind. + +### How it works? + +#### CNI Installation (main_test.go) + +1. Create the cluster with `disableDefaultCNI` parameter. To do so `CreateClusterWithConfig` is invoked with custom + configuration provided in `kind-config.yaml`. +2. Create a namespace for workloads. +3. Install Cilium as a Helm chart. First add necessary chart repository and later install the chart in `kube-system` + namespace. +4. The cluster without CNI is non-functional as nodes status is set to `NotReady`, so that the setup is waiting for Cilium + deamonset to properly configure network interface and mark nodes as `Ready`, so the tests may proceed. +5. At the end all components are being deleted. + +#### Tests (np_test.go) + +1. Upload basic Cilium configuration from `templates` folder to: + a. set `CiliumClusterwideNetworkPolicy`s to allow connections + within a cluster to `kube-dns` and from `kube-dns` to `api-server` and externally. It means that ingress and egress is denied and to enable any other traffic it is required to explicitly declare it (whitelist). (`templates/allow-dns.yaml`) + b. allow egress traffic to `api.github.com` on `443` port in `cilium-test` namespace for any nginx + pod. (`templates/allow-github.yaml`) +2. Create nginx deployment in the specified namespace. +3. Ensure that nginx pod can connect to `api.github.com`. +4. Ensure that nginx pod can't connect to `www.wikipedia.org`. + +### How to run? + +```bash +go test -c -o cilium.test . && ./cilium.test --v 4 +``` diff --git a/examples/cni/cilium/kind-config.yaml b/examples/cni/cilium/kind-config.yaml new file mode 100644 index 00000000..1502670d --- /dev/null +++ b/examples/cni/cilium/kind-config.yaml @@ -0,0 +1,7 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: + - role: control-plane + - role: worker +networking: + disableDefaultCNI: true \ No newline at end of file diff --git a/examples/cni/cilium/main_test.go b/examples/cni/cilium/main_test.go new file mode 100644 index 00000000..d2b3008e --- /dev/null +++ b/examples/cni/cilium/main_test.go @@ -0,0 +1,109 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cilium + +import ( + "context" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "os" + "sigs.k8s.io/e2e-framework/klient/k8s" + "sigs.k8s.io/e2e-framework/klient/wait" + "sigs.k8s.io/e2e-framework/klient/wait/conditions" + "sigs.k8s.io/e2e-framework/third_party/helm" + "testing" + "time" + + "sigs.k8s.io/e2e-framework/pkg/env" + "sigs.k8s.io/e2e-framework/pkg/envconf" + "sigs.k8s.io/e2e-framework/pkg/envfuncs" + "sigs.k8s.io/e2e-framework/support/kind" +) + +var ( + testEnv env.Environment +) + +func TestMain(m *testing.M) { + cfg, _ := envconf.NewFromFlags() + testEnv = env.NewWithConfig(cfg) + kindClusterName := "kind-with-cni" + namespace := "cilium-test" + testEnv.Setup( + // Create kind cluster with custom config + envfuncs.CreateClusterWithConfig( + kind.NewProvider(), + kindClusterName, + "kind-config.yaml", + kind.WithImage("kindest/node:v1.22.2")), + // Create random namespace + envfuncs.CreateNamespace(namespace), + // Install Cilium via Helm + func(ctx context.Context, cfg *envconf.Config) (context.Context, error) { + manager := helm.New(cfg.KubeconfigFile()) + err := manager.RunRepo(helm.WithArgs("add", "cilium", "https://helm.cilium.io/")) + if err != nil { + return nil, err + } + + err = manager.RunInstall( + helm.WithChart("cilium/cilium"), + helm.WithNamespace("kube-system"), + helm.WithArgs("--generate-name", "--set", "image.pullPolicy=IfNotPresent", "--set", "ipam.mode=kubernetes", "--wait")) + if err != nil { + return nil, err + } + + // Wait for a worker node to be ready + client, err := cfg.NewClient() + if err != nil { + return nil, err + } + + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: kindClusterName + "-worker"}, + } + + wait.For(conditions.New(client.Resources()).ResourceMatch(node, func(object k8s.Object) bool { + d := object.(*corev1.Node) + status := false + for _, v := range d.Status.Conditions { + if v.Type == "Ready" && v.Status == "True" { + status = true + } + } + return status + }), wait.WithTimeout(time.Minute*2)) + return ctx, nil + }) + + testEnv.Finish( + // Uninstall Cilium + func(ctx context.Context, cfg *envconf.Config) (context.Context, error) { + manager := helm.New(cfg.KubeconfigFile()) + err := manager.RunRepo(helm.WithArgs("remove", "cilium")) + if err != nil { + return nil, err + } + return ctx, nil + }, + envfuncs.DeleteNamespace(namespace), + envfuncs.ExportClusterLogs(kindClusterName, "./logs"), + envfuncs.DestroyCluster(kindClusterName), + ) + os.Exit(testEnv.Run(m)) +} diff --git a/examples/cni/cilium/np_test.go b/examples/cni/cilium/np_test.go new file mode 100644 index 00000000..6dece5d9 --- /dev/null +++ b/examples/cni/cilium/np_test.go @@ -0,0 +1,140 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cilium + +import ( + "bytes" + "context" + "sigs.k8s.io/e2e-framework/klient/decoder" + "sigs.k8s.io/e2e-framework/klient/k8s/resources" + "strings" + "testing" + "time" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/e2e-framework/klient/wait" + "sigs.k8s.io/e2e-framework/klient/wait/conditions" + + "sigs.k8s.io/e2e-framework/pkg/envconf" + "sigs.k8s.io/e2e-framework/pkg/features" +) + +func TestNetworkPolicies(t *testing.T) { + containerName := "nginx" + podName := "" + feature := features.New("FQDN whitelisting"). + Setup(func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context { + // Setup cluster network policies to only allow whitelisted traffic + r, err := resources.New(config.Client().RESTConfig()) + if err != nil { + t.Fatal(err) + } + err = decoder.ApplyWithManifestDir(ctx, r, "./templates", "*", []resources.CreateOption{}) + if err != nil { + t.Fatal(err) + } + // Create deployment + deploymentName := "test-deployment" + + deployment := newDeployment(config.Namespace(), deploymentName, 1, containerName) + client, err := config.NewClient() + if err != nil { + t.Fatal(err) + } + if err = client.Resources().Create(ctx, deployment); err != nil { + t.Fatal(err) + } + err = wait.For(conditions.New(client.Resources()).DeploymentConditionMatch(deployment, appsv1.DeploymentAvailable, corev1.ConditionTrue), wait.WithTimeout(time.Minute*5)) + if err != nil { + t.Fatal(err) + } + + pods := &corev1.PodList{} + err = client.Resources(config.Namespace()).List(context.TODO(), pods) + if err != nil || pods.Items == nil { + t.Error("error while getting pods", err) + } + podName = pods.Items[0].Name + + return ctx + }). + Assess("Nginx pod can call github api", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + client, err := c.NewClient() + if err != nil { + t.Fatal(err) + } + + var stdout, stderr bytes.Buffer + + // Curl Github API + command := []string{"curl", "-I", "https://api.github.com"} + if err := client.Resources().ExecInPod(context.TODO(), c.Namespace(), podName, containerName, command, &stdout, &stderr); err != nil { + t.Log(stderr.String()) + t.Fatal(err) + } + + httpStatus := strings.Split(stdout.String(), "\n")[0] + if !strings.Contains(httpStatus, "200") { + t.Fatal("Couldn't connect to api.github.com") + } + + return ctx + }). + Assess("Nginx pod can call github api", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + client, err := c.NewClient() + if err != nil { + t.Fatal(err) + } + var stdout, stderr bytes.Buffer + + // Curl Wikipedia + command := []string{"curl", "-I", "-m", "1", "https://www.wikipedia.org"} + if err := client.Resources().ExecInPod(context.TODO(), c.Namespace(), podName, containerName, command, &stdout, &stderr); err == nil { + t.Log(stderr.String()) + t.Fatal(err) + } + + httpStatus := strings.Split(stdout.String(), "\n")[0] + if strings.Contains(httpStatus, "200") { + t.Fatal("It should not connect to wikipedia") + } + + return ctx + }).Feature() + + _ = testEnv.Test(t, feature) + +} + +func newDeployment(namespace string, name string, replicas int32, containerName string) *appsv1.Deployment { + labels := map[string]string{"app": "nginx"} + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + Spec: appsv1.DeploymentSpec{ + Replicas: &replicas, + Selector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Labels: labels}, + Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: containerName, Image: "nginx"}}}, + }, + }, + } +} diff --git a/examples/cni/cilium/templates/allow-dns.yaml b/examples/cni/cilium/templates/allow-dns.yaml new file mode 100644 index 00000000..c00eb976 --- /dev/null +++ b/examples/cni/cilium/templates/allow-dns.yaml @@ -0,0 +1,53 @@ +# Allow all to connect to DNS +apiVersion: cilium.io/v2 +kind: CiliumClusterwideNetworkPolicy +metadata: + name: core-dns-ingress +spec: + endpointSelector: + matchLabels: + "k8s-app": kube-dns + ingress: + - fromEndpoints: + - {} + toPorts: + - ports: + - port: "53" + protocol: ANY + rules: + dns: + - matchPattern: "*" +--- +apiVersion: cilium.io/v2 +kind: CiliumClusterwideNetworkPolicy +metadata: + name: core-dns-egress +spec: + endpointSelector: {} + egress: + - toEndpoints: + - matchLabels: + "k8s-app": kube-dns + toPorts: + - ports: + - port: "53" + protocol: ANY + rules: + dns: + - matchPattern: "*" +--- +# Allow core-dns to connect to kube-apiserver and world +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: core-dns + namespace: kube-system +spec: + endpointSelector: + matchLabels: + io.cilium.k8s.policy.serviceaccount: coredns + egress: + - toEntities: + - kube-apiserver + - world +--- \ No newline at end of file diff --git a/examples/cni/cilium/templates/allow-github.yaml b/examples/cni/cilium/templates/allow-github.yaml new file mode 100644 index 00000000..f80b356e --- /dev/null +++ b/examples/cni/cilium/templates/allow-github.yaml @@ -0,0 +1,17 @@ +# Allow app:nginx pod to call api.github.com +apiVersion: "cilium.io/v2" +kind: CiliumNetworkPolicy +metadata: + name: fqdn + namespace: cilium-test +spec: + endpointSelector: + matchLabels: + app: nginx + egress: + - toFQDNs: + - matchName: "api.github.com" + toPorts: + - ports: + - port: "443" +--- diff --git a/pkg/internal/types/types.go b/pkg/internal/types/types.go index ed4fef4b..f0f710c8 100644 --- a/pkg/internal/types/types.go +++ b/pkg/internal/types/types.go @@ -25,13 +25,13 @@ import ( ) // EnvFunc represents a user-defined operation that -// can be used to customized the behavior of the +// can be used to customize the behavior of the // environment. Changes to context are expected to surface // to caller. type EnvFunc func(context.Context, *envconf.Config) (context.Context, error) // FeatureEnvFunc represents a user-defined operation that -// can be used to customized the behavior of the +// can be used to customize the behavior of the // environment. Changes to context are expected to surface // to caller. Meant for use with before/after feature hooks. // *testing.T is provided in order to provide pass/fail context to @@ -39,7 +39,7 @@ type EnvFunc func(context.Context, *envconf.Config) (context.Context, error) type FeatureEnvFunc func(context.Context, *envconf.Config, *testing.T, Feature) (context.Context, error) // TestEnvFunc represents a user-defined operation that -// can be used to customized the behavior of the +// can be used to customize the behavior of the // environment. Changes to context are expected to surface // to caller. Meant for use with before/after test hooks. type TestEnvFunc func(context.Context, *envconf.Config, *testing.T) (context.Context, error) From 5f15f4b3398bd8d584260d19c2b6f94760dcf153 Mon Sep 17 00:00:00 2001 From: Bogusz Przybyslawski Date: Thu, 19 Oct 2023 09:34:27 +0200 Subject: [PATCH 2/2] Execute gofmt --- examples/cni/cilium/main_test.go | 12 ++++++------ examples/cni/cilium/np_test.go | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/examples/cni/cilium/main_test.go b/examples/cni/cilium/main_test.go index d2b3008e..cf4a9eed 100644 --- a/examples/cni/cilium/main_test.go +++ b/examples/cni/cilium/main_test.go @@ -18,15 +18,17 @@ package cilium import ( "context" + "os" + "testing" + "time" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "os" + "sigs.k8s.io/e2e-framework/klient/k8s" "sigs.k8s.io/e2e-framework/klient/wait" "sigs.k8s.io/e2e-framework/klient/wait/conditions" "sigs.k8s.io/e2e-framework/third_party/helm" - "testing" - "time" "sigs.k8s.io/e2e-framework/pkg/env" "sigs.k8s.io/e2e-framework/pkg/envconf" @@ -34,9 +36,7 @@ import ( "sigs.k8s.io/e2e-framework/support/kind" ) -var ( - testEnv env.Environment -) +var testEnv env.Environment func TestMain(m *testing.M) { cfg, _ := envconf.NewFromFlags() diff --git a/examples/cni/cilium/np_test.go b/examples/cni/cilium/np_test.go index 6dece5d9..124376d4 100644 --- a/examples/cni/cilium/np_test.go +++ b/examples/cni/cilium/np_test.go @@ -19,12 +19,13 @@ package cilium import ( "bytes" "context" - "sigs.k8s.io/e2e-framework/klient/decoder" - "sigs.k8s.io/e2e-framework/klient/k8s/resources" "strings" "testing" "time" + "sigs.k8s.io/e2e-framework/klient/decoder" + "sigs.k8s.io/e2e-framework/klient/k8s/resources" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -119,7 +120,6 @@ func TestNetworkPolicies(t *testing.T) { }).Feature() _ = testEnv.Test(t, feature) - } func newDeployment(namespace string, name string, replicas int32, containerName string) *appsv1.Deployment {