[operator] Manage nginx-ingress-{controller,k8s-backend} (gardener#…

…7945)

* Install nginx-ingress through gardener-operator

* Handle Review comments

* Handle latest PR comments

* Rebase on top of gardener#8096

* Add validation for garden.spec.runtimeCluster.ingress.controller.kind

* Reuse gardencorev1beta1.Ingress for the Garden

* Add unit test for InterfaceMapToStringMap

charts/gardener/operator/templates/customresouredefintion.yaml

@@ -50,6 +50,43 @@ spec:
                 description: RuntimeCluster contains configuration for the runtime
                   cluster.
                 properties:
+                  ingress:
+                    description: Ingress configures Ingress specific settings for
+                      the Garden cluster. This field is immutable.
+                    properties:
+                      controller:
+                        description: Controller configures a Gardener managed Ingress
+                          Controller listening on the ingressDomain
+                        properties:
+                          kind:
+                            description: Kind defines which kind of IngressController
+                              to use. At the moment only `nginx` is supported
+                            enum:
+                            - nginx
+                            type: string
+                          providerConfig:
+                            description: ProviderConfig specifies infrastructure specific
+                              configuration for the ingressController
+                            type: object
+                            x-kubernetes-preserve-unknown-fields: true
+                        required:
+                        - kind
+                        type: object
+                      domain:
+                        description: Domain specifies the IngressDomain of the cluster
+                          pointing to the ingress controller endpoint. It will be
+                          used to construct ingress URLs for system applications running
+                          in Shoot/Garden clusters. Once set this field is immutable.
+                        maxLength: 253
+                        pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$
+                        type: string
+                        x-kubernetes-validations:
+                        - message: Value is immutable
+                          rule: self == oldSelf
+                    required:
+                    - controller
+                    - domain
+                    type: object
                   networking:
                     description: Networking defines the networking configuration of
                       the runtime cluster.
@@ -150,6 +187,7 @@ spec:
                         type: object
                     type: object
                 required:
+                - ingress
                 - networking
                 - provider
                 type: object

docs/api-reference/core.md

@@ -4815,7 +4815,7 @@ float64
 <a href="#core.gardener.cloud/v1beta1.SeedSpec">SeedSpec</a>)
 </p>
 <p>
-<p>Ingress configures the Ingress specific settings of the Seed cluster</p>
+<p>Ingress configures the Ingress specific settings of the cluster</p>
 </p>
 <table>
 <thead>
@@ -4833,8 +4833,8 @@ string
 </em>
 </td>
 <td>
-<p>Domain specifies the IngressDomain of the Seed cluster pointing to the ingress controller endpoint. It will be used
-to construct ingress URLs for system applications running in Shoot clusters. Once set this field is immutable.</p>
+<p>Domain specifies the IngressDomain of the cluster pointing to the ingress controller endpoint. It will be used
+to construct ingress URLs for system applications running in Shoot/Garden clusters. Once set this field is immutable.</p>
 </td>
 </tr>
 <tr>
@@ -4877,7 +4877,7 @@ string
 </em>
 </td>
 <td>
-<p>Kind defines which kind of IngressController to use, for example <code>nginx</code></p>
+<p>Kind defines which kind of IngressController to use. At the moment only <code>nginx</code> is supported</p>
 </td>
 </tr>
 <tr>

docs/api-reference/operator.md

@@ -1176,6 +1176,17 @@ string
 <tbody>
 <tr>
 <td>
+<code>ingress</code></br>
+<em>
+github.com/gardener/gardener/pkg/apis/core/v1beta1.Ingress
+</em>
+</td>
+<td>
+<p>Ingress configures Ingress specific settings for the Garden cluster. This field is immutable.</p>
+</td>
+</tr>
+<tr>
+<td>
 <code>networking</code></br>
 <em>
 <a href="#operator.gardener.cloud/v1alpha1.RuntimeNetworking">

docs/concepts/operator.md

@@ -30,6 +30,7 @@ As of today, this applies to:
 - `hvpa-controller` (when `HVPA` feature gate is enabled)
 - `etcd-druid`
 - `istio` control-plane
+- `nginx-ingress-controller`
 
 Those components are so-called "seed system components".
 As they were already made available by `gardener-operator`, the `gardenlet` just skips them.

docs/development/priority-classes.md

@@ -24,7 +24,7 @@ When using the `gardener-operator` for managing the garden runtime and virtual c
 | `gardener-garden-system-critical` | 999999550 | `gardener-operator`, `gardener-resource-manager`, `istio`                                 |
 | `gardener-garden-system-500`      | 999999500 | `virtual-garden-etcd-events`, `virtual-garden-etcd-main`, `virtual-garden-kube-apiserver` |
 | `gardener-garden-system-400`      | 999999400 | `virtual-garden-gardener-resource-manager`                                                |
-| `gardener-garden-system-300`      | 999999300 | `virtual-garden-kube-controller-manager`, `vpa-admission-controller`, `etcd-druid`        |
+| `gardener-garden-system-300`      | 999999300 | `virtual-garden-kube-controller-manager`, `vpa-admission-controller`, `etcd-druid`, `nginx-ingress-controller`        |
 | `gardener-garden-system-200`      | 999999200 | `vpa-recommender`, `vpa-updater`, `hvpa-controller`                                       |
 | `gardener-garden-system-100`      | 999999100 | `kube-state-metrics`                                                                      |
 

example/gardener-local/kind/cluster/templates/_extra_port_mappings.tpl

@@ -30,7 +30,7 @@
 {{- end -}}
 
 {{- define "extraPortMappings.gardener.seed.nginx" -}}
-{{- if and .Values.gardener.controlPlane.deployed .Values.gardener.seed.deployed -}}
+{{- if and .Values.gardener.controlPlane.deployed .Values.gardener.nginxIngress.deployed -}}
 - containerPort: 30448
   hostPort: 8448
 {{- end -}}

example/gardener-local/kind/cluster/values.yaml

@@ -5,6 +5,8 @@ gardener:
     deployed: false
   controlPlane:
     deployed: true
+  nginxIngress:
+    deployed: true
   seed:
     deployed: true
     istio:

example/gardener-local/kind/extensions/values.yaml

@@ -3,3 +3,5 @@ gardener:
     deployed: true
   seed:
     deployed: false
+  nginxIngress:
+    deployed: false

example/gardener-local/kind/operator/values.yaml

@@ -3,6 +3,8 @@ gardener:
     deployed: false
   seed:
     deployed: false
+  nginxIngress:
+    deployed: true
   garden:
     deployed: true
 

example/operator/10-crd-operator.gardener.cloud_gardens.yaml

@@ -50,6 +50,43 @@ spec:
                 description: RuntimeCluster contains configuration for the runtime
                   cluster.
                 properties:
+                  ingress:
+                    description: Ingress configures Ingress specific settings for
+                      the Garden cluster. This field is immutable.
+                    properties:
+                      controller:
+                        description: Controller configures a Gardener managed Ingress
+                          Controller listening on the ingressDomain
+                        properties:
+                          kind:
+                            description: Kind defines which kind of IngressController
+                              to use. At the moment only `nginx` is supported
+                            enum:
+                            - nginx
+                            type: string
+                          providerConfig:
+                            description: ProviderConfig specifies infrastructure specific
+                              configuration for the ingressController
+                            type: object
+                            x-kubernetes-preserve-unknown-fields: true
+                        required:
+                        - kind
+                        type: object
+                      domain:
+                        description: Domain specifies the IngressDomain of the cluster
+                          pointing to the ingress controller endpoint. It will be
+                          used to construct ingress URLs for system applications running
+                          in Shoot/Garden clusters. Once set this field is immutable.
+                        maxLength: 253
+                        pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$
+                        type: string
+                        x-kubernetes-validations:
+                        - message: Value is immutable
+                          rule: self == oldSelf
+                    required:
+                    - controller
+                    - domain
+                    type: object
                   networking:
                     description: Networking defines the networking configuration of
                       the runtime cluster.
@@ -150,6 +187,7 @@ spec:
                         type: object
                     type: object
                 required:
+                - ingress
                 - networking
                 - provider
                 type: object

example/operator/20-garden.yaml

@@ -14,6 +14,12 @@ metadata:
   name: garden
 spec:
   runtimeCluster:
+    ingress:
+      domain: ingress.dev.garden.example.com
+      controller:
+        kind: nginx
+      # providerConfig:
+      #   <some-optional-config-for-the-nginx-ingress-controller>
     networking:
       # Those CIDRs have been chosen to match with the kind Cluster configuration (see example/gardener-local/kind/cluster/values.yaml).
       # Generally, they have to match the CIDRs of the runtime cluster.

pkg/apis/core/v1beta1/types_seed.go

@@ -165,18 +165,22 @@ type SeedDNSProvider struct {
 	// Zones *DNSIncludeExclude `json:"zones,omitempty" protobuf:"bytes,4,opt,name=zones"`
 }
 
-// Ingress configures the Ingress specific settings of the Seed cluster
+// Ingress configures the Ingress specific settings of the cluster
 type Ingress struct {
-	// Domain specifies the IngressDomain of the Seed cluster pointing to the ingress controller endpoint. It will be used
-	// to construct ingress URLs for system applications running in Shoot clusters. Once set this field is immutable.
-	Domain string `json:"domain" protobuf:"bytes,1,opt,name=domain"`
+	// Domain specifies the IngressDomain of the cluster pointing to the ingress controller endpoint. It will be used
+	// to construct ingress URLs for system applications running in Shoot/Garden clusters. Once set this field is immutable.
+	// +kubebuilder:validation:MaxLength=253
+	// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value is immutable"
+	// +kubebuilder:validation:Pattern="^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$"
+	Domain string `json:"domain" protobuf:"bytes,1,name=domain"`
 	// Controller configures a Gardener managed Ingress Controller listening on the ingressDomain
-	Controller IngressController `json:"controller" protobuf:"bytes,2,opt,name=controller"`
+	Controller IngressController `json:"controller" protobuf:"bytes,2,name=controller"`
 }
 
 // IngressController enables a Gardener managed Ingress Controller listening on the ingressDomain
 type IngressController struct {
-	// Kind defines which kind of IngressController to use, for example `nginx`
+	// Kind defines which kind of IngressController to use. At the moment only `nginx` is supported
+	// +kubebuilder:validation:Enum="nginx"
 	Kind string `json:"kind" protobuf:"bytes,1,opt,name=kind"`
 	// ProviderConfig specifies infrastructure specific configuration for the ingressController
 	// +optional

pkg/apis/operator/v1alpha1/types.go

@@ -63,6 +63,8 @@ type GardenSpec struct {
 
 // RuntimeCluster contains configuration for the runtime cluster.
 type RuntimeCluster struct {
+	// Ingress configures Ingress specific settings for the Garden cluster. This field is immutable.
+	Ingress gardencorev1beta1.Ingress `json:"ingress"`
 	// Networking defines the networking configuration of the runtime cluster.
 	Networking RuntimeNetworking `json:"networking"`
 	// Provider defines the provider-specific information for this cluster.

pkg/component/shared/nginx_ingress.go

@@ -0,0 +1,73 @@
+// Copyright 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package shared
+
+import (
+	"github.com/Masterminds/semver"
+	corev1 "k8s.io/api/core/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/gardener/gardener/pkg/component"
+	"github.com/gardener/gardener/pkg/component/nginxingress"
+	"github.com/gardener/gardener/pkg/utils/images"
+	"github.com/gardener/gardener/pkg/utils/imagevector"
+)
+
+// NewNginxIngress returns a deployer for nginx-ingress-controller.
+func NewNginxIngress(
+	c client.Client,
+	namespaceName string,
+	targetNamespace string,
+	imageVector imagevector.ImageVector,
+	kubernetesVersion *semver.Version,
+	config map[string]string,
+	loadBalancerAnnotations map[string]string,
+	loadBalancerSourceRanges []string,
+	priorityClassName string,
+	pspDisabled bool,
+	vpaEnabled bool,
+	clusterType component.ClusterType,
+	externalTrafficPolicy corev1.ServiceExternalTrafficPolicyType,
+	ingressClass string,
+) (
+	component.DeployWaiter,
+	error,
+) {
+	imageController, err := imageVector.FindImage(images.ImageNameNginxIngressControllerSeed, imagevector.TargetVersion(kubernetesVersion.String()))
+	if err != nil {
+		return nil, err
+	}
+	imageDefaultBackend, err := imageVector.FindImage(images.ImageNameIngressDefaultBackend, imagevector.TargetVersion(kubernetesVersion.String()))
+	if err != nil {
+		return nil, err
+	}
+
+	values := nginxingress.Values{
+		ImageController:          imageController.String(),
+		ImageDefaultBackend:      imageDefaultBackend.String(),
+		IngressClass:             ingressClass,
+		ConfigData:               config,
+		LoadBalancerAnnotations:  loadBalancerAnnotations,
+		LoadBalancerSourceRanges: loadBalancerSourceRanges,
+		PriorityClassName:        priorityClassName,
+		PSPDisabled:              pspDisabled,
+		VPAEnabled:               vpaEnabled,
+		TargetNamespace:          targetNamespace,
+		ClusterType:              clusterType,
+		ExternalTrafficPolicy:    externalTrafficPolicy,
+	}
+
+	return nginxingress.New(c, namespaceName, values), nil
+}