[GEP-15] Add operations.gardener.cloud API group, Bastion resource (g…

…ardener#3974) * add Bastion type to core/v1alpha1 * add operations.gardener.cloud API group * make clean generate * set user info only on creation
nickytd · May 7, 2021 · e7d4ef8 · e7d4ef8
1 parent 917f5fa
commit e7d4ef8
Show file tree

Hide file tree

Showing 108 changed files with 9,283 additions and 65 deletions.
diff --git a/...trolplane/charts/application/templates/apiservice-v1alpha1-operations-gardener-cloud.yaml b/...trolplane/charts/application/templates/apiservice-v1alpha1-operations-gardener-cloud.yaml
@@ -0,0 +1,24 @@
+{{- if .Values.global.apiserver.enabled }}
+apiVersion: {{ include "apiserviceversion" . }}
+kind: APIService
+metadata:
+  name: v1alpha1.operations.gardener.cloud
+  labels:
+    app: gardener
+    role: apiserver
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    release: "{{ .Release.Name }}"
+    heritage: "{{ .Release.Service }}"
+spec:
+  insecureSkipTLSVerify: {{ .Values.global.apiserver.insecureSkipTLSVerify }}
+  {{- if not .Values.global.apiserver.insecureSkipTLSVerify }}
+  caBundle: {{ required ".Values.global.apiserver.caBundle is required" (b64enc .Values.global.apiserver.caBundle) }}
+  {{- end }}
+  group: operations.gardener.cloud
+  version: v1alpha1
+  groupPriorityMinimum: 10
+  versionPriority: 10
+  service:
+    name: gardener-apiserver
+    namespace: garden
+{{- end }}
diff --git a/charts/gardener/controlplane/charts/application/templates/rbac-user.yaml b/charts/gardener/controlplane/charts/application/templates/rbac-user.yaml
@@ -12,6 +12,7 @@ rules:
   - seedmanagement.gardener.cloud
   - dashboard.gardener.cloud
   - settings.gardener.cloud
+  - operations.gardener.cloud
   resources:
   - '*'
   verbs:
@@ -228,6 +229,19 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - operations.gardener.cloud
+  resources:
+  - bastions
+  verbs:
+  - create
+  - delete
+  - deletecollection
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - rbac.authorization.k8s.io
   resources:
@@ -311,6 +325,14 @@ rules:
   - get
   - list
   - watch
+- apiGroups:
+  - operations.gardener.cloud
+  resources:
+  - bastions
+  verbs:
+  - get
+  - list
+  - watch
 - apiGroups:
   - rbac.authorization.k8s.io
   resources:

diff --git a/...ner/controlplane/charts/application/templates/validatingwebhook-admission-controller.yaml b/...ner/controlplane/charts/application/templates/validatingwebhook-admission-controller.yaml
@@ -123,6 +123,14 @@ webhooks:
     - backupbuckets
     - backupentries
     - shootstates
+  - apiGroups:
+    - operations.gardener.cloud
+    apiVersions:
+    - "*"
+    operations:
+    - CREATE
+    resources:
+    - bastions
   - apiGroups:
     - core.gardener.cloud
     apiVersions:

diff --git a/charts/seed-bootstrap/charts/extensions/templates/crd-bastion.yaml b/charts/seed-bootstrap/charts/extensions/templates/crd-bastion.yaml
@@ -48,7 +48,7 @@ spec:
           description: Spec is the specification of this Bastion.
           properties:
             ingress:
-              description: Ingress controls from where the creation bastion host should be reachable.
+              description: Ingress controls from where the created bastion host should be reachable.
               items:
                 description: BastionIngressPolicy represents an ingress policy for SSH bastion hosts.
                 properties:

diff --git a/cmd/gardener-apiserver/app/gardener_apiserver.go b/cmd/gardener-apiserver/app/gardener_apiserver.go
@@ -19,6 +19,29 @@ import (
 	"errors"
 	"flag"
 
+	"github.com/gardener/gardener/pkg/api"
+	gardencore "github.com/gardener/gardener/pkg/apis/core"
+	gardencorev1alpha1 "github.com/gardener/gardener/pkg/apis/core/v1alpha1"
+	gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1"
+	v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants"
+	gardenoperations "github.com/gardener/gardener/pkg/apis/operations"
+	operationsv1alpha1 "github.com/gardener/gardener/pkg/apis/operations/v1alpha1"
+	seedmanagementv1alpha1 "github.com/gardener/gardener/pkg/apis/seedmanagement/v1alpha1"
+	settingsv1alpha1 "github.com/gardener/gardener/pkg/apis/settings/v1alpha1"
+	"github.com/gardener/gardener/pkg/apiserver"
+	admissioninitializer "github.com/gardener/gardener/pkg/apiserver/admission/initializer"
+	"github.com/gardener/gardener/pkg/apiserver/storage"
+	gardencoreclientset "github.com/gardener/gardener/pkg/client/core/clientset/internalversion"
+	gardenversionedcoreclientset "github.com/gardener/gardener/pkg/client/core/clientset/versioned"
+	gardenexternalcoreinformers "github.com/gardener/gardener/pkg/client/core/informers/externalversions"
+	gardencoreinformers "github.com/gardener/gardener/pkg/client/core/informers/internalversion"
+	clientkubernetes "github.com/gardener/gardener/pkg/client/kubernetes"
+	seedmanagementclientset "github.com/gardener/gardener/pkg/client/seedmanagement/clientset/versioned"
+	seedmanagementinformer "github.com/gardener/gardener/pkg/client/seedmanagement/informers/externalversions"
+	settingsclientset "github.com/gardener/gardener/pkg/client/settings/clientset/versioned"
+	settingsinformer "github.com/gardener/gardener/pkg/client/settings/informers/externalversions"
+	"github.com/gardener/gardener/pkg/openapi"
+
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 	corev1 "k8s.io/api/core/v1"
@@ -43,27 +66,6 @@ import (
 	"k8s.io/component-base/version"
 	"k8s.io/component-base/version/verflag"
 	"sigs.k8s.io/controller-runtime/pkg/client"
-
-	"github.com/gardener/gardener/pkg/api"
-	gardencore "github.com/gardener/gardener/pkg/apis/core"
-	gardencorev1alpha1 "github.com/gardener/gardener/pkg/apis/core/v1alpha1"
-	gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1"
-	v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants"
-	seedmanagementv1alpha1 "github.com/gardener/gardener/pkg/apis/seedmanagement/v1alpha1"
-	settingsv1alpha1 "github.com/gardener/gardener/pkg/apis/settings/v1alpha1"
-	"github.com/gardener/gardener/pkg/apiserver"
-	admissioninitializer "github.com/gardener/gardener/pkg/apiserver/admission/initializer"
-	"github.com/gardener/gardener/pkg/apiserver/storage"
-	gardencoreclientset "github.com/gardener/gardener/pkg/client/core/clientset/internalversion"
-	gardenversionedcoreclientset "github.com/gardener/gardener/pkg/client/core/clientset/versioned"
-	gardenexternalcoreinformers "github.com/gardener/gardener/pkg/client/core/informers/externalversions"
-	gardencoreinformers "github.com/gardener/gardener/pkg/client/core/informers/internalversion"
-	clientkubernetes "github.com/gardener/gardener/pkg/client/kubernetes"
-	seedmanagementclientset "github.com/gardener/gardener/pkg/client/seedmanagement/clientset/versioned"
-	seedmanagementinformer "github.com/gardener/gardener/pkg/client/seedmanagement/informers/externalversions"
-	settingsclientset "github.com/gardener/gardener/pkg/client/settings/clientset/versioned"
-	settingsinformer "github.com/gardener/gardener/pkg/client/settings/informers/externalversions"
-	"github.com/gardener/gardener/pkg/openapi"
 )
 
 // NewCommandStartGardenerAPIServer creates a *cobra.Command object with default parameters.
@@ -122,6 +124,7 @@ func NewOptions() *Options {
 				gardencorev1alpha1.SchemeGroupVersion,
 				seedmanagementv1alpha1.SchemeGroupVersion,
 				settingsv1alpha1.SchemeGroupVersion,
+				operationsv1alpha1.SchemeGroupVersion,
 			),
 		),
 		ServerRunOptions: genericoptions.NewServerRunOptions(),
@@ -366,6 +369,7 @@ func (o *Options) ApplyTo(config *apiserver.Config) error {
 		gardencorev1alpha1.SchemeGroupVersion,
 		seedmanagementv1alpha1.SchemeGroupVersion,
 		settingsv1alpha1.SchemeGroupVersion,
+		operationsv1alpha1.SchemeGroupVersion,
 	)
 
 	mergedResourceConfig, err := resourceconfig.MergeAPIResourceConfigs(resourceConfig, nil, api.Scheme)
@@ -378,6 +382,7 @@ func (o *Options) ApplyTo(config *apiserver.Config) error {
 	resourceEncodingConfig.SetResourceEncoding(gardencore.Resource("shootstates"), gardencorev1alpha1.SchemeGroupVersion, gardencore.SchemeGroupVersion)
 	// TODO: `ShootExtensionStatus` is not yet promoted to `core.gardener.cloud/v1beta1` - this can be removed once `ShootExtensionStatus` got promoted.
 	resourceEncodingConfig.SetResourceEncoding(gardencore.Resource("shootextensionstatuses"), gardencorev1alpha1.SchemeGroupVersion, gardencore.SchemeGroupVersion)
+	resourceEncodingConfig.SetResourceEncoding(gardenoperations.Resource("bastions"), operationsv1alpha1.SchemeGroupVersion, gardenoperations.SchemeGroupVersion)
 
 	storageFactory := &storage.GardenerStorageFactory{
 		DefaultStorageFactory: serverstorage.NewDefaultStorageFactory(

diff --git a/docs/deployment/gardenlet_api_access.md b/docs/deployment/gardenlet_api_access.md
@@ -32,6 +32,7 @@ Today, the following rules are implemented:
 | --------------------------- | ------------------------------------------------------------- | ----------------------------------------------------------------| ----------- |
 | `BackupBucket`              | `get`, `list`, `watch`, `create`, `update`, `patch`, `delete` | `BackupBucket` -> `Seed`                                        | Allow `get`, `list`, `watch` requests for all `BackupBucket`s. Allow only `create`, `update`, `patch`, `delete` requests for `BackupBucket`s assigned to the `gardenlet`'s `Seed`. |
 | `BackupEntry`               | `get`, `list`, `watch`, `create`, `update`, `patch`           | `BackupEntry` -> `Seed`                                         | Allow `get`, `list`, `watch` requests for all `BackupEntry`s. Allow only `create`, `update`, `patch` requests for `BackupEntry`s assigned to the `gardenlet`'s `Seed` and referencing `BackupBucket`s assigned to the `gardenlet`'s `Seed`. |
+| `Bastion`                   | `get`, `list`, `watch`, `create`, `update`, `patch`           | `Bastion` -> `Seed`                                             | Allow `get`, `list`, `watch` requests for all `Bastion`s. Allow only `create`, `update`, `patch` requests for `Bastion`s assigned to the `gardenlet`'s `Seed`. |
 | `CertificateSigningRequest` | `get`, `create`                                               | `CertificateSigningRequest` -> `Seed`                           | Allow only `get`, `create` requests for `CertificateSigningRequest`s related to the `gardenlet`'s `Seed`. |
 | `CloudProfile`              | `get`                                                         | `CloudProfile` -> `Shoot` -> `Seed`                             | Allow only `get` requests for `CloudProfile`s referenced by `Shoot`s that are assigned to the `gardenlet`'s `Seed`. |
 | `ConfigMap`                 | `get`                                                         | `ConfigMap` -> `Shoot` -> `Seed`                                | Allow only `get` requests for `ConfigMap`s referenced by `Shoot`s that are assigned to the `gardenlet`'s `Seed`. Allows reading the `kube-system/cluster-identity` `ConfigMap`. |
@@ -51,7 +52,7 @@ Today, the following rules are implemented:
 [1] If you use `ManagedSeed` resources then the gardenlet reconciling them ("parent gardenlet") may be allowed to submit certain requests for the `Seed` resources resulting out of such `ManagedSeed` reconciliations (even if the "parent gardenlet" is not responsible for them): 
 
 - ℹ️ It is allowed to delete the `Seed` resources if the corresponding `ManagedSeed` objects already have a `deletionTimestamp` (this is secure as gardenlets themselves don't have permissions for deleting `ManagedSeed`s).
-- ⚠ It is allowed to create or update `Seed` resources if the corresponding `ManagedSeed` objects use a seed template, i.e., `.spec.seedTemplate != nil`. In this case, there is at least one gardenlet in your system which is responsible for two or more `Seed`s. Please keep in mind that this use case is not recommended for production scenarios (you should only have one dedicated gardenlet per seed cluster), hence, the security improvements discussed in this document might be limited.   
+- ⚠ It is allowed to create or update `Seed` resources if the corresponding `ManagedSeed` objects use a seed template, i.e., `.spec.seedTemplate != nil`. In this case, there is at least one gardenlet in your system which is responsible for two or more `Seed`s. Please keep in mind that this use case is not recommended for production scenarios (you should only have one dedicated gardenlet per seed cluster), hence, the security improvements discussed in this document might be limited.
 
 ## `SeedAuthorizer` Authorization Webhook Enablement
 

diff --git a/docs/proposals/15-manage-bastions-and-ssh-key-pair-rotation.md b/docs/proposals/15-manage-bastions-and-ssh-key-pair-rotation.md
@@ -83,15 +83,13 @@ The following is a list of involved components, that either need to be newly int
     - on creation, sets `metadata.annotations["gardener.cloud/created-by"]` according to the user that created the resource
     - when `gardener.cloud/operation: keepalive` is set it will be removed by GAPI from the annotations and `status.lastHeartbeatTimestamp` will be set with the current timestamp. The `status.expirationTimestamp` will be calculated by taking the last heartbeat timestamp and adding `x` minutes (configurable, default `60` Minutes).
     - validates that only the creator of the bastion (see `gardener.cloud/created-by` annotation) can update `spec.ingress`
+    - validates that a Bastion can only be created for a Shoot if that Shoot is already assigned to a Seed
+    - sets `spec.seedName` and `spec.providerType` based on the `spec.shootRef`
 3. `gardenlet`
     - Watches `Bastion` resource for own seed under api group `operations.gardener.cloud` in the garden cluster
     - Creates `Bastion` custom resource under api group `extensions.gardener.cloud/v1alpha1` in the seed cluster
       - Populates bastion user data under field under `spec.userData` similar to https://github.com/gardener/gardenctl/blob/1e3e5fa1d5603e2161f45046ba7c6b5b4107369e/pkg/cmd/ssh.go#L160-L171. By this means the `spec.sshPublicKey` from the `Bastion` resource in the garden cluster will end up in the `authorized_keys` file on the bastion host.
-4. `GCM`:
-    - During reconcile of the `Bastion` resource:
-      - according to `spec.shootRef`, sets the `status.seedName`
-      - according to `spec.shootRef`, sets the `status.providerType`
-5. Gardener extension provider <infra> / Bastion Controller on Seed:
+4. Gardener extension provider <infra> / Bastion Controller on Seed:
     - With own `Bastion` Custom Resource Definition in the seed under the api group `extensions.gardener.cloud/v1alpha1`
     - Watches `Bastion` custom resources that are created by the `gardenlet` in the seed
     - Controller reads `cloudprovider` credentials from seed-shoot namespace
@@ -101,23 +99,23 @@ The following is a list of involved components, that either need to be newly int
     - Updates status of `Bastion` resource:
         - With bastion IP under `status.ingress.ip` or hostname under `status.ingress.hostname`
         - Updates the `status.lastOperation` with the status of the last reconcile operation
-6. `gardenlet`
+5. `gardenlet`
     - Syncs back the `status.ingress` and `status.conditions` of the `Bastion` resource in the seed to the garden cluster in case it changed
-7. `gardenctl`
+6. `gardenctl`
     - initiates `ssh` session once `status.conditions['BastionReady']` is true of the `Bastion` resource in the garden cluster
         - locates private `ssh` key matching `spec["sshPublicKey"]` which was configured beforehand by the user
         - reads bastion IP (`status.ingress.ip`) or hostname (`status.ingress.hostname`)
         - reads the private key from the `ssh` key pair for the shoot node
         - opens `ssh` connection to the bastion and from there to the respective shoot node
     - runs heartbeat in parallel as long as the `ssh` session is open by annotating the `Bastion` resource with `gardener.cloud/operation: keepalive`
-8. `GCM`:
+7. `GCM`:
     - Once `status.expirationTimestamp` is reached, the `Bastion` will be marked for deletion
-9. `gardenlet`:
+8. `gardenlet`:
     - Once the `Bastion` resource in the garden cluster is marked for deletion, it marks the `Bastion` resource in the seed for deletion
-10. Gardener extension provider <infra> / Bastion Controller on Seed:
+9. Gardener extension provider <infra> / Bastion Controller on Seed:
     - all created resources will be cleaned up
     - On succes, removes finalizer on `Bastion` resource in seed
-11. `gardenlet`:
+10. `gardenlet`:
     - removes finalizer on `Bastion` resource in garden cluster
 
 ### Resource Example
@@ -137,21 +135,22 @@ spec:
   shootRef: # namespace cannot be set / it's the same as .metadata.namespace
     name: my-cluster # immutable
 
+  # the following fields are set by the GAPI
+  seedName: aws-eu2
+  providerType: aws
+
   sshPublicKey: c3NoLXJzYSAuLi4K # immutable, public `ssh` key of the user
 
   ingress: # can only be updated by the creator of the bastion
   - ipBlock:
       cidr: 1.2.3.4/32 # public IP of the user. CIDR is a string representing the IP Block. Valid examples are "192.168.1.1/24" or "2001:db9::/64"
 
 status:
-  # the following fields are set by the GCM
-  seedName: aws-eu2
-  providerType: aws
-
   # the following fields are managed by the controller in the seed and synced by gardenlet
   ingress: # IP or hostname of the bastion
     ip: 1.2.3.5
     # hostname: foo.bar
+
   conditions:
   - type: BastionReady # when the `status` is true of condition type `BastionReady`, the client can initiate the `ssh` connection
     status: 'True'

diff --git a/example/100-bastion.yaml b/example/100-bastion.yaml
@@ -0,0 +1,14 @@
+# Bastion to facilitate SSH access to a Shoot's workers
+---
+apiVersion: operations.gardener.cloud/v1alpha1
+kind: Bastion
+metadata:
+  name: example-bastion
+  namespace: garden-dev
+spec:
+  shootRef:
+    name: example-shoot
+  sshPublicKey: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDcSZKq0lM9w+ElLp9I9jFvqEFbOV1+iOBX7WEe66GvPLOWl9ul03ecjhOf06+FhPsWFac1yaxo2xj+SJ+FVZ3DdSn4fjTpS9NGyQVPInSZveetRw0TV0rbYCFBTJuVqUFu6yPEgdcWq8dlUjLqnRNwlelHRcJeBfACBZDLNSxjj0oUz7ANRNCEne1ecySwuJUAz3IlNLPXFexRT0alV7Nl9hmJke3dD73nbeGbQtwvtu8GNFEoO4Eu3xOCKsLw6ILLo4FBiFcYQOZqvYZgCb4ncKM52bnABagG54upgBMZBRzOJvWp0ol+jK3Em7Vb6ufDTTVNiQY78U6BAlNZ8Xg+LUVeyk1C6vWjzAQf02eRvMdfnRCFvmwUpzbHWaVMsQm8gf3AgnTUuDR0ev1nQH/5892wZA86uLYW/wLiiSbvQsqtY1jSn9BAGFGdhXgWLAkGsd/E1vOT+vDcor6/6KjHBm0rG697A3TDBRkbXQ/1oFxcM9m17RteCaXuTiAYWMqGKDoJvTMDc4L+Uvy544pEfbOH39zfkIYE76WLAFPFsUWX6lXFjQrX3O7vEV73bCHoJnwzaNd03PSdJOw+LCzrTmxVezwli3F9wUDiBRB0HkQxIXQmncc1HSecCKALkogIK+1e1OumoWh6gPdkF4PlTMUxRitrwPWSaiUIlPfCpQ== you@example.com
+  ingress:
+    - ipBlock:
+        cidr: 1.2.3.4/32
diff --git a/hack/api-reference/extensions.md b/hack/api-reference/extensions.md
@@ -388,7 +388,7 @@ contain code to provision the SSH key on the bastion instance.</p>
 </em>
 </td>
 <td>
-<p>Ingress controls from where the creation bastion host should be reachable.</p>
+<p>Ingress controls from where the created bastion host should be reachable.</p>
 </td>
 </tr>
 </table>
@@ -1744,7 +1744,7 @@ contain code to provision the SSH key on the bastion instance.</p>
 </em>
 </td>
 <td>
-<p>Ingress controls from where the creation bastion host should be reachable.</p>
+<p>Ingress controls from where the created bastion host should be reachable.</p>
 </td>
 </tr>
 </tbody>

diff --git a/hack/api-reference/operations-config.json b/hack/api-reference/operations-config.json
@@ -0,0 +1,20 @@
+{
+    "hideMemberFields": [
+        "TypeMeta"
+    ],
+    "hideTypePatterns": [
+        "ParseError$",
+        "List$"
+    ],
+    "externalPackages": [
+        {
+            "typeMatchPrefix": "^k8s\\.io/(api|apimachinery/pkg/apis)/",
+            "docsURLTemplate": "https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#{{lower .TypeIdentifier}}-{{arrIndex .PackageSegments -1}}-{{arrIndex .PackageSegments -2}}"
+        }
+    ],
+    "typeDisplayNamePrefixOverrides": {
+        "k8s.io/api/": "Kubernetes ",
+        "k8s.io/apimachinery/pkg/apis/": "Kubernetes "
+    },
+    "markdownDisabled": false
+}