[GEP-19] Migrate shoot Alertmanager deployment and configuration (gar…

…dener#9257)

* Remove memory limits

Similar to gardener#9205

* Integrate alertmanager deployment into `Shoot` controller

* `Ingress`

* Alerting secrets

There can only be at most one alerting secret, so we don't need handling for multiple, see https://github.com/gardener/gardener/blob/3867eb16afa1bfe020f464992e7ea3ae445a2389/pkg/utils/gardener/garden.go#L231-L233

* Add explicit network policy label to shoot Prometheus

The connection to the shoot alertmanager is implicitly covered by the `to-all-scrape-targets=allowed` label. However, it's better to explicitly allow the connection since it's explicitly configured in the Prometheus config.

* Migrate existing shoots early in reconciliation flow

This allows that we can drop the "cleanup"/migration code after a couple of releases while it's ensured that all old `alertmanager` `StatefulSet`s have been migrated.

* Address PR review feedback

docs/development/priority-classes.md

@@ -49,7 +49,7 @@ When using the `gardener-operator` for managing the garden runtime and virtual c
 | `gardener-system-400` | 999998400 | `gardener-resource-manager`                                                                                                                                                            |
 | `gardener-system-300` | 999998300 | `cloud-controller-manager`, `cluster-autoscaler`, `csi-driver-controller`, `kube-controller-manager`, `kube-scheduler`, `machine-controller-manager`, `terraformer`, `vpn-seed-server` |
 | `gardener-system-200` | 999998200 | `csi-snapshot-controller`, `csi-snapshot-validation`, `cert-controller-manager`, `shoot-dns-service`, `vpa-admission-controller`, `vpa-recommender`, `vpa-updater`                     |
-| `gardener-system-100` | 999998100 | `alertmanager`, `plutono`, `kube-state-metrics`, `prometheus`, `vali`, `event-logger`                                                                                                  |
+| `gardener-system-100` | 999998100 | `alertmanager-shoot`, `plutono`, `kube-state-metrics`, `prometheus`, `vali`, `event-logger`                                                                                            |
 
 ## Shoot Clusters
 

pkg/component/observability/monitoring/alertmanager/alertmanager.go

@@ -22,6 +22,7 @@ import (
 	"k8s.io/utils/ptr"
 
 	v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants"
+	"github.com/gardener/gardener/pkg/component"
 	"github.com/gardener/gardener/pkg/utils"
 )
 
@@ -41,7 +42,7 @@ func (a *alertManager) alertManager(takeOverOldPV bool) *monitoringv1.Alertmanag
 				}),
 			},
 			PriorityClassName: a.values.PriorityClassName,
-			Replicas:          ptr.To(int32(1)),
+			Replicas:          &a.values.Replicas,
 			Image:             &a.values.Image,
 			ImagePullPolicy:   corev1.PullIfNotPresent,
 			Version:           a.values.Version,
@@ -50,9 +51,6 @@ func (a *alertManager) alertManager(takeOverOldPV bool) *monitoringv1.Alertmanag
 					corev1.ResourceCPU:    resource.MustParse("5m"),
 					corev1.ResourceMemory: resource.MustParse("20Mi"),
 				},
-				Limits: corev1.ResourceList{
-					corev1.ResourceMemory: resource.MustParse("200Mi"),
-				},
 			},
 			SecurityContext: &corev1.PodSecurityContext{RunAsUser: ptr.To(int64(0))},
 			Storage: &monitoringv1.StorageSpec{
@@ -72,6 +70,11 @@ func (a *alertManager) alertManager(takeOverOldPV bool) *monitoringv1.Alertmanag
 		},
 	}
 
+	if a.values.ClusterType == component.ClusterTypeShoot {
+		obj.Labels[v1beta1constants.GardenRole] = v1beta1constants.GardenRoleMonitoring
+		obj.Spec.PodMetadata.Labels[v1beta1constants.GardenRole] = v1beta1constants.GardenRoleMonitoring
+	}
+
 	if takeOverOldPV {
 		obj.Spec.InitContainers = append(obj.Spec.InitContainers, corev1.Container{
 			Name:            "take-over-old-pv",