diff --git a/charts/gardener/gardenlet/templates/clusterrole-gardenlet.yaml b/charts/gardener/gardenlet/templates/clusterrole-gardenlet.yaml index eeb54312129..902e27f2dfe 100644 --- a/charts/gardener/gardenlet/templates/clusterrole-gardenlet.yaml +++ b/charts/gardener/gardenlet/templates/clusterrole-gardenlet.yaml @@ -435,3 +435,11 @@ rules: - envoyfilters verbs: - delete +- apiGroups: + - machine.sapcloud.io + resources: + - machinedeployments + verbs: + - list + - watch + - get diff --git a/charts/images.yaml b/charts/images.yaml index 29503b5a3a2..63e874d50a8 100644 --- a/charts/images.yaml +++ b/charts/images.yaml @@ -122,6 +122,19 @@ images: - name: kube-proxy sourceRepository: github.com/kubernetes/kubernetes repository: registry.k8s.io/kube-proxy +- name: machine-controller-manager + sourceRepository: github.com/gardener/machine-controller-manager + repository: eu.gcr.io/gardener-project/gardener/machine-controller-manager + tag: "v0.49.2" + labels: + - name: gardener.cloud/cve-categorisation + value: + network_exposure: protected + authentication_enforced: false + user_interaction: gardener-operator + confidentiality_requirement: high + integrity_requirement: high + availability_requirement: low - name: cluster-autoscaler sourceRepository: github.com/gardener/autoscaler repository: eu.gcr.io/gardener-project/gardener/autoscaler/cluster-autoscaler diff --git a/charts/seed-monitoring/charts/plutono/dashboards/owners/machine-controller-manager/mcm-dashboard.json b/charts/seed-monitoring/charts/plutono/dashboards/owners/machine-controller-manager/mcm-dashboard.json new file mode 100644 index 00000000000..c6ebd6cb0a3 --- /dev/null +++ b/charts/seed-monitoring/charts/plutono/dashboards/owners/machine-controller-manager/mcm-dashboard.json @@ -0,0 +1,1175 @@ +{ + "description": "Information about the operations of the Machine Controller Manager", + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "id": 16, + "iteration": 1564731005347, + "links": [ + { + "icon": "external link", + "tags": [], + "targetBlank": true, + "title": "Machine Controller Manager", + "tooltip": "", + "type": "link", + "url": "https://github.com/gardener/machine-controller-manager" + } + ], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "decimals": null, + "description": "State of the managed machines.\n\n| Code | Machine State |\n|---|---|\n| 0 | Running |\n| 1 | Terminating |\n| 2 | Unknown |\n| 3 | Failed |\n| -1 | Available |\n| -2 | Pending |", + "fill": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 5, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "mcm_machine_current_status_phase", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Managed Machines States", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": "3.2", + "min": "-2.2", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "Shows the CPU usage of the Machine Controller Manager and shows the requests and limits.", + "fill": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 13, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_cpu_usage_seconds_total{pod=~\"machine-controller-manager-(.+)\"}[5m])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Current ({{pod}})", + "refId": "A" + }, + { + "expr": "sum(kube_pod_container_resource_limits{resource=\"cpu\", unit=\"core\", pod=~\"machine-controller-manager-(.+)\"}) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Limits ({{pod}})", + "refId": "C" + }, + { + "expr": "sum(kube_pod_container_resource_requests{resource=\"cpu\", unit=\"core\", pod=~\"machine-controller-manager-(.+)\"}) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Requests ({{pod}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "MCM CPU usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "Shows the memory usage of the Machine Controller Manager.", + "fill": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 11, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{pod=~\"machine-controller-manager-(.+)\"}) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Current ({{pod}})", + "refId": "A" + }, + { + "expr": "sum(kube_pod_container_resource_limits{resource=\"memory\", unit=\"byte\", pod=~\"machine-controller-manager-(.+)\"}) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Limits ({{pod}})", + "refId": "B" + }, + { + "expr": "sum(kube_pod_container_resource_requests{resource=\"memory\", unit=\"byte\", pod=~\"machine-controller-manager-(.+)\"}) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Requests ({{pod}})", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "MCM Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 2, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "Indicates if the Machine Controller Manager is frozen due to unreachable API server.\n\n0 = ok; 1= frozen", + "fill": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 13 + }, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "mcm_machine_controller_frozen", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 0.5, + "yaxis": "left" + }, + { + "colorMode": "ok", + "fill": true, + "line": true, + "op": "lt", + "value": 0.5, + "yaxis": "left" + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "MCM Frozen Status (API Server reachable)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "max": "1.2", + "min": "-0.2", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "decimals": null, + "description": "Average per Second rate over 1m of IaaS provider api calls split by services. \n\nShows also the rate of failed iaas calls if at least one failed.", + "fill": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 13 + }, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(mcm_cloud_api_requests_total[1m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{provider}} / {{service}} ({{pod}})", + "refId": "A" + }, + { + "expr": "rate(mcm_cloud_api_requests_failed_total[1m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Error: {{provider}} / {{service}} ({{pod}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "IaaS API Calls", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "decimals": 0, + "description": "The count of kubernetes resources managed by the Machine Controller Manager.", + "fill": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 19 + }, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "mcm_machine_items_total", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "machine(s)", + "refId": "A" + }, + { + "expr": "mcm_machine_set_items_total", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "machine set(s)", + "refId": "B" + }, + { + "expr": "mcm_machine_deployment_items_total", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "machine deployment(s)", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Count of Managed Resouces", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 9, + "panels": [], + "title": "Control Loops", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "Average processing time of items in the workqueue.", + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 26 + }, + "id": 19, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "${controlloop}_work_duration{quantile=\"0.5\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "p50 ({{pod}})", + "refId": "A" + }, + { + "expr": "${controlloop}_work_duration{quantile=\"0.9\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "p90 ({{pod}})", + "refId": "B" + }, + { + "expr": "${controlloop}_work_duration{quantile=\"0.99\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "p99 ({{pod}})", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Workqueue item processing time: ${controlloop}", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "How long items stay in the workqueue before they get processed.", + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 26 + }, + "id": 18, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "${controlloop}_queue_latency{quantile=\"0.5\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "p50 ({{pod}})", + "refId": "A" + }, + { + "expr": "${controlloop}_queue_latency{quantile=\"0.9\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "p90 ({{pod}})", + "refId": "B" + }, + { + "expr": "${controlloop}_queue_latency{quantile=\"0.99\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "p99 ({{pod}})", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Workqueue item latency: ${controlloop}", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "Current amount of items in the workqueue.", + "fill": 1, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 33 + }, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "${controlloop}_depth", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "${controlloop} ({{pod}})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Items in Workqueue: ${controlloop}", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "Average per second rate over 5m of workqueue item adds.", + "fill": 1, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 33 + }, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(${controlloop}_adds[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "${controlloop} ({{pod}})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Adds to Workqueue: ${controlloop}", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "Average per second rate over 5m of workqueue item retries.", + "fill": 1, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 33 + }, + "id": 17, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(${controlloop}_retries[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "${controlloop} ({{pod}})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Workqueue item retries: ${controlloop}", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "30s", + "schemaVersion": 18, + "style": "dark", + "tags": [ + "controlplane", + "seed" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "tags": [], + "text": "machine", + "value": "machine" + }, + "hide": 0, + "includeAll": false, + "label": "Control Loop", + "multi": false, + "name": "controlloop", + "options": [ + { + "selected": true, + "text": "machine", + "value": "machine" + }, + { + "selected": false, + "text": "machineset", + "value": "machineset" + }, + { + "selected": false, + "text": "machinedeployment", + "value": "machinedeployment" + }, + { + "selected": false, + "text": "node", + "value": "node" + }, + { + "selected": false, + "text": "secret", + "value": "secret" + }, + { + "selected": false, + "text": "machinesafetyapiserver", + "value": "machinesafetyapiserver" + }, + { + "selected": false, + "text": "machinesafetyorphanvms", + "value": "machinesafetyorphanvms" + }, + { + "selected": false, + "text": "machinesafetyovershooting", + "value": "machinesafetyovershooting" + } + ], + "query": "machine, machineset, machinedeployment, node, secret, machinesafetyapiserver, machinesafetyorphanvms, machinesafetyovershooting", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "14d" + ] + }, + "timezone": "", + "title": "Machine Controller Manager", + "uid": "machine-controller-manager", + "version": 1 +} diff --git a/charts/seed-monitoring/charts/plutono/templates/_helpers.tpl b/charts/seed-monitoring/charts/plutono/templates/_helpers.tpl index 18a3ccb410c..0ca77e0db9a 100644 --- a/charts/seed-monitoring/charts/plutono/templates/_helpers.tpl +++ b/charts/seed-monitoring/charts/plutono/templates/_helpers.tpl @@ -69,6 +69,12 @@ plutono-datasources-{{ include "plutono.datasources.data" . | sha256sum | trunc {{ toString $bytes | include "plutono.toCompactedJson" | indent 2 }} {{- end }} {{- end }} +{{- if .Values.gardenletManagesMCM }} +{{ range $name, $bytes := .Files.Glob "dashboards/machine-controller-manager/**.json" }} +{{ base $name }}: |- +{{ toString $bytes | include "plutono.toCompactedJson" | indent 2 }} +{{- end }} +{{- end }} {{ range $name, $bytes := .Files.Glob "dashboards/owners/worker/**.json" }} {{ base $name }}: |- {{ toString $bytes | include "plutono.toCompactedJson" | indent 2 }} diff --git a/charts/seed-monitoring/charts/plutono/values.yaml b/charts/seed-monitoring/charts/plutono/values.yaml index 07b8b0518e3..0beffd2caea 100644 --- a/charts/seed-monitoring/charts/plutono/values.yaml +++ b/charts/seed-monitoring/charts/plutono/values.yaml @@ -23,6 +23,8 @@ sni: nodeLocalDNS: enabled: false +gardenletManagesMCM: false + reversedVPN: highAvailabilityEnabled: false diff --git a/docs/deployment/feature_gates.md b/docs/deployment/feature_gates.md index 1e7c6617cd9..a91268682e0 100644 --- a/docs/deployment/feature_gates.md +++ b/docs/deployment/feature_gates.md @@ -36,6 +36,7 @@ The following tables are a summary of the feature gates that you can set on diff | FullNetworkPoliciesInRuntimeCluster | `true` | `Beta` | `1.71` | `1.72` | | FullNetworkPoliciesInRuntimeCluster | `true` | `GA` | `1.73` | | | WorkerlessShoots | `false` | `Alpha` | `1.70` | | +| MachineControllerManagerDeployment | `false` | `Alpha` | `1.73` | | ## Feature Gates for Graduated or Deprecated Features @@ -173,3 +174,4 @@ A *General Availability* (GA) feature is also referred to as a *stable* feature. | MutableShootSpecNetworkingNodes | `gardener-apiserver` | Allows updating the field `spec.networking.nodes`. The validity of the values has to be checked in the provider extensions. Only enable this feature gate when your system runs provider extensions which have implemented the validation. | | FullNetworkPoliciesInRuntimeCluster | `gardenlet`, `gardener-operator` | Enables the `NetworkPolicy` controller to place 'deny-all' network policies in all relevant namespaces in the runtime cluster. | | WorkerlessShoots | `gardener-apiserver` | WorkerlessShoots allows creation of Shoot clusters with no worker pools. | +| MachineControllerManagerDeployment | `gardenlet` | Enables Gardener to take over the deployment of the machine-controller-manager. If enabled, all registered provider extensions must support injecting the provider-specific MCM sidecar container into the deployment via the `controlplane` webhook. | diff --git a/docs/extensions/conventions.md b/docs/extensions/conventions.md index 6eb7ad98a54..3591ade2454 100644 --- a/docs/extensions/conventions.md +++ b/docs/extensions/conventions.md @@ -29,7 +29,7 @@ As there is no formal process to validate non-existence of conflicts between two *The resource name should be prefixed with `extensions.gardener.cloud:-:`*, for example: -* `extensions.gardener.cloud:provider-aws:machine-controller-manager` +* `extensions.gardener.cloud:provider-aws:some-controller-manager` * `extensions.gardener.cloud:extension-certificate-service:cert-broker` ## How to create resources in the shoot cluster? diff --git a/docs/extensions/logging-and-monitoring.md b/docs/extensions/logging-and-monitoring.md index ac9f4e44a71..355834a8cab 100644 --- a/docs/extensions/logging-and-monitoring.md +++ b/docs/extensions/logging-and-monitoring.md @@ -110,7 +110,7 @@ To contribute its own configuration to the fluent-bit agents data pipelines, an > **Note:** Take care to provide the correct data pipeline elements in the corresponding fields and not to mix them. -**Example:** Logging configuration for provider-specific (OpenStack) worker controller deploying a `machine-controller-manager` component into a shoot namespace that reuses the `kube-apiserver-parser` defined in [logging.go](../../pkg/component/kubeapiserver/logging.go) to parse the component logs: +**Example:** Logging configuration for provider-specific `cloud-controller-manager` deployed into shoot namespaces that reuses the `kube-apiserver-parser` defined in [logging.go](../../pkg/component/kubeapiserver/logging.go) to parse the component logs: ```yaml apiVersion: fluentbit.fluent.io/v1alpha2 @@ -118,14 +118,14 @@ kind: ClusterFilter metadata: labels: fluentbit.gardener/type: "seed" - name: machine-controller-manager-openstack + name: cloud-controller-manager-aws-cloud-controller-manager spec: filters: - parser: keyName: log parser: kube-apiserver-parser reserveData: true - match: kubernetes.*machine-controller-manager*openstack* + match: kubernetes.*cloud-controller-manager*aws-cloud-controller-manager* ``` Further details how to define parsers and use them with examples can be found in the following [guide](../development/log_parsers.md). diff --git a/docs/extensions/worker.md b/docs/extensions/worker.md index a884f1414ae..9806ce9b214 100644 --- a/docs/extensions/worker.md +++ b/docs/extensions/worker.md @@ -11,8 +11,9 @@ Generally, there are provider-specific `MachineClass` objects (`AWSMachineClass` A machine class describes **where** and **how** to create virtual machines (in which networks, region, availability zone, SSH key, user-data for bootstrapping, etc.), while a `Machine` results in an actual virtual machine. You can read up [more information](https://github.com/gardener/machine-controller-manager) in the machine-controller-manager's [repository](https://github.com/gardener/machine-controller-manager). -Before the introduction of the `Worker` extension resource, Gardener was deploying the machine-controller-manager, the machine classes, and the machine deployments itself. -Now, Gardener commissions an external, provider-specific controller to take over these tasks. +The `gardenlet` has a feature gate `MachineControllerManagerDeployment` which controls whether it is deploying the `machine-controller-manager`. +If set to `true`, provider extensions only have to inject their specific out-of-tree `machine-controller-manager` sidecar container into the `Deployment`. +If set to `false`, provider extensions have to take care of th full deployment (including the generic `machine-controller-manager` container). ## What needs to be implemented to support a new worker provider? @@ -116,7 +117,8 @@ The `spec.pools[].nodeTemplate.capacity` field contains the resource information The `spec.pools[].machineControllerManager` field allows to configure the settings for machine-controller-manager component. Providers must populate these settings on worker-pool to the related [fields](https://github.com/gardener/machine-controller-manager/blob/master/kubernetes/machine_objects/machine-deployment.yaml#L30-L34) in MachineDeployment. -When seeing such a resource, your controller must make sure that it deploys the machine-controller-manager next to the control plane in the seed cluster. +When observing such a resource, the controller must make sure that it deploys the machine-controller-manager next to the control plane in the seed cluster (only if `gardenlet`'s `MachineControllerManagerDeployment` feature gate is disabled). If the feature gate is enabled, the controller must only inject its provider-specific sidecar container into the `machine-controller-manager` `Deployment` managed by `gardenlet`. + After that, it must compute the desired machine classes and the desired machine deployments. Typically, one class maps to one deployment, and one class/deployment is created per availability zone. Following this convention, the created resource would look like this: @@ -307,8 +309,9 @@ You can take a look at the below referenced example implementation for the AWS p ## That sounds like a lot that needs to be done, can you help me? All of the described behaviour is mostly the same for every provider. -The only difference is maybe the version/configuration of the machine-controller-manager, and the machine class specification itself. +The only difference is maybe the version/configuration of the provider-specific `machine-controller-manager` sidecar container, and the machine class specification itself. You can take a look at our [extension library](https://github.com/gardener/gardener/blob/master/extensions), especially the [worker controller](../../extensions/pkg/controller/worker) part where you will find a lot of utilities that you can use. +Note that there are also utility functions for getting the default sidecar container specification or corresponding VPA container policy in the [`machinecontrollermanager` package](../../pkg/component/machinecontrollermanager) called `ProviderSidecarContainer` and `ProviderSidecarVPAContainerPolicy`. Also, using the library you only need to implement your provider specifics - all the things that can be handled generically can be taken for free and do not need to be re-implemented. Take a look at the [AWS worker controller](https://github.com/gardener/gardener-extension-provider-aws/tree/master/pkg/controller/worker) for finding an example. diff --git a/extensions/pkg/controller/cmd/options.go b/extensions/pkg/controller/cmd/options.go index 7e0f06126d6..a87233946b8 100644 --- a/extensions/pkg/controller/cmd/options.go +++ b/extensions/pkg/controller/cmd/options.go @@ -73,6 +73,9 @@ const ( // GardenerVersionFlag is the name of the command line flag containing the Gardener version. GardenerVersionFlag = "gardener-version" + // GardenletManagesMCMFlag is the name of the command line flag containing the Gardener version. + // TODO(rfranzke): Remove this flag when all provider extensions support the feature, see https://github.com/gardener/gardener/issues/7594. + GardenletManagesMCMFlag = "gardenlet-manages-mcm" // LogLevelFlag is the name of the command line flag containing the log level. LogLevelFlag = "log-level" @@ -491,21 +494,25 @@ type SwitchConfig struct { // GeneralOptions are command line options that can be set for general configuration. type GeneralOptions struct { - // GardenerVersion string + // GardenerVersion is the version of the Gardener. GardenerVersion string + // GardenletManagesMCM specifies whether gardenlet manages the machine-controller-manager. + GardenletManagesMCM bool config *GeneralConfig } // GeneralConfig is a completed general configuration. type GeneralConfig struct { - // GardenerVersion string + // GardenerVersion is the version of the Gardener. GardenerVersion string + // GardenletManagesMCM specifies whether gardenlet manages the machine-controller-manager. + GardenletManagesMCM bool } // Complete implements Complete. func (r *GeneralOptions) Complete() error { - r.config = &GeneralConfig{r.GardenerVersion} + r.config = &GeneralConfig{r.GardenerVersion, r.GardenletManagesMCM} return nil } @@ -517,4 +524,5 @@ func (r *GeneralOptions) Completed() *GeneralConfig { // AddFlags implements Flagger.AddFlags. func (r *GeneralOptions) AddFlags(fs *pflag.FlagSet) { fs.StringVar(&r.GardenerVersion, GardenerVersionFlag, "", "Version of the gardenlet.") + fs.BoolVar(&r.GardenletManagesMCM, GardenletManagesMCMFlag, false, "Specifies whether gardenlet manages the machine-controller-manager.") } diff --git a/extensions/pkg/controller/worker/genericactuator/actuator.go b/extensions/pkg/controller/worker/genericactuator/actuator.go index fe6c2cb94f7..ef2ef4147af 100644 --- a/extensions/pkg/controller/worker/genericactuator/actuator.go +++ b/extensions/pkg/controller/worker/genericactuator/actuator.go @@ -51,6 +51,7 @@ const GardenPurposeMachineClass = "machineclass" type genericActuator struct { delegateFactory DelegateFactory + mcmManaged bool mcmName string mcmSeedChart chart.Interface mcmShootChart chart.Interface @@ -68,6 +69,7 @@ type genericActuator struct { // NewActuator creates a new Actuator that reconciles // Worker resources of Gardener's `extensions.gardener.cloud` API group. // It provides a default implementation that allows easier integration of providers. +// If machine-controller-manager should not be managed then only the delegateFactory must be provided. func NewActuator( delegateFactory DelegateFactory, mcmName string, @@ -78,6 +80,7 @@ func NewActuator( ) worker.Actuator { return &genericActuator{ delegateFactory: delegateFactory, + mcmManaged: mcmName != "" && mcmSeedChart != nil && mcmShootChart != nil && imageVector != nil && chartRendererFactory != nil, mcmName: mcmName, mcmSeedChart: mcmSeedChart, mcmShootChart: mcmShootChart, diff --git a/extensions/pkg/controller/worker/genericactuator/actuator_migrate.go b/extensions/pkg/controller/worker/genericactuator/actuator_migrate.go index 1ed1c0d5dbb..7d761d10580 100644 --- a/extensions/pkg/controller/worker/genericactuator/actuator_migrate.go +++ b/extensions/pkg/controller/worker/genericactuator/actuator_migrate.go @@ -42,13 +42,15 @@ func (a *genericActuator) Migrate(ctx context.Context, log logr.Logger, worker * return fmt.Errorf("could not keep objects of managed resource containing mcm chart for worker '%s': %w", kubernetesutils.ObjectName(worker), err) } - // Make sure machine-controller-manager is deleted before deleting the machines. - if err := a.deleteMachineControllerManager(ctx, log, worker); err != nil { - return fmt.Errorf("failed deleting machine-controller-manager: %w", err) - } - - if err := a.waitUntilMachineControllerManagerIsDeleted(ctx, log, worker.Namespace); err != nil { - return fmt.Errorf("failed deleting machine-controller-manager: %w", err) + if a.mcmManaged { + // Make sure machine-controller-manager is deleted before deleting the machines. + if err := a.deleteMachineControllerManager(ctx, log, worker); err != nil { + return fmt.Errorf("failed deleting machine-controller-manager: %w", err) + } + + if err := a.waitUntilMachineControllerManagerIsDeleted(ctx, log, worker.Namespace); err != nil { + return fmt.Errorf("failed deleting machine-controller-manager: %w", err) + } } if err := a.shallowDeleteAllObjects(ctx, log, worker.Namespace, &machinev1alpha1.MachineList{}); err != nil { diff --git a/extensions/pkg/controller/worker/genericactuator/actuator_restore.go b/extensions/pkg/controller/worker/genericactuator/actuator_restore.go index e8f4695d841..625a8cede2b 100644 --- a/extensions/pkg/controller/worker/genericactuator/actuator_restore.go +++ b/extensions/pkg/controller/worker/genericactuator/actuator_restore.go @@ -93,6 +93,13 @@ func RestoreWithoutReconcile( return fmt.Errorf("failed to restore the machine deployment config: %w", err) } + // Scale the machine-controller-manager to 1 now that all resources have been restored. + if !extensionscontroller.IsHibernated(cluster) { + if err := scaleMachineControllerManager(ctx, log, cl, worker, 1); err != nil { + return fmt.Errorf("failed to scale up machine-controller-manager: %w", err) + } + } + return nil } diff --git a/extensions/pkg/controller/worker/genericactuator/machine_controller_manager.go b/extensions/pkg/controller/worker/genericactuator/machine_controller_manager.go index 9270859ddec..02f467db726 100644 --- a/extensions/pkg/controller/worker/genericactuator/machine_controller_manager.go +++ b/extensions/pkg/controller/worker/genericactuator/machine_controller_manager.go @@ -48,6 +48,14 @@ const ( type ReplicaCount func() int32 func (a *genericActuator) deployMachineControllerManager(ctx context.Context, logger logr.Logger, workerObj *extensionsv1alpha1.Worker, cluster *extensionscontroller.Cluster, workerDelegate WorkerDelegate, replicas ReplicaCount) error { + if !a.mcmManaged { + logger.Info("Skip machine-controller-manager deployment since gardenlet manages it - deleting monitoring ConfigMap and extension-worker-mcm-shoot ManagedResource") + if err := a.client.Delete(ctx, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "machine-controller-manager-monitoring-config", Namespace: workerObj.Namespace}}); client.IgnoreNotFound(err) != nil { + return err + } + return managedresources.Delete(ctx, a.client, workerObj.Namespace, McmShootResourceName, false) + } + logger.Info("Deploying the machine-controller-manager") mcmValues, err := workerDelegate.GetMachineControllerManagerChartValues(ctx) @@ -71,7 +79,7 @@ func (a *genericActuator) deployMachineControllerManager(ctx context.Context, lo return fmt.Errorf("could not apply MCM chart in seed for worker '%s': %w", kubernetesutils.ObjectName(workerObj), err) } - if err := a.applyMachineControllerManagerShootChart(ctx, workerDelegate, workerObj, cluster); err != nil { + if err := a.applyMachineControllerManagerShootChart(ctx, logger, workerDelegate, workerObj, cluster); err != nil { return fmt.Errorf("could not apply machine-controller-manager shoot chart: %w", err) } @@ -84,6 +92,14 @@ func (a *genericActuator) deployMachineControllerManager(ctx context.Context, lo } func (a *genericActuator) deleteMachineControllerManager(ctx context.Context, logger logr.Logger, workerObj *extensionsv1alpha1.Worker) error { + if !a.mcmManaged { + logger.Info("Skip machine-controller-manager deployment since gardenlet manages it - deleting monitoring ConfigMap and extension-worker-mcm-shoot ManagedResource") + if err := a.client.Delete(ctx, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "machine-controller-manager-monitoring-config", Namespace: workerObj.Namespace}}); client.IgnoreNotFound(err) != nil { + return err + } + return managedresources.Delete(ctx, a.client, workerObj.Namespace, McmShootResourceName, false) + } + logger.Info("Deleting the machine-controller-manager") if err := managedresources.Delete(ctx, a.client, workerObj.Namespace, McmShootResourceName, false); err != nil { return fmt.Errorf("could not delete managed resource containing mcm chart for worker '%s': %w", kubernetesutils.ObjectName(workerObj), err) @@ -123,7 +139,12 @@ func scaleMachineControllerManager(ctx context.Context, logger logr.Logger, cl c return client.IgnoreNotFound(kubernetes.ScaleDeployment(ctx, cl, kubernetesutils.Key(worker.Namespace, McmDeploymentName), replicas)) } -func (a *genericActuator) applyMachineControllerManagerShootChart(ctx context.Context, workerDelegate WorkerDelegate, workerObj *extensionsv1alpha1.Worker, cluster *controller.Cluster) error { +func (a *genericActuator) applyMachineControllerManagerShootChart(ctx context.Context, logger logr.Logger, workerDelegate WorkerDelegate, workerObj *extensionsv1alpha1.Worker, cluster *controller.Cluster) error { + if !a.mcmManaged { + logger.Info("Skip machine-controller-manager shoot chart application since gardenlet manages it") + return nil + } + // Create shoot chart renderer chartRenderer, err := a.chartRendererFactory.NewChartRendererForShoot(cluster.Shoot.Spec.Kubernetes.Version) if err != nil { diff --git a/extensions/pkg/webhook/controlplane/genericmutator/mock/mocks.go b/extensions/pkg/webhook/controlplane/genericmutator/mock/mocks.go index e6323d40895..bfabe77ca8c 100644 --- a/extensions/pkg/webhook/controlplane/genericmutator/mock/mocks.go +++ b/extensions/pkg/webhook/controlplane/genericmutator/mock/mocks.go @@ -16,6 +16,7 @@ import ( gomock "github.com/golang/mock/gomock" v1 "k8s.io/api/apps/v1" v10 "k8s.io/api/core/v1" + v11 "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" v1beta1 "k8s.io/kubelet/config/v1beta1" ) @@ -211,6 +212,34 @@ func (mr *MockEnsurerMockRecorder) EnsureKubernetesGeneralConfiguration(arg0, ar return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EnsureKubernetesGeneralConfiguration", reflect.TypeOf((*MockEnsurer)(nil).EnsureKubernetesGeneralConfiguration), arg0, arg1, arg2, arg3) } +// EnsureMachineControllerManagerDeployment mocks base method. +func (m *MockEnsurer) EnsureMachineControllerManagerDeployment(arg0 context.Context, arg1 context0.GardenContext, arg2, arg3 *v1.Deployment) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "EnsureMachineControllerManagerDeployment", arg0, arg1, arg2, arg3) + ret0, _ := ret[0].(error) + return ret0 +} + +// EnsureMachineControllerManagerDeployment indicates an expected call of EnsureMachineControllerManagerDeployment. +func (mr *MockEnsurerMockRecorder) EnsureMachineControllerManagerDeployment(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EnsureMachineControllerManagerDeployment", reflect.TypeOf((*MockEnsurer)(nil).EnsureMachineControllerManagerDeployment), arg0, arg1, arg2, arg3) +} + +// EnsureMachineControllerManagerVPA mocks base method. +func (m *MockEnsurer) EnsureMachineControllerManagerVPA(arg0 context.Context, arg1 context0.GardenContext, arg2, arg3 *v11.VerticalPodAutoscaler) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "EnsureMachineControllerManagerVPA", arg0, arg1, arg2, arg3) + ret0, _ := ret[0].(error) + return ret0 +} + +// EnsureMachineControllerManagerVPA indicates an expected call of EnsureMachineControllerManagerVPA. +func (mr *MockEnsurerMockRecorder) EnsureMachineControllerManagerVPA(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EnsureMachineControllerManagerVPA", reflect.TypeOf((*MockEnsurer)(nil).EnsureMachineControllerManagerVPA), arg0, arg1, arg2, arg3) +} + // EnsureVPNSeedServerDeployment mocks base method. func (m *MockEnsurer) EnsureVPNSeedServerDeployment(arg0 context.Context, arg1 context0.GardenContext, arg2, arg3 *v1.Deployment) error { m.ctrl.T.Helper() diff --git a/extensions/pkg/webhook/controlplane/genericmutator/mutator.go b/extensions/pkg/webhook/controlplane/genericmutator/mutator.go index 0bb3e5819b3..528f17f79f6 100644 --- a/extensions/pkg/webhook/controlplane/genericmutator/mutator.go +++ b/extensions/pkg/webhook/controlplane/genericmutator/mutator.go @@ -25,6 +25,7 @@ import ( "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + vpaautoscalingv1 "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" @@ -38,10 +39,11 @@ import ( extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1" "github.com/gardener/gardener/pkg/component/extensions/operatingsystemconfig/original/components/kubelet" "github.com/gardener/gardener/pkg/component/extensions/operatingsystemconfig/utils" + "github.com/gardener/gardener/pkg/component/machinecontrollermanager" kubernetesutils "github.com/gardener/gardener/pkg/utils/kubernetes" ) -// Ensurer ensures that various standard Kubernets controlplane objects conform to the provider requirements. +// Ensurer ensures that various standard Kubernetes control plane objects conform to the provider requirements. // If they don't initially, they are mutated accordingly. type Ensurer interface { // EnsureKubeAPIServerService ensures that the kube-apiserver service conforms to the provider requirements. @@ -59,6 +61,12 @@ type Ensurer interface { // EnsureClusterAutoscalerDeployment ensures that the cluster-autoscaler deployment conforms to the provider requirements. // "old" might be "nil" and must always be checked. EnsureClusterAutoscalerDeployment(ctx context.Context, gctx extensionscontextwebhook.GardenContext, new, old *appsv1.Deployment) error + // EnsureMachineControllerManagerDeployment ensures that the machine-controller-manager deployment conforms to the provider requirements. + // "old" might be "nil" and must always be checked. + EnsureMachineControllerManagerDeployment(ctx context.Context, gctx extensionscontextwebhook.GardenContext, new, old *appsv1.Deployment) error + // EnsureMachineControllerManagerVPA ensures that the machine-controller-manager VPA settings conform to the provider requirements. + // "old" might be "nil" and must always be checked. + EnsureMachineControllerManagerVPA(ctx context.Context, gctx extensionscontextwebhook.GardenContext, new, old *vpaautoscalingv1.VerticalPodAutoscaler) error // EnsureETCD ensures that the etcds conform to the respective provider requirements. // "old" might be "nil" and must always be checked. EnsureETCD(ctx context.Context, gctx extensionscontextwebhook.GardenContext, new, old *druidv1alpha1.Etcd) error @@ -166,6 +174,9 @@ func (m *mutator) Mutate(ctx context.Context, new, old client.Object) error { case v1beta1constants.DeploymentNameKubeScheduler: extensionswebhook.LogMutation(m.logger, x.Kind, x.Namespace, x.Name) return m.ensurer.EnsureKubeSchedulerDeployment(ctx, gctx, x, oldDep) + case v1beta1constants.DeploymentNameMachineControllerManager: + extensionswebhook.LogMutation(m.logger, x.Kind, x.Namespace, x.Name) + return m.ensurer.EnsureMachineControllerManagerDeployment(ctx, gctx, x, oldDep) case v1beta1constants.DeploymentNameClusterAutoscaler: extensionswebhook.LogMutation(m.logger, x.Kind, x.Namespace, x.Name) return m.ensurer.EnsureClusterAutoscalerDeployment(ctx, gctx, x, oldDep) @@ -173,6 +184,20 @@ func (m *mutator) Mutate(ctx context.Context, new, old client.Object) error { extensionswebhook.LogMutation(m.logger, x.Kind, x.Namespace, x.Name) return m.ensurer.EnsureVPNSeedServerDeployment(ctx, gctx, x, oldDep) } + case *vpaautoscalingv1.VerticalPodAutoscaler: + var oldVPA *vpaautoscalingv1.VerticalPodAutoscaler + if old != nil { + var ok bool + oldVPA, ok = old.(*vpaautoscalingv1.VerticalPodAutoscaler) + if !ok { + return errors.New("could not cast old object to vpaautoscalingv1.VerticalPodAutoscaler") + } + } + switch x.Name { + case machinecontrollermanager.VPAName: + extensionswebhook.LogMutation(m.logger, x.Kind, x.Namespace, x.Name) + return m.ensurer.EnsureMachineControllerManagerVPA(ctx, gctx, x, oldVPA) + } case *druidv1alpha1.Etcd: switch x.Name { case v1beta1constants.ETCDMain, v1beta1constants.ETCDEvents: diff --git a/extensions/pkg/webhook/controlplane/genericmutator/mutator_test.go b/extensions/pkg/webhook/controlplane/genericmutator/mutator_test.go index ea3583e4f04..dd9f138dfd5 100644 --- a/extensions/pkg/webhook/controlplane/genericmutator/mutator_test.go +++ b/extensions/pkg/webhook/controlplane/genericmutator/mutator_test.go @@ -29,6 +29,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + vpaautoscalingv1 "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" @@ -150,10 +151,15 @@ var _ = Describe("Mutator", func() { nil, ), Entry( - "other deployments than kube-apiserver, kube-controller-manager, and kube-scheduler", + "other deployments than kube-apiserver, kube-controller-manager, machine-controller-manager, and kube-scheduler", &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "test"}}, nil, ), + Entry( + "other VPAs than machine-controller-manager", + &vpaautoscalingv1.VerticalPodAutoscaler{ObjectMeta: metav1.ObjectMeta{Name: "test"}}, + nil, + ), Entry( "other etcds than etcd-main and etcd-events", &druidv1alpha1.Etcd{ObjectMeta: metav1.ObjectMeta{Name: "test"}}, @@ -234,6 +240,20 @@ var _ = Describe("Mutator", func() { ensurer.EXPECT().EnsureClusterAutoscalerDeployment(context.TODO(), gomock.Any(), newObj, oldObj).Return(nil) }, ), + Entry( + "EnsureMachineControllerManagerDeployment with a machine-controller-manager deployment", + func() { + newObj = &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: v1beta1constants.DeploymentNameMachineControllerManager}} + ensurer.EXPECT().EnsureMachineControllerManagerDeployment(context.TODO(), gomock.Any(), newObj, oldObj).Return(nil) + }, + ), + Entry( + "EnsureMachineControllerManagerVPA with a machine-controller-manager VPA", + func() { + newObj = &vpaautoscalingv1.VerticalPodAutoscaler{ObjectMeta: metav1.ObjectMeta{Name: "machine-controller-manager-vpa"}} + ensurer.EXPECT().EnsureMachineControllerManagerVPA(context.TODO(), gomock.Any(), newObj, oldObj).Return(nil) + }, + ), Entry( "EnsureClusterAutoscalerDeployment with a cluster-autoscaler deployment and existing deployment", func() { @@ -259,11 +279,11 @@ var _ = Describe("Mutator", func() { ), ) - DescribeTable("EnsureETCD", func(new, old *druidv1alpha1.Etcd) { + DescribeTable("EnsureETCD", func(newObj, oldObj *druidv1alpha1.Etcd) { client := mockclient.NewMockClient(ctrl) client.EXPECT().Get(context.TODO(), clusterKey, &extensionsv1alpha1.Cluster{}).DoAndReturn(clientGet(clusterObject(cluster))) - ensurer.EXPECT().EnsureETCD(context.TODO(), gomock.Any(), new, old).Return(nil).Do(func(ctx context.Context, gctx extensionscontextwebhook.GardenContext, new, old *druidv1alpha1.Etcd) { + ensurer.EXPECT().EnsureETCD(context.TODO(), gomock.Any(), newObj, oldObj).Return(nil).Do(func(ctx context.Context, gctx extensionscontextwebhook.GardenContext, new, old *druidv1alpha1.Etcd) { _, err := gctx.GetCluster(ctx) if err != nil { logger.Error(err, "Failed to get cluster object") @@ -274,7 +294,7 @@ var _ = Describe("Mutator", func() { Expect(err).To(Not(HaveOccurred())) // Call Mutate method and check the result - err = mutator.Mutate(context.TODO(), new, old) + err = mutator.Mutate(context.TODO(), newObj, oldObj) Expect(err).To(Not(HaveOccurred())) }, Entry( diff --git a/extensions/pkg/webhook/controlplane/genericmutator/noopensurer.go b/extensions/pkg/webhook/controlplane/genericmutator/noopensurer.go index 2ae45c8c7f0..b289dafa326 100644 --- a/extensions/pkg/webhook/controlplane/genericmutator/noopensurer.go +++ b/extensions/pkg/webhook/controlplane/genericmutator/noopensurer.go @@ -22,6 +22,7 @@ import ( druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + vpaautoscalingv1 "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" extensionscontextwebhook "github.com/gardener/gardener/extensions/pkg/webhook/context" @@ -53,6 +54,16 @@ func (e *NoopEnsurer) EnsureKubeSchedulerDeployment(ctx context.Context, gctx ex return nil } +// EnsureMachineControllerManagerDeployment ensures that the machine-controller-manager deployment conforms to the provider requirements. +func (e *NoopEnsurer) EnsureMachineControllerManagerDeployment(ctx context.Context, gctx extensionscontextwebhook.GardenContext, new, old *appsv1.Deployment) error { + return nil +} + +// EnsureMachineControllerManagerVPA ensures that the machine-controller-manager deployment conforms to the provider requirements. +func (e *NoopEnsurer) EnsureMachineControllerManagerVPA(ctx context.Context, gctx extensionscontextwebhook.GardenContext, new, old *vpaautoscalingv1.VerticalPodAutoscaler) error { + return nil +} + // EnsureClusterAutoscalerDeployment ensures that the cluster-autoscaler deployment conforms to the provider requirements. func (e *NoopEnsurer) EnsureClusterAutoscalerDeployment(ctx context.Context, gctx extensionscontextwebhook.GardenContext, new, old *appsv1.Deployment) error { return nil diff --git a/extensions/pkg/webhook/utils.go b/extensions/pkg/webhook/utils.go index ab31ad9adac..afeb8c5b8ea 100644 --- a/extensions/pkg/webhook/utils.go +++ b/extensions/pkg/webhook/utils.go @@ -22,6 +22,7 @@ import ( "github.com/coreos/go-systemd/v22/unit" "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" + vpaautoscalingv1 "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1" ) @@ -227,6 +228,17 @@ func EnsureNoContainerWithName(items []corev1.Container, name string) []corev1.C return items } +// EnsureVPAContainerResourcePolicyWithName ensures that a container policy with a name equal to the name of the given +// container policy exists in the given slice and is equal to the given container policy. +func EnsureVPAContainerResourcePolicyWithName(items []vpaautoscalingv1.ContainerResourcePolicy, item vpaautoscalingv1.ContainerResourcePolicy) []vpaautoscalingv1.ContainerResourcePolicy { + if i := vpaContainerResourcePolicyWithNameIndex(items, item.ContainerName); i < 0 { + items = append(items, item) + } else if !reflect.DeepEqual(items[i], item) { + items = append(append(items[:i], item), items[i+1:]...) + } + return items +} + // EnsurePVCWithName ensures that a PVC with a name equal to the name of the given PVC exists // in the given slice and is equal to the given PVC. func EnsurePVCWithName(items []corev1.PersistentVolumeClaim, item corev1.PersistentVolumeClaim) []corev1.PersistentVolumeClaim { @@ -303,6 +315,15 @@ func containerWithNameIndex(items []corev1.Container, name string) int { return -1 } +func vpaContainerResourcePolicyWithNameIndex(items []vpaautoscalingv1.ContainerResourcePolicy, name string) int { + for i, item := range items { + if item.ContainerName == name { + return i + } + } + return -1 +} + func unitWithNameIndex(items []extensionsv1alpha1.Unit, name string) int { for i, item := range items { if item.Name == name { diff --git a/pkg/component/machinecontrollermanager/machine_controller_manager.go b/pkg/component/machinecontrollermanager/machine_controller_manager.go index 6dddf4e8dc1..6a31b37942e 100644 --- a/pkg/component/machinecontrollermanager/machine_controller_manager.go +++ b/pkg/component/machinecontrollermanager/machine_controller_manager.go @@ -59,6 +59,8 @@ const ( containerName = "machine-controller-manager" serviceName = "machine-controller-manager" managedResourceTargetName = "shoot-core-machine-controller-manager" + // VPAName is the name of the vertical pod autoscaler for the machine-controller-manager. + VPAName = "machine-controller-manager-vpa" ) // Interface contains functions for a machine-controller-manager deployer. @@ -489,7 +491,7 @@ func (m *machineControllerManager) emptyPodDisruptionBudget() client.Object { } func (m *machineControllerManager) emptyVPA() *vpaautoscalingv1.VerticalPodAutoscaler { - return &vpaautoscalingv1.VerticalPodAutoscaler{ObjectMeta: metav1.ObjectMeta{Name: "machine-controller-manager-vpa", Namespace: m.namespace}} + return &vpaautoscalingv1.VerticalPodAutoscaler{ObjectMeta: metav1.ObjectMeta{Name: VPAName, Namespace: m.namespace}} } func (m *machineControllerManager) emptyManagedResource() *resourcesv1alpha1.ManagedResource { diff --git a/pkg/component/machinecontrollermanager/provider.go b/pkg/component/machinecontrollermanager/provider.go new file mode 100644 index 00000000000..c8cd4ffedbd --- /dev/null +++ b/pkg/component/machinecontrollermanager/provider.go @@ -0,0 +1,87 @@ +// Copyright 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package machinecontrollermanager + +import ( + "strconv" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/intstr" + vpaautoscalingv1 "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" + + gardenerutils "github.com/gardener/gardener/pkg/utils/gardener" +) + +// ProviderSidecarContainer returns a corev1.Container object which can be injected into the machine-controller-manager +// deployment managed by the gardenlet. This function can be used in provider-specific control plane webhook +// implementations when the standard sidecar container is required. +func ProviderSidecarContainer(namespace, providerName, image string) corev1.Container { + const metricsPort = 10259 + return corev1.Container{ + Name: providerSidecarContainerName(providerName), + Image: image, + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{ + "./machine-controller", + "--control-kubeconfig=inClusterConfig", + "--machine-creation-timeout=20m", + "--machine-drain-timeout=2h", + "--machine-health-timeout=10m", + "--machine-safety-apiserver-statuscheck-timeout=30s", + "--machine-safety-apiserver-statuscheck-period=1m", + "--machine-safety-orphan-vms-period=30m", + "--namespace=" + namespace, + "--port=" + strconv.Itoa(metricsPort), + "--target-kubeconfig=" + gardenerutils.PathGenericKubeconfig, + "--v=3", + }, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/healthz", + Port: intstr.FromInt(metricsPort), + Scheme: corev1.URISchemeHTTP, + }, + }, + InitialDelaySeconds: 30, + TimeoutSeconds: 5, + PeriodSeconds: 10, + SuccessThreshold: 1, + FailureThreshold: 3, + }, + VolumeMounts: []corev1.VolumeMount{{ + Name: "kubeconfig", + MountPath: gardenerutils.VolumeMountPathGenericKubeconfig, + ReadOnly: true, + }}, + } +} + +// ProviderSidecarVPAContainerPolicy returns a vpaautoscalingv1.ContainerResourcePolicy object which can be injected +// into the machine-controller-manager-vpa VPA managed by the gardenlet. This function can be used in provider-specific +// control plane webhook implementations when the standard container policy for the sidecar is required. +func ProviderSidecarVPAContainerPolicy(providerName string, minAllowed, maxAllowed corev1.ResourceList) vpaautoscalingv1.ContainerResourcePolicy { + ccv := vpaautoscalingv1.ContainerControlledValuesRequestsOnly + return vpaautoscalingv1.ContainerResourcePolicy{ + ContainerName: providerSidecarContainerName(providerName), + ControlledValues: &ccv, + MinAllowed: minAllowed, + MaxAllowed: maxAllowed, + } +} + +func providerSidecarContainerName(providerName string) string { + return "machine-controller-manager-" + providerName +} diff --git a/pkg/component/machinecontrollermanager/provider_test.go b/pkg/component/machinecontrollermanager/provider_test.go new file mode 100644 index 00000000000..786544b273c --- /dev/null +++ b/pkg/component/machinecontrollermanager/provider_test.go @@ -0,0 +1,91 @@ +// Copyright 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package machinecontrollermanager_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/util/intstr" + vpaautoscalingv1 "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" + + . "github.com/gardener/gardener/pkg/component/machinecontrollermanager" +) + +var _ = Describe("Provider", func() { + var provider = "provider-test" + + It("should return a default provider-specific sidecar container object", func() { + var ( + namespace = "test-namespace" + image = "provider-test:latest" + ) + + Expect(ProviderSidecarContainer(namespace, provider, image)).To(Equal(corev1.Container{ + Name: "machine-controller-manager-" + provider, + Image: image, + ImagePullPolicy: "IfNotPresent", + Command: []string{ + "./machine-controller", + "--control-kubeconfig=inClusterConfig", + "--machine-creation-timeout=20m", + "--machine-drain-timeout=2h", + "--machine-health-timeout=10m", + "--machine-safety-apiserver-statuscheck-timeout=30s", + "--machine-safety-apiserver-statuscheck-period=1m", + "--machine-safety-orphan-vms-period=30m", + "--namespace=" + namespace, + "--port=10259", + "--target-kubeconfig=/var/run/secrets/gardener.cloud/shoot/generic-kubeconfig/kubeconfig", + "--v=3", + }, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/healthz", + Port: intstr.FromInt(10259), + Scheme: "HTTP", + }, + }, + InitialDelaySeconds: 30, + TimeoutSeconds: 5, + PeriodSeconds: 10, + SuccessThreshold: 1, + FailureThreshold: 3, + }, + VolumeMounts: []corev1.VolumeMount{{ + Name: "kubeconfig", + MountPath: "/var/run/secrets/gardener.cloud/shoot/generic-kubeconfig", + ReadOnly: true, + }}, + })) + }) + + It("should return a default VPA container policy object for the provider-specific sidecar container", func() { + var ( + ccv = vpaautoscalingv1.ContainerControlledValuesRequestsOnly + minAllowed = corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("1M")} + maxAllowed = corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("5M")} + ) + + Expect(ProviderSidecarVPAContainerPolicy(provider, minAllowed, maxAllowed)).To(Equal(vpaautoscalingv1.ContainerResourcePolicy{ + ContainerName: "machine-controller-manager-" + provider, + ControlledValues: &ccv, + MinAllowed: minAllowed, + MaxAllowed: maxAllowed, + })) + }) +}) diff --git a/pkg/features/features.go b/pkg/features/features.go index d453249cddf..c8964330dad 100644 --- a/pkg/features/features.go +++ b/pkg/features/features.go @@ -86,6 +86,13 @@ const ( // owner: @acumino @ary1992 @shafeeqes // alpha: v1.70.0 WorkerlessShoots featuregate.Feature = "WorkerlessShoots" + + // MachineControllerManagerDeployment enables Gardener to take over the deployment of the + // machine-controller-manager. If enabled, all registered provider extensions must support injecting the + // provider-specific MCM provider sidecar container into the deployment via the `controlplane` webhook. + // owner: @rfranzke @JensAc @mreiger + // alpha: v1.73.0 + MachineControllerManagerDeployment featuregate.Feature = "MachineControllerManagerDeployment" ) // DefaultFeatureGate is the central feature gate map used by all gardener components. @@ -122,6 +129,7 @@ var allFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{ MutableShootSpecNetworkingNodes: {Default: false, PreRelease: featuregate.Alpha}, FullNetworkPoliciesInRuntimeCluster: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, WorkerlessShoots: {Default: false, PreRelease: featuregate.Alpha}, + MachineControllerManagerDeployment: {Default: false, PreRelease: featuregate.Alpha}, } // GetFeatures returns a feature gate map with the respective specifications. Non-existing feature gates are ignored. diff --git a/pkg/gardenlet/controller/managedseed/charttest/charttest.go b/pkg/gardenlet/controller/managedseed/charttest/charttest.go index 809f4bfe6e1..426b6a36466 100644 --- a/pkg/gardenlet/controller/managedseed/charttest/charttest.go +++ b/pkg/gardenlet/controller/managedseed/charttest/charttest.go @@ -392,6 +392,11 @@ func getGardenletClusterRole(labels map[string]string) *rbacv1.ClusterRole { Resources: []string{"destinationrules", "gateways", "virtualservices", "envoyfilters"}, Verbs: []string{"delete"}, }, + { + APIGroups: []string{"machine.sapcloud.io"}, + Resources: []string{"machinedeployments"}, + Verbs: []string{"list", "watch", "get"}, + }, }, } } diff --git a/pkg/gardenlet/controller/seed/seed/reconciler_delete.go b/pkg/gardenlet/controller/seed/seed/reconciler_delete.go index 269886d2d2b..b96c6be3449 100644 --- a/pkg/gardenlet/controller/seed/seed/reconciler_delete.go +++ b/pkg/gardenlet/controller/seed/seed/reconciler_delete.go @@ -52,6 +52,7 @@ import ( "github.com/gardener/gardener/pkg/component/vpa" "github.com/gardener/gardener/pkg/component/vpnauthzserver" "github.com/gardener/gardener/pkg/controllerutils" + "github.com/gardener/gardener/pkg/features" seedpkg "github.com/gardener/gardener/pkg/operation/seed" "github.com/gardener/gardener/pkg/utils/flow" "github.com/gardener/gardener/pkg/utils/managedresources" @@ -192,17 +193,18 @@ func (r *Reconciler) runDeleteSeedFlow( // setup for flow graph var ( - dnsRecord = getManagedIngressDNSRecord(log, seedClient, r.GardenNamespace, seed.GetInfo().Spec.DNS, secretData, seed.GetIngressFQDN("*"), "") - autoscaler = clusterautoscaler.NewBootstrapper(seedClient, r.GardenNamespace) - kubeAPIServerIngress = kubeapiserverexposure.NewIngress(seedClient, r.GardenNamespace, kubeapiserverexposure.IngressValues{}) - kubeAPIServerService = kubeapiserverexposure.NewInternalNameService(seedClient, r.GardenNamespace) - nginxIngress = nginxingress.New(seedClient, r.GardenNamespace, nginxingress.Values{}) - dwdWeeder = dependencywatchdog.NewBootstrapper(seedClient, r.GardenNamespace, dependencywatchdog.BootstrapperValues{Role: dependencywatchdog.RoleWeeder}) - dwdProber = dependencywatchdog.NewBootstrapper(seedClient, r.GardenNamespace, dependencywatchdog.BootstrapperValues{Role: dependencywatchdog.RoleProber}) - systemResources = seedsystem.New(seedClient, r.GardenNamespace, seedsystem.Values{}) - vpnAuthzServer = vpnauthzserver.New(seedClient, r.GardenNamespace, "", kubernetesVersion) - istioCRDs = istio.NewCRD(r.SeedClientSet.ChartApplier()) - istio = istio.NewIstio(seedClient, r.SeedClientSet.ChartRenderer(), istio.Values{ + dnsRecord = getManagedIngressDNSRecord(log, seedClient, r.GardenNamespace, seed.GetInfo().Spec.DNS, secretData, seed.GetIngressFQDN("*"), "") + clusterAutoscaler = clusterautoscaler.NewBootstrapper(seedClient, r.GardenNamespace) + machineControllerManager = machinecontrollermanager.NewBootstrapper(seedClient, r.GardenNamespace) + kubeAPIServerIngress = kubeapiserverexposure.NewIngress(seedClient, r.GardenNamespace, kubeapiserverexposure.IngressValues{}) + kubeAPIServerService = kubeapiserverexposure.NewInternalNameService(seedClient, r.GardenNamespace) + nginxIngress = nginxingress.New(seedClient, r.GardenNamespace, nginxingress.Values{}) + dwdWeeder = dependencywatchdog.NewBootstrapper(seedClient, r.GardenNamespace, dependencywatchdog.BootstrapperValues{Role: dependencywatchdog.RoleWeeder}) + dwdProber = dependencywatchdog.NewBootstrapper(seedClient, r.GardenNamespace, dependencywatchdog.BootstrapperValues{Role: dependencywatchdog.RoleProber}) + systemResources = seedsystem.New(seedClient, r.GardenNamespace, seedsystem.Values{}) + vpnAuthzServer = vpnauthzserver.New(seedClient, r.GardenNamespace, "", kubernetesVersion) + istioCRDs = istio.NewCRD(r.SeedClientSet.ChartApplier()) + istio = istio.NewIstio(seedClient, r.SeedClientSet.ChartRenderer(), istio.Values{ Istiod: istio.IstiodValues{ Enabled: !seedIsGarden, Namespace: v1beta1constants.IstioSystemNamespace, @@ -225,8 +227,12 @@ func (r *Reconciler) runDeleteSeedFlow( Dependencies: flow.NewTaskIDs(destroyDNSRecord), }) destroyClusterAutoscaler = g.Add(flow.Task{ - Name: "Destroying cluster-autoscaler", - Fn: component.OpDestroyAndWait(autoscaler).Destroy, + Name: "Destroying cluster-autoscaler resources", + Fn: component.OpDestroyAndWait(clusterAutoscaler).Destroy, + }) + destroyMachineControllerManager = g.Add(flow.Task{ + Name: "Destroying machine-controller-manager resources", + Fn: flow.TaskFn(component.OpDestroyAndWait(machineControllerManager).Destroy).DoIf(features.DefaultFeatureGate.Enabled(features.MachineControllerManagerDeployment)), }) destroyNginxIngress = g.Add(flow.Task{ Name: "Destroying nginx-ingress", @@ -274,6 +280,7 @@ func (r *Reconciler) runDeleteSeedFlow( syncPointCleanedUp = flow.NewTaskIDs( destroyNginxIngress, destroyClusterAutoscaler, + destroyMachineControllerManager, destroyDWDWeeder, destroyDWDProber, destroyKubeAPIServerIngress, diff --git a/pkg/gardenlet/controller/seed/seed/reconciler_reconcile.go b/pkg/gardenlet/controller/seed/seed/reconciler_reconcile.go index cac9a642502..e4f5b32592d 100644 --- a/pkg/gardenlet/controller/seed/seed/reconciler_reconcile.go +++ b/pkg/gardenlet/controller/seed/seed/reconciler_reconcile.go @@ -624,6 +624,10 @@ func (r *Reconciler) runReconcileSeedFlow( componentsFunctions = append(componentsFunctions, eventlogger.CentralLoggingConfiguration) } + if features.DefaultFeatureGate.Enabled(features.MachineControllerManagerDeployment) { + componentsFunctions = append(componentsFunctions, machinecontrollermanager.CentralLoggingConfiguration) + } + // Fetch component specific logging configurations for _, componentFn := range componentsFunctions { loggingConfig, err := componentFn() @@ -901,9 +905,13 @@ func (r *Reconciler) runReconcileSeedFlow( Dependencies: flow.NewTaskIDs(nginxLBReady), }) _ = g.Add(flow.Task{ - Name: "Deploying cluster-autoscaler", + Name: "Deploying cluster-autoscaler resources", Fn: clusterautoscaler.NewBootstrapper(seedClient, r.GardenNamespace).Deploy, }) + _ = g.Add(flow.Task{ + Name: "Deploying machine-controller-manager resources", + Fn: flow.TaskFn(machinecontrollermanager.NewBootstrapper(seedClient, r.GardenNamespace).Deploy).DoIf(features.DefaultFeatureGate.Enabled(features.MachineControllerManagerDeployment)), + }) _ = g.Add(flow.Task{ Name: "Deploying dependency-watchdog-weeder", Fn: dwdWeeder.Deploy, diff --git a/pkg/gardenlet/controller/shoot/shoot/reconciler_delete.go b/pkg/gardenlet/controller/shoot/shoot/reconciler_delete.go index 9473d1d07f4..c2b76f6f699 100644 --- a/pkg/gardenlet/controller/shoot/shoot/reconciler_delete.go +++ b/pkg/gardenlet/controller/shoot/shoot/reconciler_delete.go @@ -30,6 +30,7 @@ import ( v1beta1helper "github.com/gardener/gardener/pkg/apis/core/v1beta1/helper" extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1" "github.com/gardener/gardener/pkg/client/kubernetes/clientmap/keys" + "github.com/gardener/gardener/pkg/features" "github.com/gardener/gardener/pkg/operation" botanistpkg "github.com/gardener/gardener/pkg/operation/botanist" "github.com/gardener/gardener/pkg/utils/errors" @@ -402,6 +403,13 @@ func (r *Reconciler) runDeleteShootFlow(ctx context.Context, o *operation.Operat }).SkipIf(o.Shoot.IsWorkerless), Dependencies: flow.NewTaskIDs(destroyWorker), }) + _ = g.Add(flow.Task{ + Name: "Deleting machine-controller-manager", + Fn: flow.TaskFn(func(ctx context.Context) error { + return botanist.Shoot.Components.ControlPlane.MachineControllerManager.Destroy(ctx) + }).SkipIf(o.Shoot.IsWorkerless).DoIf(features.DefaultFeatureGate.Enabled(features.MachineControllerManagerDeployment)), + Dependencies: flow.NewTaskIDs(waitUntilWorkerDeleted), + }) deleteAllOperatingSystemConfigs = g.Add(flow.Task{ Name: "Deleting operating system config resources", Fn: flow.TaskFn(func(ctx context.Context) error { diff --git a/pkg/gardenlet/controller/shoot/shoot/reconciler_migrate.go b/pkg/gardenlet/controller/shoot/shoot/reconciler_migrate.go index a1f94e56d71..81b15c7098f 100644 --- a/pkg/gardenlet/controller/shoot/shoot/reconciler_migrate.go +++ b/pkg/gardenlet/controller/shoot/shoot/reconciler_migrate.go @@ -27,6 +27,7 @@ import ( v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" v1beta1helper "github.com/gardener/gardener/pkg/apis/core/v1beta1/helper" extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1" + "github.com/gardener/gardener/pkg/features" "github.com/gardener/gardener/pkg/operation" botanistpkg "github.com/gardener/gardener/pkg/operation/botanist" errorsutils "github.com/gardener/gardener/pkg/utils/errors" @@ -201,10 +202,24 @@ func (r *Reconciler) runMigrateShootFlow(ctx context.Context, o *operation.Opera Fn: flow.TaskFn(botanist.WaitUntilManagedResourcesDeleted).Timeout(10 * time.Minute), Dependencies: flow.NewTaskIDs(deleteManagedResources), }) + deleteMachineControllerManager = g.Add(flow.Task{ + Name: "Deleting machine-controller-manager", + Fn: flow.TaskFn(func(ctx context.Context) error { + return botanist.Shoot.Components.ControlPlane.MachineControllerManager.Destroy(ctx) + }).SkipIf(o.Shoot.IsWorkerless).DoIf(features.DefaultFeatureGate.Enabled(features.MachineControllerManagerDeployment)), + Dependencies: flow.NewTaskIDs(waitForManagedResourcesDeletion), + }) + waitUntilMachineControllerManagerDeleted = g.Add(flow.Task{ + Name: "Waiting until machine-controller-manager has been deleted", + Fn: flow.TaskFn(func(ctx context.Context) error { + return botanist.Shoot.Components.ControlPlane.MachineControllerManager.WaitCleanup(ctx) + }).SkipIf(o.Shoot.IsWorkerless).DoIf(features.DefaultFeatureGate.Enabled(features.MachineControllerManagerDeployment)), + Dependencies: flow.NewTaskIDs(deleteMachineControllerManager), + }) migrateExtensionResources = g.Add(flow.Task{ Name: "Migrating extension resources", Fn: botanist.MigrateExtensionResourcesInParallel, - Dependencies: flow.NewTaskIDs(waitForManagedResourcesDeletion), + Dependencies: flow.NewTaskIDs(waitUntilMachineControllerManagerDeleted), }) waitUntilExtensionResourcesMigrated = g.Add(flow.Task{ Name: "Waiting until extension resources have been migrated", diff --git a/pkg/gardenlet/controller/shoot/shoot/reconciler_reconcile.go b/pkg/gardenlet/controller/shoot/shoot/reconciler_reconcile.go index f5005506e16..ccd7c6c8ac5 100644 --- a/pkg/gardenlet/controller/shoot/shoot/reconciler_reconcile.go +++ b/pkg/gardenlet/controller/shoot/shoot/reconciler_reconcile.go @@ -29,6 +29,7 @@ import ( "github.com/gardener/gardener/pkg/component" "github.com/gardener/gardener/pkg/component/kubeapiserver" "github.com/gardener/gardener/pkg/controllerutils" + "github.com/gardener/gardener/pkg/features" "github.com/gardener/gardener/pkg/gardenlet/controller/shoot/shoot/helper" "github.com/gardener/gardener/pkg/operation" botanistpkg "github.com/gardener/gardener/pkg/operation/botanist" @@ -637,10 +638,15 @@ func (r *Reconciler) runReconcileShootFlow(ctx context.Context, o *operation.Ope Fn: flow.TaskFn(botanist.ScaleClusterAutoscalerToZero).RetryUntilTimeout(defaultInterval, defaultTimeout).SkipIf(o.Shoot.IsWorkerless).DoIf(o.Shoot.HibernationEnabled), Dependencies: flow.NewTaskIDs(deployManagedResourcesForAddons, deployManagedResourceForCloudConfigExecutor), }) + deployMachineControllerManager = g.Add(flow.Task{ + Name: "Deploying machine-controller-manager", + Fn: flow.TaskFn(botanist.DeployMachineControllerManager).SkipIf(o.Shoot.IsWorkerless).DoIf(features.DefaultFeatureGate.Enabled(features.MachineControllerManagerDeployment)), + Dependencies: flow.NewTaskIDs(deployCloudProviderSecret, deployReferencedResources, waitUntilInfrastructureReady, initializeShootClients, waitUntilOperatingSystemConfigReady, waitUntilNetworkIsReady, createNewServiceAccountSecrets, scaleClusterAutoscalerToZero), + }) deployWorker = g.Add(flow.Task{ Name: "Configuring shoot worker pools", Fn: flow.TaskFn(botanist.DeployWorker).RetryUntilTimeout(defaultInterval, defaultTimeout).SkipIf(o.Shoot.IsWorkerless), - Dependencies: flow.NewTaskIDs(deployCloudProviderSecret, deployReferencedResources, waitUntilInfrastructureReady, initializeShootClients, waitUntilOperatingSystemConfigReady, waitUntilNetworkIsReady, createNewServiceAccountSecrets, scaleClusterAutoscalerToZero), + Dependencies: flow.NewTaskIDs(deployMachineControllerManager), }) waitUntilWorkerStatusUpdate = g.Add(flow.Task{ Name: "Waiting until worker resource status is updated with latest machine deployments", @@ -654,11 +660,6 @@ func (r *Reconciler) runReconcileShootFlow(ctx context.Context, o *operation.Ope Fn: flow.TaskFn(botanist.DeployClusterAutoscaler).RetryUntilTimeout(defaultInterval, defaultTimeout).SkipIf(o.Shoot.IsWorkerless || o.Shoot.HibernationEnabled), Dependencies: flow.NewTaskIDs(waitUntilWorkerStatusUpdate, deployManagedResourcesForAddons, deployManagedResourceForCloudConfigExecutor), }) - _ = g.Add(flow.Task{ - Name: "Reconciling Plutono for Shoot in Seed for the logging stack", - Fn: flow.TaskFn(botanist.DeploySeedPlutono).RetryUntilTimeout(defaultInterval, 2*time.Minute), - Dependencies: flow.NewTaskIDs(deploySeedLogging), - }) waitUntilWorkerReady = g.Add(flow.Task{ Name: "Waiting until shoot worker nodes have been reconciled", Fn: flow.TaskFn(func(ctx context.Context) error { @@ -666,6 +667,17 @@ func (r *Reconciler) runReconcileShootFlow(ctx context.Context, o *operation.Ope }).SkipIf(o.Shoot.IsWorkerless || skipReadiness), Dependencies: flow.NewTaskIDs(deployWorker, waitUntilWorkerStatusUpdate, deployManagedResourceForCloudConfigExecutor), }) + _ = g.Add(flow.Task{ + Name: "Scaling down machine-controller-manager", + Fn: flow.TaskFn(botanist.ScaleMachineControllerManagerToZero).RetryUntilTimeout(defaultInterval, defaultTimeout).SkipIf(o.Shoot.IsWorkerless).DoIf(o.Shoot.HibernationEnabled && features.DefaultFeatureGate.Enabled(features.MachineControllerManagerDeployment)), + Dependencies: flow.NewTaskIDs(waitUntilWorkerReady), + }) + + _ = g.Add(flow.Task{ + Name: "Reconciling Plutono for Shoot in Seed for the logging stack", + Fn: flow.TaskFn(botanist.DeploySeedPlutono).RetryUntilTimeout(defaultInterval, 2*time.Minute), + Dependencies: flow.NewTaskIDs(deploySeedLogging), + }) nginxLBReady = g.Add(flow.Task{ Name: "Waiting until nginx ingress LoadBalancer is ready", Fn: flow.TaskFn(botanist.WaitUntilNginxIngressServiceIsReady).DoIf(v1beta1helper.NginxIngressEnabled(botanist.Shoot.GetInfo().Spec.Addons)).SkipIf(o.Shoot.IsWorkerless || o.Shoot.HibernationEnabled), diff --git a/pkg/gardenlet/features/features.go b/pkg/gardenlet/features/features.go index d00dbce8b78..2aba0160923 100644 --- a/pkg/gardenlet/features/features.go +++ b/pkg/gardenlet/features/features.go @@ -30,5 +30,6 @@ func RegisterFeatureGates() { features.CoreDNSQueryRewriting, features.IPv6SingleStack, features.FullNetworkPoliciesInRuntimeCluster, + features.MachineControllerManagerDeployment, ))) } diff --git a/pkg/operation/botanist/botanist.go b/pkg/operation/botanist/botanist.go index d915a1843f2..d198ba0ba3d 100644 --- a/pkg/operation/botanist/botanist.go +++ b/pkg/operation/botanist/botanist.go @@ -29,6 +29,7 @@ import ( extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1" "github.com/gardener/gardener/pkg/component/etcd" "github.com/gardener/gardener/pkg/component/logging/kuberbacproxy" + "github.com/gardener/gardener/pkg/features" "github.com/gardener/gardener/pkg/operation" gardenerutils "github.com/gardener/gardener/pkg/utils/gardener" secretsmanager "github.com/gardener/gardener/pkg/utils/secrets/manager" @@ -166,6 +167,12 @@ func New(ctx context.Context, o *operation.Operation) (*Botanist, error) { if err != nil { return nil, err } + if features.DefaultFeatureGate.Enabled(features.MachineControllerManagerDeployment) { + o.Shoot.Components.ControlPlane.MachineControllerManager, err = b.DefaultMachineControllerManager(ctx) + if err != nil { + return nil, err + } + } } // system components diff --git a/pkg/operation/botanist/machinecontrollermanager.go b/pkg/operation/botanist/machinecontrollermanager.go new file mode 100644 index 00000000000..1eec0562a85 --- /dev/null +++ b/pkg/operation/botanist/machinecontrollermanager.go @@ -0,0 +1,90 @@ +// Copyright 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package botanist + +import ( + "context" + + machinev1alpha1 "github.com/gardener/machine-controller-manager/pkg/apis/machine/v1alpha1" + "sigs.k8s.io/controller-runtime/pkg/client" + + gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" + v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + "github.com/gardener/gardener/pkg/client/kubernetes" + "github.com/gardener/gardener/pkg/component/machinecontrollermanager" + "github.com/gardener/gardener/pkg/utils/images" + "github.com/gardener/gardener/pkg/utils/imagevector" + kubernetesutils "github.com/gardener/gardener/pkg/utils/kubernetes" +) + +// DefaultMachineControllerManager returns a deployer for the machine-controller-manager. +func (b *Botanist) DefaultMachineControllerManager(ctx context.Context) (machinecontrollermanager.Interface, error) { + image, err := b.ImageVector.FindImage(images.ImageNameMachineControllerManager, imagevector.RuntimeVersion(b.SeedVersion()), imagevector.TargetVersion(b.ShootVersion())) + if err != nil { + return nil, err + } + + machineDeploymentList := &machinev1alpha1.MachineDeploymentList{} + if err := b.SeedClientSet.Client().List(ctx, machineDeploymentList, client.InNamespace(b.Shoot.SeedNamespace)); err != nil { + return nil, err + } + + var replicas int32 = 1 + switch { + // if there are any existing machine deployments present with a positive replica count then MCM is needed. + case machineDeploymentWithPositiveReplicaCountExist(machineDeploymentList): + replicas = 1 + // If the cluster is hibernated then there is no further need of MCM and therefore its desired replicas is 0 + case b.Shoot.HibernationEnabled && b.Shoot.GetInfo().Status.IsHibernated: + replicas = 0 + // If the cluster is created with hibernation enabled, then desired replicas for MCM is 0 + case b.Shoot.HibernationEnabled && (b.Shoot.GetInfo().Status.LastOperation == nil || b.Shoot.GetInfo().Status.LastOperation.Type == gardencorev1beta1.LastOperationTypeCreate): + replicas = 0 + // If shoot is either waking up or in the process of hibernation then, MCM is required and therefore its desired replicas is 1 + case b.Shoot.HibernationEnabled != b.Shoot.GetInfo().Status.IsHibernated: + replicas = 1 + } + + return machinecontrollermanager.New( + b.SeedClientSet.Client(), + b.Shoot.SeedNamespace, + b.SecretsManager, + machinecontrollermanager.Values{ + Image: image.String(), + Replicas: replicas, + RuntimeKubernetesVersion: b.Seed.KubernetesVersion, + }, + ), nil +} + +// DeployMachineControllerManager deploys the machine-controller-manager. +func (b *Botanist) DeployMachineControllerManager(ctx context.Context) error { + b.Shoot.Components.ControlPlane.MachineControllerManager.SetNamespaceUID(b.SeedNamespaceObject.UID) + return b.Shoot.Components.ControlPlane.MachineControllerManager.Deploy(ctx) +} + +// ScaleMachineControllerManagerToZero scales machine-controller-manager replicas to zero. +func (b *Botanist) ScaleMachineControllerManagerToZero(ctx context.Context) error { + return kubernetes.ScaleDeployment(ctx, b.SeedClientSet.Client(), kubernetesutils.Key(b.Shoot.SeedNamespace, v1beta1constants.DeploymentNameMachineControllerManager), 0) +} + +func machineDeploymentWithPositiveReplicaCountExist(existingMachineDeployments *machinev1alpha1.MachineDeploymentList) bool { + for _, machineDeployment := range existingMachineDeployments.Items { + if machineDeployment.Status.Replicas > 0 { + return true + } + } + return false +} diff --git a/pkg/operation/botanist/machinecontrollermanager_test.go b/pkg/operation/botanist/machinecontrollermanager_test.go new file mode 100644 index 00000000000..432cca269ad --- /dev/null +++ b/pkg/operation/botanist/machinecontrollermanager_test.go @@ -0,0 +1,226 @@ +// Copyright 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package botanist_test + +import ( + "context" + "fmt" + + "github.com/Masterminds/semver" + machinev1alpha1 "github.com/gardener/machine-controller-manager/pkg/apis/machine/v1alpha1" + "github.com/golang/mock/gomock" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + . "github.com/onsi/gomega/gstruct" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" + + gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" + "github.com/gardener/gardener/pkg/client/kubernetes" + kubernetesmock "github.com/gardener/gardener/pkg/client/kubernetes/mock" + mockmachinecontrollermanager "github.com/gardener/gardener/pkg/component/machinecontrollermanager/mock" + mockclient "github.com/gardener/gardener/pkg/mock/controller-runtime/client" + "github.com/gardener/gardener/pkg/operation" + . "github.com/gardener/gardener/pkg/operation/botanist" + seedpkg "github.com/gardener/gardener/pkg/operation/seed" + shootpkg "github.com/gardener/gardener/pkg/operation/shoot" + "github.com/gardener/gardener/pkg/utils/imagevector" + secretsmanager "github.com/gardener/gardener/pkg/utils/secrets/manager" + fakesecretsmanager "github.com/gardener/gardener/pkg/utils/secrets/manager/fake" +) + +var _ = Describe("MachineControllerManager", func() { + var ( + ctx = context.TODO() + fakeErr = fmt.Errorf("fake err") + + ctrl *gomock.Controller + kubernetesClient *kubernetesmock.MockInterface + fakeClient client.Client + fakeSecretsManager secretsmanager.Interface + + shoot *gardencorev1beta1.Shoot + deployment *appsv1.Deployment + + botanist *Botanist + namespace = "shoot--foo--bar" + ) + + BeforeEach(func() { + ctrl = gomock.NewController(GinkgoT()) + + kubernetesClient = kubernetesmock.NewMockInterface(ctrl) + fakeClient = fakeclient.NewClientBuilder().WithScheme(kubernetes.SeedScheme).Build() + fakeSecretsManager = fakesecretsmanager.New(fakeClient, namespace) + + shoot = &gardencorev1beta1.Shoot{Spec: gardencorev1beta1.ShootSpec{Kubernetes: gardencorev1beta1.Kubernetes{Version: "1.25.0"}}} + deployment = &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "machine-controller-manager", Namespace: namespace}} + + botanist = &Botanist{Operation: &operation.Operation{}} + botanist.SeedClientSet = kubernetesClient + botanist.SecretsManager = fakeSecretsManager + botanist.Seed = &seedpkg.Seed{KubernetesVersion: semver.MustParse("1.25.0")} + botanist.Shoot = &shootpkg.Shoot{SeedNamespace: namespace} + botanist.Shoot.SetInfo(shoot) + + DeferCleanup(func() { + ctrl.Finish() + }) + }) + + Describe("#DefaultMachineControllerManager", func() { + BeforeEach(func() { + kubernetesClient.EXPECT().Version() + + By("Create secrets managed outside of this package for whose secretsmanager.Get() will be called") + Expect(fakeClient.Create(ctx, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "generic-token-kubeconfig", Namespace: namespace}})).To(Succeed()) + }) + + It("should return an error because the image cannot be found", func() { + botanist.ImageVector = imagevector.ImageVector{} + + machineControllerManager, err := botanist.DefaultMachineControllerManager(ctx) + Expect(machineControllerManager).To(BeNil()) + Expect(err).To(HaveOccurred()) + }) + + DescribeTable("it should successfully create a machine-controller-manager interface", + func(expectedReplicas int, prepTest func()) { + kubernetesClient.EXPECT().Client().Return(fakeClient).Times(2) + botanist.ImageVector = imagevector.ImageVector{{Name: "machine-controller-manager"}} + + if prepTest != nil { + prepTest() + } + + machineControllerManager, err := botanist.DefaultMachineControllerManager(ctx) + Expect(machineControllerManager).NotTo(BeNil()) + Expect(err).NotTo(HaveOccurred()) + + Expect(machineControllerManager.Deploy(ctx)).To(Succeed()) + Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(deployment), deployment)).To(Succeed()) + Expect(deployment.Spec.Replicas).To(PointTo(Equal(int32(expectedReplicas)))) + }, + + Entry("with default replicas", 1, nil), + Entry("when machine deployments with positive replica count exist", 1, func() { + machineDeployment := &machinev1alpha1.MachineDeployment{ + ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: namespace}, + Spec: machinev1alpha1.MachineDeploymentSpec{Replicas: 5}, + } + Expect(fakeClient.Create(ctx, machineDeployment)).To(Succeed()) + }), + Entry("when shoot is fully hibernated", 0, func() { + botanist.Shoot.HibernationEnabled = true + shoot.Status.IsHibernated = true + botanist.Shoot.SetInfo(shoot) + }), + Entry("when shoot shall be hibernated but last operation is nil", 0, func() { + botanist.Shoot.HibernationEnabled = true + shoot.Status.LastOperation = nil + botanist.Shoot.SetInfo(shoot) + }), + Entry("when shoot shall be hibernated but last operation is create", 0, func() { + botanist.Shoot.HibernationEnabled = true + shoot.Status.LastOperation = &gardencorev1beta1.LastOperation{Type: gardencorev1beta1.LastOperationTypeCreate} + botanist.Shoot.SetInfo(shoot) + }), + Entry("when shoot shall be hibernated but last operation is not create", 1, func() { + botanist.Shoot.HibernationEnabled = true + shoot.Status.LastOperation = &gardencorev1beta1.LastOperation{Type: gardencorev1beta1.LastOperationTypeReconcile} + botanist.Shoot.SetInfo(shoot) + }), + Entry("when shoot shall be hibernated but process is not finished yet", 1, func() { + botanist.Shoot.HibernationEnabled = true + shoot.Status.LastOperation = &gardencorev1beta1.LastOperation{Type: gardencorev1beta1.LastOperationTypeReconcile} + shoot.Status.IsHibernated = false + botanist.Shoot.SetInfo(shoot) + }), + Entry("when shoot shall be woken up but process is not finished yet", 1, func() { + botanist.Shoot.HibernationEnabled = false + shoot.Status.LastOperation = &gardencorev1beta1.LastOperation{Type: gardencorev1beta1.LastOperationTypeReconcile} + shoot.Status.IsHibernated = true + botanist.Shoot.SetInfo(shoot) + }), + ) + }) + + Describe("#DeployMachineControllerManager", func() { + var ( + machineControllerManager *mockmachinecontrollermanager.MockInterface + namespaceUID = types.UID("5678") + ) + + BeforeEach(func() { + machineControllerManager = mockmachinecontrollermanager.NewMockInterface(ctrl) + machineControllerManager.EXPECT().SetNamespaceUID(namespaceUID) + + botanist.SeedNamespaceObject = &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + UID: namespaceUID, + }, + } + botanist.Shoot = &shootpkg.Shoot{ + Components: &shootpkg.Components{ + ControlPlane: &shootpkg.ControlPlane{ + MachineControllerManager: machineControllerManager, + }, + }, + } + }) + + It("should set the namespace uid and deploy", func() { + machineControllerManager.EXPECT().Deploy(ctx) + Expect(botanist.DeployMachineControllerManager(ctx)).To(Succeed()) + }) + + It("should fail when the deploy function fails", func() { + machineControllerManager.EXPECT().Deploy(ctx).Return(fakeErr) + Expect(botanist.DeployMachineControllerManager(ctx)).To(Equal(fakeErr)) + }) + }) + + Describe("#ScaleMachineControllerManagerToZero", func() { + var ( + mockClient *mockclient.MockClient + mockSubresourceClient *mockclient.MockSubResourceClient + patch = client.RawPatch(types.MergePatchType, []byte(`{"spec":{"replicas":0}}`)) + ) + + BeforeEach(func() { + mockClient = mockclient.NewMockClient(ctrl) + mockSubresourceClient = mockclient.NewMockSubResourceClient(ctrl) + + kubernetesClient.EXPECT().Client().Return(mockClient) + mockClient.EXPECT().SubResource("scale").Return(mockSubresourceClient) + + botanist.SeedClientSet = kubernetesClient + }) + + It("should scale the CA deployment", func() { + mockSubresourceClient.EXPECT().Patch(ctx, deployment, patch) + Expect(botanist.ScaleMachineControllerManagerToZero(ctx)).To(Succeed()) + }) + + It("should fail when the scale call fails", func() { + mockSubresourceClient.EXPECT().Patch(ctx, deployment, patch).Return(fakeErr) + Expect(botanist.ScaleMachineControllerManagerToZero(ctx)).To(MatchError(fakeErr)) + }) + }) +}) diff --git a/pkg/operation/botanist/monitoring.go b/pkg/operation/botanist/monitoring.go index 331be2dc820..eb2ee24b601 100644 --- a/pkg/operation/botanist/monitoring.go +++ b/pkg/operation/botanist/monitoring.go @@ -33,6 +33,7 @@ import ( "github.com/gardener/gardener/pkg/component" "github.com/gardener/gardener/pkg/component/etcd" "github.com/gardener/gardener/pkg/controllerutils" + "github.com/gardener/gardener/pkg/features" gardenlethelper "github.com/gardener/gardener/pkg/gardenlet/apis/config/helper" "github.com/gardener/gardener/pkg/operation/common" "github.com/gardener/gardener/pkg/utils" @@ -99,6 +100,10 @@ func (b *Botanist) DeploySeedMonitoring(ctx context.Context) error { if b.Shoot.WantsClusterAutoscaler { monitoringComponents = append(monitoringComponents, b.Shoot.Components.ControlPlane.ClusterAutoscaler) } + + if features.DefaultFeatureGate.Enabled(features.MachineControllerManagerDeployment) { + monitoringComponents = append(monitoringComponents, b.Shoot.Components.ControlPlane.MachineControllerManager) + } } for _, component := range monitoringComponents { @@ -195,6 +200,7 @@ func (b *Botanist) DeploySeedMonitoring(ctx context.Context) error { "nodeLocalDNS": map[string]interface{}{ "enabled": b.Shoot.NodeLocalDNSEnabled, }, + "gardenletManagesMCM": features.DefaultFeatureGate.Enabled(features.MachineControllerManagerDeployment), "ingress": map[string]interface{}{ "class": ingressClass, "authSecretName": credentialsSecret.Name, @@ -402,7 +408,7 @@ func (b *Botanist) DeploySeedPlutono(ctx context.Context) error { return kubernetesutils.DeleteObject(ctx, b.GardenClient, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: secretName, Namespace: b.Shoot.GetInfo().Namespace}}) } - //TODO(rickardsjp, istvanballok): Remove in release v1.77 once the Grafana to Plutono migration is complete. + // TODO(rickardsjp, istvanballok): Remove in release v1.77 once the Grafana to Plutono migration is complete. if err := b.DeleteGrafana(ctx); err != nil { return err } diff --git a/pkg/operation/shoot/types.go b/pkg/operation/shoot/types.go index e23f6779286..636b857601e 100644 --- a/pkg/operation/shoot/types.go +++ b/pkg/operation/shoot/types.go @@ -46,6 +46,7 @@ import ( "github.com/gardener/gardener/pkg/component/kubernetesdashboard" "github.com/gardener/gardener/pkg/component/kubescheduler" "github.com/gardener/gardener/pkg/component/kubestatemetrics" + "github.com/gardener/gardener/pkg/component/machinecontrollermanager" "github.com/gardener/gardener/pkg/component/nodelocaldns" "github.com/gardener/gardener/pkg/component/resourcemanager" "github.com/gardener/gardener/pkg/component/vpa" @@ -116,21 +117,22 @@ type Components struct { // ControlPlane contains references to K8S control plane components. type ControlPlane struct { - ClusterAutoscaler clusterautoscaler.Interface - EtcdMain etcd.Interface - EtcdEvents etcd.Interface - EtcdCopyBackupsTask etcdcopybackupstask.Interface - KubeAPIServerIngress component.Deployer - KubeAPIServerService component.DeployWaiter - KubeAPIServerSNI component.DeployWaiter - KubeAPIServerSNIPhase component.Phase - KubeAPIServer kubeapiserver.Interface - KubeScheduler kubescheduler.Interface - KubeControllerManager kubecontrollermanager.Interface - KubeStateMetrics kubestatemetrics.Interface - ResourceManager resourcemanager.Interface - VerticalPodAutoscaler vpa.Interface - VPNSeedServer vpnseedserver.Interface + ClusterAutoscaler clusterautoscaler.Interface + EtcdMain etcd.Interface + EtcdEvents etcd.Interface + EtcdCopyBackupsTask etcdcopybackupstask.Interface + KubeAPIServerIngress component.Deployer + KubeAPIServerService component.DeployWaiter + KubeAPIServerSNI component.DeployWaiter + KubeAPIServerSNIPhase component.Phase + KubeAPIServer kubeapiserver.Interface + KubeScheduler kubescheduler.Interface + KubeControllerManager kubecontrollermanager.Interface + KubeStateMetrics kubestatemetrics.Interface + MachineControllerManager machinecontrollermanager.Interface + ResourceManager resourcemanager.Interface + VerticalPodAutoscaler vpa.Interface + VPNSeedServer vpnseedserver.Interface } // Extensions contains references to extension resources. diff --git a/pkg/utils/images/images.go b/pkg/utils/images/images.go index 280d4cecf37..28916fc1c8c 100644 --- a/pkg/utils/images/images.go +++ b/pkg/utils/images/images.go @@ -81,6 +81,8 @@ const ( ImageNameKubernetesDashboard = "kubernetes-dashboard" // ImageNameKubernetesDashboardMetricsScraper is a constant for an image in the image vector with name 'kubernetes-dashboard-metrics-scraper'. ImageNameKubernetesDashboardMetricsScraper = "kubernetes-dashboard-metrics-scraper" + // ImageNameMachineControllerManager is a constant for an image in the image vector with name 'machine-controller-manager'. + ImageNameMachineControllerManager = "machine-controller-manager" // ImageNameMetricsServer is a constant for an image in the image vector with name 'metrics-server'. ImageNameMetricsServer = "metrics-server" // ImageNameNginxIngressController is a constant for an image in the image vector with name 'nginx-ingress-controller'. diff --git a/skaffold.yaml b/skaffold.yaml index 154aec0172e..39302f1a949 100644 --- a/skaffold.yaml +++ b/skaffold.yaml @@ -597,6 +597,7 @@ build: - pkg/component/extensions/operatingsystemconfig/utils - pkg/component/kubeapiserver/constants - pkg/component/kubeproxy + - pkg/component/machinecontrollermanager - pkg/controller/service - pkg/controllerutils - pkg/controllerutils/mapper diff --git a/test/integration/gardenlet/controllerinstallation/controllerinstallation/controllerinstallation_test.go b/test/integration/gardenlet/controllerinstallation/controllerinstallation/controllerinstallation_test.go index b01c69d332e..a9bdc738547 100644 --- a/test/integration/gardenlet/controllerinstallation/controllerinstallation/controllerinstallation_test.go +++ b/test/integration/gardenlet/controllerinstallation/controllerinstallation/controllerinstallation_test.go @@ -212,6 +212,7 @@ var _ = Describe("ControllerInstallation controller tests", func() { HVPAForShootedSeed: false IPv6SingleStack: false KMSv2: false + MachineControllerManagerDeployment: false MutableShootSpecNetworkingNodes: false OpenAPIEnums: true OpenAPIV3: true