diff --git a/CHANGELOG.md b/CHANGELOG.md index 12b0ace79d..4caa5f677b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,34 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 Extending the adopted spec, each change should have a link to its corresponding pull request appended. +## [34.0.0](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v33.1.0...v34.0.0) (2024-10-30) + + +### ⚠ BREAKING CHANGES + +* **TPG>=5.44.2:** add standard cluster support for `insecureKubeletReadonlyPortEnabled` ([#2082](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/2082)) +* **deps:** Update Terraform terraform-google-modules/project-factory/google to v17 ([#2109](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/2109)) + +### Features + +* Add create_ignore_already_exists to workload_identity ([#2142](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/2142)) ([76d779c](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/76d779c314727bfccf64bfd34c61e1ce56bc55b9)) +* **safer-cluster:** add create_service_account variable ([#2138](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/2138)) ([cccabcb](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/cccabcb0eca2a7755908e8d17b40ffba87f4839e)) +* **TPG>=5.44.2:** add standard cluster support for `insecureKubeletReadonlyPortEnabled` ([#2082](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/2082)) ([96626d5](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/96626d541617cb99a3ee19b019ba51c6146f69c0)) + + +### Bug Fixes + +* add kubelet_config to default-pool ([#2147](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/2147)) ([7828a0f](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/7828a0ff35136a9e5fa91a7d26ba07bda734c31b)) +* add target tags to `node_pool_auto_config` for standard clusters ([#2118](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/2118)) ([d5f66e8](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/d5f66e86dd163d505ff455a9c5fafe7d659da36f)) +* correct confidential node field name ([#2140](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/2140)) ([373c969](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/373c96954af9ce53dd3eb062f372efa832378334)) +* **deps:** Update Terraform terraform-google-modules/project-factory/google to v17 ([#2109](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/2109)) ([3073b04](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/3073b048cb79524aeaaf46ff2835b79a6f7b7f23)) +* fix typo in kube-dns-upstream-nameservers configmap ([#2149](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/2149)) ([843c56c](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/843c56cef4157a3bcc281a8bd08021baa733dae1)) +* **node_pools:** Update Variants: Add recreate for `secondary_boot_disk` and `local_ssd_ephemeral_count` (beta) ([#2139](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/2139)) ([66231d5](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/66231d5ad91e282e750b4c9c3b398ab97aa8c147)) +* remove dns from autopilot ([#2152](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/2152)) ([2e645cd](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/2e645cd4248b1a3663fde9f9fea4372f408e7b36)) +* remove unnecessary lookup for safer_cluster nodepool ID ([#2160](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/2160)) ([179a8ec](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/179a8ecd241690215253ec860ca041b7b3781e9e)) +* send provider enum values for `insecureKubeletReadonlyPortEnabled` ([#2145](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/2145)) ([922ab1d](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/922ab1daa32b97325b8c62a19f0b24ee71e189e5)) +* update resource names to snake_case ([#2151](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/2151)) ([375d27c](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/375d27cffddb1af851462c2b3a773a25e4dd40ac)) + ## [33.1.0](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v33.0.4...v33.1.0) (2024-10-09) diff --git a/README.md b/README.md index 6520607820..a8ae382eb6 100644 --- a/README.md +++ b/README.md @@ -190,6 +190,7 @@ Then perform the following commands on the root folder: | http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no | | identity\_namespace | The workload pool to attach all Kubernetes service accounts to. (Default value of `enabled` automatically sets project-based pool `[project_id].svc.id.goog`) | `string` | `"enabled"` | no | | initial\_node\_count | The number of nodes to create in this cluster's default node pool. | `number` | `0` | no | +| insecure\_kubelet\_readonly\_port\_enabled | Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`. | `bool` | `null` | no | | ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | `bool` | `false` | no | | ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | `string` | `"60s"` | no | | ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes | @@ -319,6 +320,7 @@ The node_pools variable takes the following parameters: | gpu_partition_size | Size of partitions to create on the GPU | null | Optional | | image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS_CONTAINERD | Optional | | initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource. Defaults to the value of min_count | " " | Optional | +| insecure_kubelet_readonly_port_enabled | (boolean) Whether or not to enable the insecure Kubelet readonly port. | null | Optional | | key | The key required for the taint | | Required | | logging_variant | The type of logging agent that is deployed by default for newly created node pools in the cluster. Valid values include DEFAULT and MAX_THROUGHPUT. | DEFAULT | Optional | | local_ssd_count | The amount of local SSD disks that will be attached to each cluster node and may be used as a `hostpath` volume or a `local` PersistentVolume. | 0 | Optional | diff --git a/autogen/main/README.md b/autogen/main/README.md index 9031338c76..9ccbf7736d 100644 --- a/autogen/main/README.md +++ b/autogen/main/README.md @@ -213,6 +213,7 @@ The node_pools variable takes the following parameters: | gpu_partition_size | Size of partitions to create on the GPU | null | Optional | | image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS_CONTAINERD | Optional | | initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource. Defaults to the value of min_count | " " | Optional | +| insecure_kubelet_readonly_port_enabled | (boolean) Whether or not to enable the insecure Kubelet readonly port. | null | Optional | | key | The key required for the taint | | Required | | logging_variant | The type of logging agent that is deployed by default for newly created node pools in the cluster. Valid values include DEFAULT and MAX_THROUGHPUT. | DEFAULT | Optional | | local_ssd_count | The amount of local SSD disks that will be attached to each cluster node and may be used as a `hostpath` volume or a `local` PersistentVolume. | 0 | Optional | diff --git a/autogen/main/cluster.tf.tmpl b/autogen/main/cluster.tf.tmpl index 87683c6bda..b3dffb490d 100644 --- a/autogen/main/cluster.tf.tmpl +++ b/autogen/main/cluster.tf.tmpl @@ -282,10 +282,10 @@ resource "google_container_cluster" "primary" { {% if autopilot_cluster != true %} dynamic "node_pool_auto_config" { - for_each = var.cluster_autoscaling.enabled && length(var.network_tags) > 0 ? [1] : [] + for_each = var.cluster_autoscaling.enabled && (length(var.network_tags) > 0 || var.add_cluster_firewall_rules) ? [1] : [] content { network_tags { - tags = var.network_tags + tags = var.add_cluster_firewall_rules ? (concat(var.network_tags, [local.cluster_network_tag])) : var.network_tags } } } @@ -530,6 +530,21 @@ resource "google_container_cluster" "primary" { } } + dynamic "kubelet_config" { + for_each = length(setintersection( + keys(var.node_pools[0]), + ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "insecure_kubelet_readonly_port_enabled", "pod_pids_limit"] + )) != 0 || var.insecure_kubelet_readonly_port_enabled != null ? [1] : [] + + content { + cpu_manager_policy = lookup(var.node_pools[0], "cpu_manager_policy", "static") + cpu_cfs_quota = lookup(var.node_pools[0], "cpu_cfs_quota", null) + cpu_cfs_quota_period = lookup(var.node_pools[0], "cpu_cfs_quota_period", null) + insecure_kubelet_readonly_port_enabled = lookup(var.node_pools[0], "insecure_kubelet_readonly_port_enabled", var.insecure_kubelet_readonly_port_enabled) != null ? upper(tostring(lookup(var.node_pools[0], "insecure_kubelet_readonly_port_enabled", var.insecure_kubelet_readonly_port_enabled))) : null + pod_pids_limit = lookup(var.node_pools[0], "pod_pids_limit", null) + } + } + service_account = lookup(var.node_pools[0], "service_account", local.service_account) tags = concat( @@ -661,7 +676,6 @@ resource "google_container_cluster" "primary" { } } } - {% if beta_cluster %} node_pool_defaults { node_config_defaults { @@ -675,15 +689,17 @@ resource "google_container_cluster" "primary" { } {% endif %} {% if autopilot_cluster != true %} + {% if beta_cluster %} gcfs_config { enabled = var.enable_gcfs } {% endif %} + insecure_kubelet_readonly_port_enabled = var.insecure_kubelet_readonly_port_enabled != null ? upper(tostring(var.insecure_kubelet_readonly_port_enabled)) : null + {% endif %} } } - {% endif %} - {% if beta_cluster %} + {% if beta_cluster %} depends_on = [google_project_iam_member.service_agent] {% endif %} } @@ -706,6 +722,9 @@ locals { "enable_secure_boot", "enable_integrity_monitoring", "local_ssd_count", + {% if beta_cluster %} + "local_ssd_ephemeral_count", + {% endif %} "machine_type", "placement_policy", "max_pods_per_node", @@ -723,6 +742,7 @@ locals { "reservation_affinity_key", "reservation_affinity_values", "enable_confidential_nodes", + "secondary_boot_disk", ] } @@ -733,7 +753,7 @@ locals { resource "random_id" "name" { for_each = merge(local.node_pools, local.windows_node_pools) byte_length = 2 - prefix = format("%s-", lookup(each.value, "name")) + prefix = "${each.key}-" keepers = merge( zipmap( local.force_node_pool_recreation_resources, @@ -1042,14 +1062,15 @@ resource "google_container_node_pool" "windows_pools" { dynamic "kubelet_config" { for_each = length(setintersection( keys(each.value), - ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "pod_pids_limit"] + ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "insecure_kubelet_readonly_port_enabled", "pod_pids_limit"] )) != 0 ? [1] : [] content { - cpu_manager_policy = lookup(each.value, "cpu_manager_policy", "static") - cpu_cfs_quota = lookup(each.value, "cpu_cfs_quota", null) - cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null) - pod_pids_limit = lookup(each.value, "pod_pids_limit", null) + cpu_manager_policy = lookup(each.value, "cpu_manager_policy", "static") + cpu_cfs_quota = lookup(each.value, "cpu_cfs_quota", null) + cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null) + insecure_kubelet_readonly_port_enabled = lookup(each.value, "insecure_kubelet_readonly_port_enabled", null) != null ? upper(tostring(each.value.insecure_kubelet_readonly_port_enabled)) : null + pod_pids_limit = lookup(each.value, "pod_pids_limit", null) } } {% if beta_cluster %} @@ -1088,7 +1109,7 @@ resource "google_container_node_pool" "windows_pools" { } dynamic "confidential_nodes" { - for_each = lookup(each.value, "enable_confidential_nodes", null) != null ? [each.value.confidential_nodes] : [] + for_each = lookup(each.value, "enable_confidential_nodes", null) != null ? [each.value.enable_confidential_nodes] : [] content { enabled = confidential_nodes.value } diff --git a/autogen/main/dns.tf.tmpl b/autogen/main/dns.tf.tmpl index b9db91ca2e..24abfcb85a 100644 --- a/autogen/main/dns.tf.tmpl +++ b/autogen/main/dns.tf.tmpl @@ -16,11 +16,12 @@ {{ autogeneration_note }} +{% if autopilot_cluster != true %} /****************************************** Manage kube-dns configmaps *****************************************/ -resource "kubernetes_config_map_v1_data" "kube-dns" { +resource "kubernetes_config_map_v1_data" "kube_dns" { count = local.custom_kube_dns_config && !local.upstream_nameservers_config ? 1 : 0 metadata { @@ -44,7 +45,7 @@ EOF ] } -resource "kubernetes_config_map_v1_data" "kube-dns-upstream-namservers" { +resource "kubernetes_config_map_v1_data" "kube_dns_upstream_nameservers" { count = !local.custom_kube_dns_config && local.upstream_nameservers_config ? 1 : 0 metadata { @@ -68,7 +69,7 @@ EOF ] } -resource "kubernetes_config_map_v1_data" "kube-dns-upstream-nameservers-and-stub-domains" { +resource "kubernetes_config_map_v1_data" "kube_dns_upstream_nameservers_and_stub_domains" { count = local.custom_kube_dns_config && local.upstream_nameservers_config ? 1 : 0 metadata { @@ -95,3 +96,4 @@ EOF {% endif %} ] } +{% endif %} diff --git a/autogen/main/main.tf.tmpl b/autogen/main/main.tf.tmpl index a7c8afe484..806feda05b 100644 --- a/autogen/main/main.tf.tmpl +++ b/autogen/main/main.tf.tmpl @@ -78,8 +78,10 @@ locals { {% endif %} +{% if autopilot_cluster != true %} custom_kube_dns_config = length(keys(var.stub_domains)) > 0 upstream_nameservers_config = length(var.upstream_nameservers) > 0 +{% endif %} network_project_id = var.network_project_id != "" ? var.network_project_id : var.project_id zone_count = length(var.zones) cluster_type = var.regional ? "regional" : "zonal" diff --git a/autogen/main/masq.tf.tmpl b/autogen/main/masq.tf.tmpl index 28d9b602a1..3d3129fe50 100644 --- a/autogen/main/masq.tf.tmpl +++ b/autogen/main/masq.tf.tmpl @@ -19,7 +19,7 @@ /****************************************** Create ip-masq-agent confimap *****************************************/ -resource "kubernetes_config_map" "ip-masq-agent" { +resource "kubernetes_config_map" "ip_masq_agent" { count = var.configure_ip_masq ? 1 : 0 metadata { diff --git a/autogen/main/moved.tf.tmpl b/autogen/main/moved.tf.tmpl new file mode 100644 index 0000000000..0c8076bec3 --- /dev/null +++ b/autogen/main/moved.tf.tmpl @@ -0,0 +1,64 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{% if autopilot_cluster != true %} +# Updates for kebab to snake case, to match best practices and Google style. +moved { + from = kubernetes_config_map_v1_data.kube-dns + to = kubernetes_config_map_v1_data.kube_dns +} + +# Typo fix and snake case at the same time +moved { + from = kubernetes_config_map_v1_data.kube-dns-upstream-namservers + to = kubernetes_config_map_v1_data.kube_dns_upstream_nameservers +} + +moved { + from = kubernetes_config_map_v1_data.kube-dns-upstream-nameservers-and-stub-domains + to = kubernetes_config_map_v1_data.kube_dns_upstream_nameservers_and_stub_domains +} +{% endif %} + +moved { + from = kubernetes_config_map.ip-masq-agent + to = kubernetes_config_map.ip_masq_agent +} + +moved { + from = google_project_iam_member.cluster_service_account-nodeService_account + to = google_project_iam_member.cluster_service_account_node_service_account +} + +moved { + from = google_project_iam_member.cluster_service_account-metric_writer + to = google_project_iam_member.cluster_service_account_metric_writer +} + +moved { + from = google_project_iam_member.cluster_service_account-resourceMetadata-writer + to = google_project_iam_member.cluster_service_account_resource_metadata_writer +} + +moved { + from = google_project_iam_member.cluster_service_account-gcr + to = google_project_iam_member.cluster_service_account_gcr +} + +moved { + from = google_project_iam_member.cluster_service_account-artifact-registry + to = google_project_iam_member.cluster_service_account_artifact_registry +} diff --git a/autogen/main/sa.tf.tmpl b/autogen/main/sa.tf.tmpl index f0cfb0b19d..bf1beb8f69 100644 --- a/autogen/main/sa.tf.tmpl +++ b/autogen/main/sa.tf.tmpl @@ -46,35 +46,35 @@ resource "google_service_account" "cluster_service_account" { display_name = "Terraform-managed service account for cluster ${var.name}" } -resource "google_project_iam_member" "cluster_service_account-nodeService_account" { +resource "google_project_iam_member" "cluster_service_account_node_service_account" { count = var.create_service_account ? 1 : 0 project = google_service_account.cluster_service_account[0].project role = "roles/container.defaultNodeServiceAccount" member = google_service_account.cluster_service_account[0].member } -resource "google_project_iam_member" "cluster_service_account-metric_writer" { +resource "google_project_iam_member" "cluster_service_account_metric_writer" { count = var.create_service_account ? 1 : 0 project = google_service_account.cluster_service_account[0].project role = "roles/monitoring.metricWriter" member = google_service_account.cluster_service_account[0].member } -resource "google_project_iam_member" "cluster_service_account-resourceMetadata-writer" { +resource "google_project_iam_member" "cluster_service_account_resource_metadata_writer" { count = var.create_service_account ? 1 : 0 project = google_service_account.cluster_service_account[0].project role = "roles/stackdriver.resourceMetadata.writer" member = google_service_account.cluster_service_account[0].member } -resource "google_project_iam_member" "cluster_service_account-gcr" { +resource "google_project_iam_member" "cluster_service_account_gcr" { for_each = var.create_service_account && var.grant_registry_access ? toset(local.registry_projects_list) : [] project = each.key role = "roles/storage.objectViewer" member = "serviceAccount:${google_service_account.cluster_service_account[0].email}" } -resource "google_project_iam_member" "cluster_service_account-artifact-registry" { +resource "google_project_iam_member" "cluster_service_account_artifact_registry" { for_each = var.create_service_account && var.grant_registry_access ? toset(local.registry_projects_list) : [] project = each.key role = "roles/artifactregistry.reader" diff --git a/autogen/main/variables.tf.tmpl b/autogen/main/variables.tf.tmpl index 8a0666a82d..7466acecd9 100644 --- a/autogen/main/variables.tf.tmpl +++ b/autogen/main/variables.tf.tmpl @@ -109,6 +109,12 @@ variable "service_external_ips" { } {% if autopilot_cluster != true %} +variable "insecure_kubelet_readonly_port_enabled" { + type = bool + description = "Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`." + default = null +} + variable "datapath_provider" { type = string description = "The desired datapath provider for this cluster. By default, `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation. `ADVANCED_DATAPATH` enables Dataplane-V2 feature." @@ -351,6 +357,7 @@ variable "network_tags" { default = [] } +{% if autopilot_cluster != true %} variable "stub_domains" { type = map(list(string)) description = "Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server" @@ -362,6 +369,7 @@ variable "upstream_nameservers" { description = "If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf" default = [] } +{% endif %} variable "non_masquerade_cidrs" { type = list(string) diff --git a/autogen/main/versions.tf.tmpl b/autogen/main/versions.tf.tmpl index 88c2b21eff..e20bc44b3e 100644 --- a/autogen/main/versions.tf.tmpl +++ b/autogen/main/versions.tf.tmpl @@ -24,11 +24,11 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 5.40.0, < 7" + version = ">= 5.44.2, !=6.0.0, !=6.0.1, !=6.1.0, !=6.2.0, !=6.3.0, !=6.4.0, !=6.5.0, !=6.6.0, < 7" } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.40.0, < 7" + version = ">= 5.44.2, !=6.0.0, !=6.0.1, !=6.1.0, !=6.2.0, !=6.3.0, !=6.4.0, !=6.5.0, !=6.6.0, < 7" } kubernetes = { source = "hashicorp/kubernetes" @@ -40,7 +40,7 @@ terraform { } } provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine{% if module_registry_name %}:{{ module_registry_name }}{% endif %}/v33.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine{% if module_registry_name %}:{{ module_registry_name }}{% endif %}/v34.0.0" } {% elif beta_cluster and autopilot_cluster %} required_providers { @@ -64,7 +64,7 @@ terraform { } } provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine{% if module_registry_name %}:{{ module_registry_name }}{% endif %}/v33.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine{% if module_registry_name %}:{{ module_registry_name }}{% endif %}/v34.0.0" } {% elif autopilot_cluster %} required_providers { @@ -83,13 +83,13 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine{% if module_registry_name %}:{{ module_registry_name }}{% endif %}/v33.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine{% if module_registry_name %}:{{ module_registry_name }}{% endif %}/v34.0.0" } {% else %} required_providers { google = { source = "hashicorp/google" - version = ">= 5.40.0, < 7" + version = ">= 5.44.2, !=6.0.0, !=6.0.1, !=6.1.0, !=6.2.0, !=6.3.0, !=6.4.0, !=6.5.0, !=6.6.0, < 7" } kubernetes = { source = "hashicorp/kubernetes" @@ -101,7 +101,7 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine{% if module_registry_name %}:{{ module_registry_name }}{% endif %}/v33.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine{% if module_registry_name %}:{{ module_registry_name }}{% endif %}/v34.0.0" } {% endif %} } diff --git a/autogen/safer-cluster/main.tf.tmpl b/autogen/safer-cluster/main.tf.tmpl index 95a2fc6e2e..3a06a87e3f 100644 --- a/autogen/safer-cluster/main.tf.tmpl +++ b/autogen/safer-cluster/main.tf.tmpl @@ -119,11 +119,16 @@ module "gke" { // All applications should run with an identity defined via Workload Identity anyway. // - Use a service account passed as a parameter to the module, in case the user // wants to maintain control of their service accounts. - create_service_account = var.compute_engine_service_account == "" ? true : false service_account = var.compute_engine_service_account registry_project_ids = var.registry_project_ids grant_registry_access = var.grant_registry_access + // If create_service_account is explicitly set to false we short-circuit the + // compute_engine_service_account check to potentially avoid an error (see variables.tf documentation). + // Otherwise if true (the default), we check if compute_engine_service_account is set for backwards compatability + // before the create_service_account variable was added. + create_service_account = var.create_service_account == false ? var.create_service_account : (var.compute_engine_service_account == "" ? true : false) + issue_client_certificate = false cluster_resource_labels = var.cluster_resource_labels @@ -209,7 +214,7 @@ module "gke" { // Enabling vulnerability and audit for workloads workload_vulnerability_mode = var.workload_vulnerability_mode workload_config_audit_mode = var.workload_config_audit_mode - + // Enabling security posture security_posture_mode = var.security_posture_mode security_posture_vulnerability_mode = var.security_posture_vulnerability_mode diff --git a/autogen/safer-cluster/variables.tf.tmpl b/autogen/safer-cluster/variables.tf.tmpl index eeb1b77ec4..678eaa2a3a 100644 --- a/autogen/safer-cluster/variables.tf.tmpl +++ b/autogen/safer-cluster/variables.tf.tmpl @@ -411,10 +411,16 @@ variable "authenticator_security_group" { variable "compute_engine_service_account" { type = string - description = "Use the given service account for nodes rather than creating a new dedicated service account." + description = "Use the given service account for nodes rather than creating a new dedicated service account. If set then also set var.create_service_account to false to avoid 'value depends on resource attributes that cannot be determined until apply' errors." default = "" } +variable "create_service_account" { + type = bool + description = "Defines if service account specified to run nodes should be created. Explicitly set to false if var.compute_engine_service_account is set to avoid 'value depends on resource attributes that cannot be determined until apply' errors." + default = true +} + variable "enable_shielded_nodes" { type = bool description = "Enable Shielded Nodes features on all nodes in this cluster." diff --git a/autogen/safer-cluster/versions.tf.tmpl b/autogen/safer-cluster/versions.tf.tmpl index bb39e0d3ec..10bfc954ef 100644 --- a/autogen/safer-cluster/versions.tf.tmpl +++ b/autogen/safer-cluster/versions.tf.tmpl @@ -23,6 +23,6 @@ terraform { required_version = ">=1.3" provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine{% if module_registry_name %}:{{ module_registry_name }}{% endif %}/v33.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine{% if module_registry_name %}:{{ module_registry_name }}{% endif %}/v34.0.0" } } diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index 1b66772fb6..d8a56f4802 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -134,6 +134,22 @@ steps: - verify simple-regional-private-local name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'cft test run TestSimpleRegionalPrivate --stage teardown --verbose'] +- id: apply simple-regional-cluster-autoscaling + waitFor: + - create-all + - destroy simple-regional-private-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestSimpleRegionalClusterAutoscaling --stage apply --verbose'] +- id: verify simple-regional-cluster-autoscaling + waitFor: + - apply simple-regional-cluster-autoscaling + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestSimpleRegionalClusterAutoscaling --stage verify --verbose'] +- id: destroy simple-regional-cluster-autoscaling + waitFor: + - verify simple-regional-cluster-autoscaling + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestSimpleRegionalClusterAutoscaling --stage teardown --verbose'] - id: init simple-regional-with-kubeconfig-local waitFor: - create-all diff --git a/cluster.tf b/cluster.tf index bc3ff4bf47..24fc980d55 100644 --- a/cluster.tf +++ b/cluster.tf @@ -204,10 +204,10 @@ resource "google_container_cluster" "primary" { } dynamic "node_pool_auto_config" { - for_each = var.cluster_autoscaling.enabled && length(var.network_tags) > 0 ? [1] : [] + for_each = var.cluster_autoscaling.enabled && (length(var.network_tags) > 0 || var.add_cluster_firewall_rules) ? [1] : [] content { network_tags { - tags = var.network_tags + tags = var.add_cluster_firewall_rules ? (concat(var.network_tags, [local.cluster_network_tag])) : var.network_tags } } } @@ -407,6 +407,21 @@ resource "google_container_cluster" "primary" { } } + dynamic "kubelet_config" { + for_each = length(setintersection( + keys(var.node_pools[0]), + ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "insecure_kubelet_readonly_port_enabled", "pod_pids_limit"] + )) != 0 || var.insecure_kubelet_readonly_port_enabled != null ? [1] : [] + + content { + cpu_manager_policy = lookup(var.node_pools[0], "cpu_manager_policy", "static") + cpu_cfs_quota = lookup(var.node_pools[0], "cpu_cfs_quota", null) + cpu_cfs_quota_period = lookup(var.node_pools[0], "cpu_cfs_quota_period", null) + insecure_kubelet_readonly_port_enabled = lookup(var.node_pools[0], "insecure_kubelet_readonly_port_enabled", var.insecure_kubelet_readonly_port_enabled) != null ? upper(tostring(lookup(var.node_pools[0], "insecure_kubelet_readonly_port_enabled", var.insecure_kubelet_readonly_port_enabled))) : null + pod_pids_limit = lookup(var.node_pools[0], "pod_pids_limit", null) + } + } + service_account = lookup(var.node_pools[0], "service_account", local.service_account) tags = concat( @@ -500,6 +515,13 @@ resource "google_container_cluster" "primary" { } } } + + node_pool_defaults { + node_config_defaults { + insecure_kubelet_readonly_port_enabled = var.insecure_kubelet_readonly_port_enabled != null ? upper(tostring(var.insecure_kubelet_readonly_port_enabled)) : null + } + } + } /****************************************** Create Container Cluster node pools @@ -739,14 +761,15 @@ resource "google_container_node_pool" "pools" { dynamic "kubelet_config" { for_each = length(setintersection( keys(each.value), - ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "pod_pids_limit"] + ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "insecure_kubelet_readonly_port_enabled", "pod_pids_limit"] )) != 0 ? [1] : [] content { - cpu_manager_policy = lookup(each.value, "cpu_manager_policy", "static") - cpu_cfs_quota = lookup(each.value, "cpu_cfs_quota", null) - cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null) - pod_pids_limit = lookup(each.value, "pod_pids_limit", null) + cpu_manager_policy = lookup(each.value, "cpu_manager_policy", "static") + cpu_cfs_quota = lookup(each.value, "cpu_cfs_quota", null) + cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null) + insecure_kubelet_readonly_port_enabled = lookup(each.value, "insecure_kubelet_readonly_port_enabled", null) != null ? upper(tostring(each.value.insecure_kubelet_readonly_port_enabled)) : null + pod_pids_limit = lookup(each.value, "pod_pids_limit", null) } } @@ -774,7 +797,7 @@ resource "google_container_node_pool" "pools" { } dynamic "confidential_nodes" { - for_each = lookup(each.value, "enable_confidential_nodes", null) != null ? [each.value.confidential_nodes] : [] + for_each = lookup(each.value, "enable_confidential_nodes", null) != null ? [each.value.enable_confidential_nodes] : [] content { enabled = confidential_nodes.value } @@ -1029,14 +1052,15 @@ resource "google_container_node_pool" "windows_pools" { dynamic "kubelet_config" { for_each = length(setintersection( keys(each.value), - ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "pod_pids_limit"] + ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "insecure_kubelet_readonly_port_enabled", "pod_pids_limit"] )) != 0 ? [1] : [] content { - cpu_manager_policy = lookup(each.value, "cpu_manager_policy", "static") - cpu_cfs_quota = lookup(each.value, "cpu_cfs_quota", null) - cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null) - pod_pids_limit = lookup(each.value, "pod_pids_limit", null) + cpu_manager_policy = lookup(each.value, "cpu_manager_policy", "static") + cpu_cfs_quota = lookup(each.value, "cpu_cfs_quota", null) + cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null) + insecure_kubelet_readonly_port_enabled = lookup(each.value, "insecure_kubelet_readonly_port_enabled", null) != null ? upper(tostring(each.value.insecure_kubelet_readonly_port_enabled)) : null + pod_pids_limit = lookup(each.value, "pod_pids_limit", null) } } @@ -1049,7 +1073,7 @@ resource "google_container_node_pool" "windows_pools" { } dynamic "confidential_nodes" { - for_each = lookup(each.value, "enable_confidential_nodes", null) != null ? [each.value.confidential_nodes] : [] + for_each = lookup(each.value, "enable_confidential_nodes", null) != null ? [each.value.enable_confidential_nodes] : [] content { enabled = confidential_nodes.value } diff --git a/dns.tf b/dns.tf index bf0d05b723..c120c2be92 100644 --- a/dns.tf +++ b/dns.tf @@ -20,7 +20,7 @@ Manage kube-dns configmaps *****************************************/ -resource "kubernetes_config_map_v1_data" "kube-dns" { +resource "kubernetes_config_map_v1_data" "kube_dns" { count = local.custom_kube_dns_config && !local.upstream_nameservers_config ? 1 : 0 metadata { @@ -42,7 +42,7 @@ EOF ] } -resource "kubernetes_config_map_v1_data" "kube-dns-upstream-namservers" { +resource "kubernetes_config_map_v1_data" "kube_dns_upstream_nameservers" { count = !local.custom_kube_dns_config && local.upstream_nameservers_config ? 1 : 0 metadata { @@ -64,7 +64,7 @@ EOF ] } -resource "kubernetes_config_map_v1_data" "kube-dns-upstream-nameservers-and-stub-domains" { +resource "kubernetes_config_map_v1_data" "kube_dns_upstream_nameservers_and_stub_domains" { count = local.custom_kube_dns_config && local.upstream_nameservers_config ? 1 : 0 metadata { diff --git a/docs/upgrading_to_v34.0.md b/docs/upgrading_to_v34.0.md new file mode 100644 index 0000000000..f2f184be0a --- /dev/null +++ b/docs/upgrading_to_v34.0.md @@ -0,0 +1,54 @@ +# Upgrading to v34.0 + +The v34.0 release of _kubernetes-engine_ is a backwards incompatible release. + +### safer-cluster modules: Added create_service_account variable + +This only affects users of the `safer-cluster` modules that have set `var.compute_engine_service_account` to something other than the default `""`. + +A variable `var.create_service_account` was added to the `safer-cluster` modules that when explicitly set to `false` avoids the following error withing the `private-cluster` modules: + +```sh +Error: Invalid count argument + + on .terraform/modules/gke_cluster.gke/modules/beta-private-cluster/sa.tf line 35, in resource "random_string" "cluster_service_account_suffix": + 35: count = var.create_service_account && var.service_account_name == "" ? 1 : 0 + +The "count" value depends on resource attributes that cannot be determined +until apply, so Terraform cannot predict how many instances will be created. +To work around this, use the -target argument to first apply only the +resources that the count depends on. +``` + +This seems to happen if `var.compute_engine_service_account` is passed in, and the externally created service account is being created at the same time, so the name/email is not computed yet: + +```terraform +resource "google_service_account" "cluster_service_account" { + project = var.project_id + account_id = "tf-gke-${var.cluster_name}-${random_string.cluster_service_account_suffix.result}" + display_name = "Terraform-managed service account for cluster ${var.cluster_name}" +} + +module "gke" { + source = "terraform-google-modules/kubernetes-engine/google//modules/safer-cluster" + version = "~> 33.0" + + project_id = var.project_id + name = var.cluster_name + + create_service_account = false + compute_engine_service_account = google_service_account.cluster_service_account.email +} +``` + +By explicitly passing a `var.create_service_account = false` it short circuits the calculations dependent on `var.service_account_name`: + +```terraform +resource "random_string" "cluster_service_account_suffix" { + count = var.create_service_account && var.service_account_name == "" ? 1 : 0 + upper = false + lower = true + special = false + length = 4 +} +``` diff --git a/examples/acm-terraform-blog-part1/terraform/gke.tf b/examples/acm-terraform-blog-part1/terraform/gke.tf index 4cd6bfe1a5..e53f3f5026 100644 --- a/examples/acm-terraform-blog-part1/terraform/gke.tf +++ b/examples/acm-terraform-blog-part1/terraform/gke.tf @@ -16,7 +16,7 @@ module "enabled_google_apis" { source = "terraform-google-modules/project-factory/google//modules/project_services" - version = "~> 16.0" + version = "~> 17.0" project_id = var.project disable_services_on_destroy = false diff --git a/examples/acm-terraform-blog-part2/terraform/gke.tf b/examples/acm-terraform-blog-part2/terraform/gke.tf index 98f387dbe5..5b703a338a 100644 --- a/examples/acm-terraform-blog-part2/terraform/gke.tf +++ b/examples/acm-terraform-blog-part2/terraform/gke.tf @@ -16,7 +16,7 @@ module "enabled_google_apis" { source = "terraform-google-modules/project-factory/google//modules/project_services" - version = "~> 16.0" + version = "~> 17.0" project_id = var.project disable_services_on_destroy = false diff --git a/examples/acm-terraform-blog-part3/terraform/gke.tf b/examples/acm-terraform-blog-part3/terraform/gke.tf index 74bcb4c271..6e15206d96 100644 --- a/examples/acm-terraform-blog-part3/terraform/gke.tf +++ b/examples/acm-terraform-blog-part3/terraform/gke.tf @@ -16,7 +16,7 @@ module "enabled_google_apis" { source = "terraform-google-modules/project-factory/google//modules/project_services" - version = "~> 16.0" + version = "~> 17.0" project_id = var.project disable_services_on_destroy = false diff --git a/examples/autopilot_private_firewalls/network.tf b/examples/autopilot_private_firewalls/network.tf index acb96e0289..39e76f1869 100644 --- a/examples/autopilot_private_firewalls/network.tf +++ b/examples/autopilot_private_firewalls/network.tf @@ -23,9 +23,10 @@ module "gcp-network" { subnets = [ { - subnet_name = local.subnet_name - subnet_ip = "10.0.0.0/17" - subnet_region = var.region + subnet_name = local.subnet_name + subnet_ip = "10.0.0.0/17" + subnet_region = var.region + subnet_private_access = true }, { subnet_name = local.master_auth_subnetwork diff --git a/examples/deploy_service/main.tf b/examples/deploy_service/main.tf index b055bf0d60..f64819d197 100644 --- a/examples/deploy_service/main.tf +++ b/examples/deploy_service/main.tf @@ -55,7 +55,7 @@ resource "kubernetes_pod" "nginx-example" { spec { container { - image = "nginx:1.27.1" + image = "nginx:1.27.2" name = "nginx-example" } } diff --git a/examples/node_pool/main.tf b/examples/node_pool/main.tf index 6d61682f5b..0e7805fe33 100644 --- a/examples/node_pool/main.tf +++ b/examples/node_pool/main.tf @@ -67,20 +67,21 @@ module "gke" { service_account = var.compute_engine_service_account }, { - name = "pool-03" - machine_type = "n1-standard-2" - node_locations = "${var.region}-b,${var.region}-c" - autoscaling = false - node_count = 2 - disk_type = "pd-standard" - auto_upgrade = true - service_account = var.compute_engine_service_account - pod_range = "test" - sandbox_enabled = true - cpu_manager_policy = "static" - cpu_cfs_quota = true - local_ssd_ephemeral_count = 2 - pod_pids_limit = 4096 + name = "pool-03" + machine_type = "n1-standard-2" + node_locations = "${var.region}-b,${var.region}-c" + autoscaling = false + node_count = 2 + disk_type = "pd-standard" + auto_upgrade = true + service_account = var.compute_engine_service_account + pod_range = "test" + sandbox_enabled = true + cpu_manager_policy = "static" + cpu_cfs_quota = true + insecure_kubelet_readonly_port_enabled = false + local_ssd_ephemeral_count = 2 + pod_pids_limit = 4096 }, { name = "pool-04" diff --git a/examples/node_pool_update_variant/main.tf b/examples/node_pool_update_variant/main.tf index c0c3ba0b15..bc351529d0 100644 --- a/examples/node_pool_update_variant/main.tf +++ b/examples/node_pool_update_variant/main.tf @@ -61,11 +61,12 @@ module "gke" { node_pools = [ { - name = "pool-01" - min_count = 1 - max_count = 2 - service_account = var.compute_engine_service_account - auto_upgrade = true + name = "pool-01" + min_count = 1 + max_count = 2 + service_account = var.compute_engine_service_account + auto_upgrade = true + insecure_kubelet_readonly_port_enabled = false }, { name = "pool-02" diff --git a/examples/private_zonal_with_networking/main.tf b/examples/private_zonal_with_networking/main.tf index fbaf3d95ca..446d6436ef 100644 --- a/examples/private_zonal_with_networking/main.tf +++ b/examples/private_zonal_with_networking/main.tf @@ -79,6 +79,8 @@ module "gke" { master_ipv4_cidr_block = "172.16.0.0/28" deletion_protection = false + insecure_kubelet_readonly_port_enabled = false + master_authorized_networks = [ { cidr_block = data.google_compute_subnetwork.subnetwork.ip_cidr_range diff --git a/examples/safer_cluster_iap_bastion/apis.tf b/examples/safer_cluster_iap_bastion/apis.tf index 68334b926a..2e8e7d9221 100644 --- a/examples/safer_cluster_iap_bastion/apis.tf +++ b/examples/safer_cluster_iap_bastion/apis.tf @@ -16,7 +16,7 @@ module "enabled_google_apis" { source = "terraform-google-modules/project-factory/google//modules/project_services" - version = "~> 16.0" + version = "~> 17.0" project_id = var.project_id disable_services_on_destroy = false diff --git a/examples/safer_cluster_iap_bastion/bastion.tf b/examples/safer_cluster_iap_bastion/bastion.tf index 7fea1c368d..97f44227fd 100644 --- a/examples/safer_cluster_iap_bastion/bastion.tf +++ b/examples/safer_cluster_iap_bastion/bastion.tf @@ -21,7 +21,7 @@ locals { module "bastion" { source = "terraform-google-modules/bastion-host/google" - version = "~> 7.0" + version = "~> 8.0" network = module.vpc.network_self_link subnet = module.vpc.subnets_self_links[0] diff --git a/examples/safer_cluster_iap_bastion/network.tf b/examples/safer_cluster_iap_bastion/network.tf index f06384c58f..3d47a78cb8 100644 --- a/examples/safer_cluster_iap_bastion/network.tf +++ b/examples/safer_cluster_iap_bastion/network.tf @@ -17,7 +17,7 @@ module "vpc" { source = "terraform-google-modules/network/google" - version = ">= 7.5" + version = "~> 9.3" project_id = module.enabled_google_apis.project_id network_name = var.network_name diff --git a/examples/simple_autopilot_public/network.tf b/examples/simple_autopilot_public/network.tf index 436d471869..e773c507b6 100644 --- a/examples/simple_autopilot_public/network.tf +++ b/examples/simple_autopilot_public/network.tf @@ -23,9 +23,10 @@ module "gcp-network" { subnets = [ { - subnet_name = local.subnet_name - subnet_ip = "10.0.0.0/17" - subnet_region = var.region + subnet_name = local.subnet_name + subnet_ip = "10.0.0.0/17" + subnet_region = var.region + subnet_private_access = true }, { subnet_name = local.master_auth_subnetwork diff --git a/examples/simple_regional_cluster_autoscaling/README.md b/examples/simple_regional_cluster_autoscaling/README.md new file mode 100644 index 0000000000..16a8ad5f9d --- /dev/null +++ b/examples/simple_regional_cluster_autoscaling/README.md @@ -0,0 +1,39 @@ +# Simple Regional Cluster with Node Autoscaling + +This example illustrates how to create a simple private cluster with cluster +level node autoprovisioning. + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| cluster\_name\_suffix | A suffix to append to the default cluster name | `string` | `""` | no | +| compute\_engine\_service\_account | Service account to associate to the nodes in the cluster | `any` | n/a | yes | +| ip\_range\_pods | The secondary ip range to use for pods | `any` | n/a | yes | +| ip\_range\_services | The secondary ip range to use for services | `any` | n/a | yes | +| network | The VPC network to host the cluster in | `any` | n/a | yes | +| project\_id | The project ID to host the cluster in | `any` | n/a | yes | +| region | The region to host the cluster in | `any` | n/a | yes | +| subnetwork | The subnetwork to host the cluster in | `any` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| ca\_certificate | n/a | +| client\_token | n/a | +| cluster\_name | Cluster name | +| ip\_range\_pods | The secondary IP range used for pods | +| ip\_range\_services | The secondary IP range used for services | +| kubernetes\_endpoint | n/a | +| location | n/a | +| master\_kubernetes\_version | The master Kubernetes version | +| network | n/a | +| project\_id | n/a | +| region | n/a | +| service\_account | The default service account used for running nodes. | +| subnetwork | n/a | +| zones | List of zones in which the cluster resides | + + diff --git a/examples/simple_regional_cluster_autoscaling/main.tf b/examples/simple_regional_cluster_autoscaling/main.tf new file mode 100644 index 0000000000..9669572632 --- /dev/null +++ b/examples/simple_regional_cluster_autoscaling/main.tf @@ -0,0 +1,70 @@ +/** + * Copyright 2018-2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + cluster_type = "simple-rgnl-cluster-autosc" +} + +data "google_client_config" "default" {} + +provider "kubernetes" { + host = "https://${module.gke.endpoint}" + token = data.google_client_config.default.access_token + cluster_ca_certificate = base64decode(module.gke.ca_certificate) +} + +module "gke" { + source = "terraform-google-modules/kubernetes-engine/google" + version = "~> 33.0" + + project_id = var.project_id + name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" + regional = true + region = var.region + network = var.network + subnetwork = var.subnetwork + ip_range_pods = var.ip_range_pods + ip_range_services = var.ip_range_services + create_service_account = false + service_account = var.compute_engine_service_account + default_max_pods_per_node = 20 + remove_default_node_pool = true + deletion_protection = false + + add_cluster_firewall_rules = true + firewall_inbound_ports = ["8443", "9443", "15017"] + + # Just an example + network_tags = ["egress-internet"] + + cluster_autoscaling = { + enabled = true + autoscaling_profile = "OPTIMIZE_UTILIZATION" + min_cpu_cores = 4 + max_cpu_cores = 86 + min_memory_gb = 16 + max_memory_gb = 256 + disk_size = 100 + disk_type = "pd-standard" + image_type = "COS_CONTAINERD" + gpu_resources = [] + auto_repair = true + auto_upgrade = true + strategy = "SURGE" + max_surge = 1 + max_unavailable = 0 + } +} diff --git a/examples/simple_regional_cluster_autoscaling/outputs.tf b/examples/simple_regional_cluster_autoscaling/outputs.tf new file mode 100644 index 0000000000..e527390d91 --- /dev/null +++ b/examples/simple_regional_cluster_autoscaling/outputs.tf @@ -0,0 +1,36 @@ +/** + * Copyright 2018-2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "kubernetes_endpoint" { + sensitive = true + value = module.gke.endpoint +} + +output "client_token" { + sensitive = true + value = base64encode(data.google_client_config.default.access_token) +} + +output "ca_certificate" { + sensitive = true + value = module.gke.ca_certificate +} + +output "service_account" { + description = "The default service account used for running nodes." + value = module.gke.service_account +} + diff --git a/examples/simple_regional_cluster_autoscaling/test_outputs.tf b/examples/simple_regional_cluster_autoscaling/test_outputs.tf new file mode 100644 index 0000000000..ea5f6c4e41 --- /dev/null +++ b/examples/simple_regional_cluster_autoscaling/test_outputs.tf @@ -0,0 +1,63 @@ +/** + * Copyright 2018-2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// These outputs are used to test the module with kitchen-terraform +// They do not need to be included in real-world uses of this module + +output "project_id" { + value = var.project_id +} + +output "region" { + value = module.gke.region +} + +output "cluster_name" { + description = "Cluster name" + value = module.gke.name +} + +output "network" { + value = var.network +} + +output "subnetwork" { + value = var.subnetwork +} + +output "location" { + value = module.gke.location +} + +output "ip_range_pods" { + description = "The secondary IP range used for pods" + value = var.ip_range_pods +} + +output "ip_range_services" { + description = "The secondary IP range used for services" + value = var.ip_range_services +} + +output "zones" { + description = "List of zones in which the cluster resides" + value = module.gke.zones +} + +output "master_kubernetes_version" { + description = "The master Kubernetes version" + value = module.gke.master_version +} diff --git a/examples/simple_regional_cluster_autoscaling/variables.tf b/examples/simple_regional_cluster_autoscaling/variables.tf new file mode 100644 index 0000000000..2ce217972e --- /dev/null +++ b/examples/simple_regional_cluster_autoscaling/variables.tf @@ -0,0 +1,49 @@ +/** + * Copyright 2018-2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "The project ID to host the cluster in" +} + +variable "cluster_name_suffix" { + description = "A suffix to append to the default cluster name" + default = "" +} + +variable "region" { + description = "The region to host the cluster in" +} + +variable "network" { + description = "The VPC network to host the cluster in" +} + +variable "subnetwork" { + description = "The subnetwork to host the cluster in" +} + +variable "ip_range_pods" { + description = "The secondary ip range to use for pods" +} + +variable "ip_range_services" { + description = "The secondary ip range to use for services" +} + +variable "compute_engine_service_account" { + description = "Service account to associate to the nodes in the cluster" +} + diff --git a/examples/simple_regional_cluster_autoscaling/versions.tf b/examples/simple_regional_cluster_autoscaling/versions.tf new file mode 100644 index 0000000000..62771b4e77 --- /dev/null +++ b/examples/simple_regional_cluster_autoscaling/versions.tf @@ -0,0 +1,27 @@ +/** + * Copyright 2021-2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + kubernetes = { + source = "hashicorp/kubernetes" + } + } + required_version = ">= 0.13" +} diff --git a/examples/simple_windows_node_pool/main.tf b/examples/simple_windows_node_pool/main.tf index d3dde58491..024d70e546 100644 --- a/examples/simple_windows_node_pool/main.tf +++ b/examples/simple_windows_node_pool/main.tf @@ -63,7 +63,7 @@ module "gke" { autoscaling = false auto_upgrade = true node_count = 1 - machine_type = "n2-standard-4" + machine_type = "n2-standard-8" image_type = "WINDOWS_LTSC_CONTAINERD" }, ] diff --git a/examples/simple_zonal_with_acm/README.md b/examples/simple_zonal_with_acm/README.md index 986b66e3cd..40e3686485 100644 --- a/examples/simple_zonal_with_acm/README.md +++ b/examples/simple_zonal_with_acm/README.md @@ -1,6 +1,6 @@ # Simple Zonal Cluster -This example illustrates how to create a simple cluster and install [Anthos Config Management](https://cloud.google.com/anthos-config-management/docs/)'s [Config Sync](https://cloud.google.com/anthos-config-management/docs/config-sync-overview) and [Policy Controller](https://cloud.google.com/anthos-config-management/docs/concepts/policy-controller) with the [Policy Essentials v2022 policy bundle](https://cloud.google.com/anthos-config-management/docs/how-to/using-policy-essentials-v2022). +This example illustrates how to create a simple cluster and install [Anthos Config Management](https://cloud.google.com/anthos-config-management/docs/)'s [Config Sync](https://cloud.google.com/anthos-config-management/docs/config-sync-overview). It incorporates the standard cluster module and the [ACM install module](../../modules/acm). @@ -27,12 +27,6 @@ After applying the Terraform configuration, you can run the following commands t kubectl describe ns shipping-dev ``` -4. You can also use `kubectl` to view any policy violations on the cluster: - - ``` - kubectl get constraint -l policycontroller.gke.io/bundleName=policy-essentials-v2022 -o json | jq -cC '.items[]| [.metadata.name,.status.totalViolations]' - ``` - ## Inputs diff --git a/examples/simple_zonal_with_acm/acm.tf b/examples/simple_zonal_with_acm/acm.tf index 3c15a155ea..acf6aba34a 100644 --- a/examples/simple_zonal_with_acm/acm.tf +++ b/examples/simple_zonal_with_acm/acm.tf @@ -26,6 +26,8 @@ module "acm" { sync_branch = "1.0.0" policy_dir = "foo-corp" + enable_policy_controller = false + enable_fleet_feature = var.enable_fleet_feature secret_type = "ssh" diff --git a/examples/simple_zonal_with_poco/README.md b/examples/simple_zonal_with_poco/README.md new file mode 100644 index 0000000000..b8bfd18509 --- /dev/null +++ b/examples/simple_zonal_with_poco/README.md @@ -0,0 +1,37 @@ +# Simple Zonal Cluster + +This example illustrates how to create a simple cluster and install [Policy Controller](https://cloud.google.com/anthos-config-management/docs/concepts/policy-controller) with the [Pod Security Standards Baseline policy bundle](https://cloud.google.com/kubernetes-engine/enterprise/policy-controller/docs/how-to/using-pss-baseline). + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| cluster\_name\_suffix | A suffix to append to the default cluster name | `string` | `""` | no | +| enable\_fleet\_feature | Whether to enable the Policy Controller feature on the fleet. | `bool` | `true` | no | +| project\_id | The project ID to host the cluster in | `string` | n/a | yes | +| region | The region to host the cluster in | `string` | `"us-central1"` | no | +| zone | The zone to host the cluster in | `string` | `"us-central1-a"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| cluster\_name | Cluster name | +| ip\_range\_pods | The secondary IP range used for pods | +| ip\_range\_services | The secondary IP range used for services | +| location | n/a | +| network | n/a | +| project\_id | Standard test outputs | +| region | n/a | +| service\_account | The default service account used for running nodes. | +| subnetwork | n/a | +| zones | List of zones in which the cluster resides | + + + +To provision this example, run the following from within this directory: +- `terraform init` to get the plugins +- `terraform plan` to see the infrastructure plan +- `terraform apply` to apply the infrastructure build +- `terraform destroy` to destroy the built infrastructure diff --git a/examples/simple_zonal_with_poco/main.tf b/examples/simple_zonal_with_poco/main.tf new file mode 100644 index 0000000000..37f806a5ef --- /dev/null +++ b/examples/simple_zonal_with_poco/main.tf @@ -0,0 +1,53 @@ +/** + * Copyright 2018-2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + cluster_type = "simple-zonal-poco" +} + +provider "google" { + region = var.region +} + +module "gke" { + source = "terraform-google-modules/kubernetes-engine/google" + version = "~> 33.0" + + project_id = var.project_id + fleet_project = var.project_id + regional = false + region = var.region + zones = [var.zone] + + name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" + + network = google_compute_network.main.name + subnetwork = google_compute_subnetwork.main.name + ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name + ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name + + service_account = "create" + deletion_protection = false + node_pools = [ + { + name = "poco-node-pool" + autoscaling = false + auto_upgrade = true + node_count = 4 + machine_type = "e2-standard-4" + }, + ] +} diff --git a/examples/simple_zonal_with_poco/network.tf b/examples/simple_zonal_with_poco/network.tf new file mode 100644 index 0000000000..0f2a3d3e84 --- /dev/null +++ b/examples/simple_zonal_with_poco/network.tf @@ -0,0 +1,45 @@ +/** + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "random_string" "suffix" { + length = 4 + special = false + upper = false +} + +resource "google_compute_network" "main" { + project = var.project_id + name = "cft-gke-test-${random_string.suffix.result}" + auto_create_subnetworks = false +} + +resource "google_compute_subnetwork" "main" { + project = var.project_id + name = "cft-gke-test-${random_string.suffix.result}" + ip_cidr_range = "10.0.0.0/17" + region = var.region + network = google_compute_network.main.self_link + + secondary_ip_range { + range_name = "cft-gke-test-pods-${random_string.suffix.result}" + ip_cidr_range = "192.168.0.0/18" + } + + secondary_ip_range { + range_name = "cft-gke-test-services-${random_string.suffix.result}" + ip_cidr_range = "192.168.64.0/18" + } +} diff --git a/examples/simple_zonal_with_poco/outputs.tf b/examples/simple_zonal_with_poco/outputs.tf new file mode 100644 index 0000000000..f161d5bc89 --- /dev/null +++ b/examples/simple_zonal_with_poco/outputs.tf @@ -0,0 +1,61 @@ +/** + * Copyright 2018-2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "service_account" { + description = "The default service account used for running nodes." + value = module.gke.service_account +} + +# Standard test outputs +output "project_id" { + value = var.project_id +} + +output "region" { + value = module.gke.region +} + +output "cluster_name" { + description = "Cluster name" + value = module.gke.name +} + +output "network" { + value = google_compute_network.main.name +} + +output "subnetwork" { + value = google_compute_subnetwork.main.name +} + +output "location" { + value = module.gke.location +} + +output "ip_range_pods" { + description = "The secondary IP range used for pods" + value = google_compute_subnetwork.main.secondary_ip_range[0].range_name +} + +output "ip_range_services" { + description = "The secondary IP range used for services" + value = google_compute_subnetwork.main.secondary_ip_range[1].range_name +} + +output "zones" { + description = "List of zones in which the cluster resides" + value = module.gke.zones +} diff --git a/examples/simple_zonal_with_poco/policy.tf b/examples/simple_zonal_with_poco/policy.tf new file mode 100644 index 0000000000..cfbb98cd85 --- /dev/null +++ b/examples/simple_zonal_with_poco/policy.tf @@ -0,0 +1,51 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "google_gke_hub_feature" "poco_feature" { + name = "policycontroller" + project = var.project_id + location = "global" + + count = var.enable_fleet_feature ? 1 : 0 +} + +resource "google_gke_hub_feature_membership" "poco_feature_member" { + project = var.project_id + location = "global" + + feature = "policycontroller" + membership = module.gke.fleet_membership + membership_location = module.gke.region + + policycontroller { + policy_controller_hub_config { + install_spec = "INSTALL_SPEC_ENABLED" + policy_content { + template_library { + installation = "ALL" + } + bundles { + bundle_name = "pss-baseline-v2022" + } + } + referential_rules_enabled = true + } + } + + depends_on = [ + google_gke_hub_feature.poco_feature + ] +} diff --git a/examples/simple_zonal_with_poco/variables.tf b/examples/simple_zonal_with_poco/variables.tf new file mode 100644 index 0000000000..9a3bb7461e --- /dev/null +++ b/examples/simple_zonal_with_poco/variables.tf @@ -0,0 +1,44 @@ +/** + * Copyright 2018-2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "The project ID to host the cluster in" + type = string +} + +variable "cluster_name_suffix" { + description = "A suffix to append to the default cluster name" + type = string + default = "" +} + +variable "region" { + description = "The region to host the cluster in" + type = string + default = "us-central1" +} + +variable "zone" { + type = string + description = "The zone to host the cluster in" + default = "us-central1-a" +} + +variable "enable_fleet_feature" { + description = "Whether to enable the Policy Controller feature on the fleet." + type = bool + default = true +} diff --git a/examples/simple_zonal_with_poco/versions.tf b/examples/simple_zonal_with_poco/versions.tf new file mode 100644 index 0000000000..2e21959c69 --- /dev/null +++ b/examples/simple_zonal_with_poco/versions.tf @@ -0,0 +1,27 @@ +/** + * Copyright 2021-2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + random = { + source = "hashicorp/random" + } + } + required_version = ">= 1.3" +} diff --git a/masq.tf b/masq.tf index b356aee251..cbb862d720 100644 --- a/masq.tf +++ b/masq.tf @@ -19,7 +19,7 @@ /****************************************** Create ip-masq-agent confimap *****************************************/ -resource "kubernetes_config_map" "ip-masq-agent" { +resource "kubernetes_config_map" "ip_masq_agent" { count = var.configure_ip_masq ? 1 : 0 metadata { diff --git a/modules/acm/versions.tf b/modules/acm/versions.tf index 513d6d9f1c..66b512bb3b 100644 --- a/modules/acm/versions.tf +++ b/modules/acm/versions.tf @@ -19,11 +19,11 @@ terraform { required_version = ">= 0.13.0" provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:acm/v33.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:acm/v34.0.0" } provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:acm/v33.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:acm/v34.0.0" } required_providers { diff --git a/modules/asm/README.md b/modules/asm/README.md index 0c8d7197af..5b3e8f5ead 100644 --- a/modules/asm/README.md +++ b/modules/asm/README.md @@ -2,6 +2,9 @@ This module installs [Anthos Service Mesh](https://cloud.google.com/service-mesh/docs) (ASM) in a Kubernetes Engine (GKE) cluster. +> [!WARNING] +> This Terraform Kubernetes Engine ASM Submodule has been deprecated in release v34.0.0 and will be removed entirely in a future release. It is recommended to migrate to the [google_gke_hub_feature](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/gke_hub_feature#example-usage---enable-fleet-default-member-config-service-mesh) and [google_gke_hub_feature_membership](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/gke_hub_feature_membership#example-usage---service-mesh) resources. + ## Usage The ASM module requires Terraform 0.14+. There are a few prerequisites to using this module that can be done either through Terraform and/or manually: diff --git a/modules/asm/versions.tf b/modules/asm/versions.tf index cfaff579cd..1227961368 100644 --- a/modules/asm/versions.tf +++ b/modules/asm/versions.tf @@ -36,10 +36,10 @@ terraform { } provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:asm/v33.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:asm/v34.0.0" } provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:asm/v33.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:asm/v34.0.0" } } diff --git a/modules/auth/versions.tf b/modules/auth/versions.tf index b1083d8ede..a1a3c2543b 100644 --- a/modules/auth/versions.tf +++ b/modules/auth/versions.tf @@ -26,6 +26,6 @@ terraform { } provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:auth/v33.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:auth/v34.0.0" } } diff --git a/modules/beta-autopilot-private-cluster/README.md b/modules/beta-autopilot-private-cluster/README.md index 930a3c0199..bbd101ad5b 100644 --- a/modules/beta-autopilot-private-cluster/README.md +++ b/modules/beta-autopilot-private-cluster/README.md @@ -152,10 +152,8 @@ Then perform the following commands on the root folder: | shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | | stack\_type | The stack type to use for this cluster. Either `IPV4` or `IPV4_IPV6`. Defaults to `IPV4`. | `string` | `"IPV4"` | no | | stateful\_ha | Whether the Stateful HA Addon is enabled for this cluster. | `bool` | `false` | no | -| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | | subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes | | timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | -| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | `list(string)` | `[]` | no | | workload\_config\_audit\_mode | (beta) Sets which mode of auditing should be used for the cluster's workloads. Accepted values are DISABLED, BASIC. | `string` | `"DISABLED"` | no | | workload\_vulnerability\_mode | (beta) Sets which mode to use for Protect workload vulnerability scanning feature. Accepted values are DISABLED, BASIC. | `string` | `""` | no | | zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | `list(string)` | `[]` | no | diff --git a/modules/beta-autopilot-private-cluster/dns.tf b/modules/beta-autopilot-private-cluster/dns.tf index d9c4d7518f..3353d2dbb6 100644 --- a/modules/beta-autopilot-private-cluster/dns.tf +++ b/modules/beta-autopilot-private-cluster/dns.tf @@ -16,73 +16,3 @@ // This file was automatically generated from a template in ./autogen/main -/****************************************** - Manage kube-dns configmaps - *****************************************/ - -resource "kubernetes_config_map_v1_data" "kube-dns" { - count = local.custom_kube_dns_config && !local.upstream_nameservers_config ? 1 : 0 - - metadata { - name = "kube-dns" - namespace = "kube-system" - } - - data = { - stubDomains = < 0 - upstream_nameservers_config = length(var.upstream_nameservers) > 0 - network_project_id = var.network_project_id != "" ? var.network_project_id : var.project_id - zone_count = length(var.zones) - cluster_type = var.regional ? "regional" : "zonal" + network_project_id = var.network_project_id != "" ? var.network_project_id : var.project_id + zone_count = length(var.zones) + cluster_type = var.regional ? "regional" : "zonal" cluster_subnet_cidr = var.add_cluster_firewall_rules ? data.google_compute_subnetwork.gke_subnetwork[0].ip_cidr_range : null cluster_alias_ranges_cidr = var.add_cluster_firewall_rules ? { for range in toset(data.google_compute_subnetwork.gke_subnetwork[0].secondary_ip_range) : range.range_name => range.ip_cidr_range } : {} diff --git a/modules/beta-autopilot-private-cluster/masq.tf b/modules/beta-autopilot-private-cluster/masq.tf index 65d3cc83c3..617d6afb9f 100644 --- a/modules/beta-autopilot-private-cluster/masq.tf +++ b/modules/beta-autopilot-private-cluster/masq.tf @@ -19,7 +19,7 @@ /****************************************** Create ip-masq-agent confimap *****************************************/ -resource "kubernetes_config_map" "ip-masq-agent" { +resource "kubernetes_config_map" "ip_masq_agent" { count = var.configure_ip_masq ? 1 : 0 metadata { diff --git a/modules/beta-autopilot-private-cluster/moved.tf b/modules/beta-autopilot-private-cluster/moved.tf new file mode 100644 index 0000000000..5535a5721e --- /dev/null +++ b/modules/beta-autopilot-private-cluster/moved.tf @@ -0,0 +1,46 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +moved { + from = kubernetes_config_map.ip-masq-agent + to = kubernetes_config_map.ip_masq_agent +} + +moved { + from = google_project_iam_member.cluster_service_account-nodeService_account + to = google_project_iam_member.cluster_service_account_node_service_account +} + +moved { + from = google_project_iam_member.cluster_service_account-metric_writer + to = google_project_iam_member.cluster_service_account_metric_writer +} + +moved { + from = google_project_iam_member.cluster_service_account-resourceMetadata-writer + to = google_project_iam_member.cluster_service_account_resource_metadata_writer +} + +moved { + from = google_project_iam_member.cluster_service_account-gcr + to = google_project_iam_member.cluster_service_account_gcr +} + +moved { + from = google_project_iam_member.cluster_service_account-artifact-registry + to = google_project_iam_member.cluster_service_account_artifact_registry +} diff --git a/modules/beta-autopilot-private-cluster/sa.tf b/modules/beta-autopilot-private-cluster/sa.tf index 34106c5e62..8082a60312 100644 --- a/modules/beta-autopilot-private-cluster/sa.tf +++ b/modules/beta-autopilot-private-cluster/sa.tf @@ -46,35 +46,35 @@ resource "google_service_account" "cluster_service_account" { display_name = "Terraform-managed service account for cluster ${var.name}" } -resource "google_project_iam_member" "cluster_service_account-nodeService_account" { +resource "google_project_iam_member" "cluster_service_account_node_service_account" { count = var.create_service_account ? 1 : 0 project = google_service_account.cluster_service_account[0].project role = "roles/container.defaultNodeServiceAccount" member = google_service_account.cluster_service_account[0].member } -resource "google_project_iam_member" "cluster_service_account-metric_writer" { +resource "google_project_iam_member" "cluster_service_account_metric_writer" { count = var.create_service_account ? 1 : 0 project = google_service_account.cluster_service_account[0].project role = "roles/monitoring.metricWriter" member = google_service_account.cluster_service_account[0].member } -resource "google_project_iam_member" "cluster_service_account-resourceMetadata-writer" { +resource "google_project_iam_member" "cluster_service_account_resource_metadata_writer" { count = var.create_service_account ? 1 : 0 project = google_service_account.cluster_service_account[0].project role = "roles/stackdriver.resourceMetadata.writer" member = google_service_account.cluster_service_account[0].member } -resource "google_project_iam_member" "cluster_service_account-gcr" { +resource "google_project_iam_member" "cluster_service_account_gcr" { for_each = var.create_service_account && var.grant_registry_access ? toset(local.registry_projects_list) : [] project = each.key role = "roles/storage.objectViewer" member = "serviceAccount:${google_service_account.cluster_service_account[0].email}" } -resource "google_project_iam_member" "cluster_service_account-artifact-registry" { +resource "google_project_iam_member" "cluster_service_account_artifact_registry" { for_each = var.create_service_account && var.grant_registry_access ? toset(local.registry_projects_list) : [] project = each.key role = "roles/artifactregistry.reader" diff --git a/modules/beta-autopilot-private-cluster/variables.tf b/modules/beta-autopilot-private-cluster/variables.tf index b8d087b804..19a37b243a 100644 --- a/modules/beta-autopilot-private-cluster/variables.tf +++ b/modules/beta-autopilot-private-cluster/variables.tf @@ -185,17 +185,6 @@ variable "network_tags" { default = [] } -variable "stub_domains" { - type = map(list(string)) - description = "Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server" - default = {} -} - -variable "upstream_nameservers" { - type = list(string) - description = "If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf" - default = [] -} variable "non_masquerade_cidrs" { type = list(string) diff --git a/modules/beta-autopilot-private-cluster/versions.tf b/modules/beta-autopilot-private-cluster/versions.tf index f3b6dd5e0d..daa0faa199 100644 --- a/modules/beta-autopilot-private-cluster/versions.tf +++ b/modules/beta-autopilot-private-cluster/versions.tf @@ -39,6 +39,6 @@ terraform { } } provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:beta-autopilot-private-cluster/v33.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:beta-autopilot-private-cluster/v34.0.0" } } diff --git a/modules/beta-autopilot-public-cluster/README.md b/modules/beta-autopilot-public-cluster/README.md index 9df58422ce..52614332f8 100644 --- a/modules/beta-autopilot-public-cluster/README.md +++ b/modules/beta-autopilot-public-cluster/README.md @@ -140,10 +140,8 @@ Then perform the following commands on the root folder: | shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | | stack\_type | The stack type to use for this cluster. Either `IPV4` or `IPV4_IPV6`. Defaults to `IPV4`. | `string` | `"IPV4"` | no | | stateful\_ha | Whether the Stateful HA Addon is enabled for this cluster. | `bool` | `false` | no | -| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | | subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes | | timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | -| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | `list(string)` | `[]` | no | | workload\_config\_audit\_mode | (beta) Sets which mode of auditing should be used for the cluster's workloads. Accepted values are DISABLED, BASIC. | `string` | `"DISABLED"` | no | | workload\_vulnerability\_mode | (beta) Sets which mode to use for Protect workload vulnerability scanning feature. Accepted values are DISABLED, BASIC. | `string` | `""` | no | | zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | `list(string)` | `[]` | no | diff --git a/modules/beta-autopilot-public-cluster/dns.tf b/modules/beta-autopilot-public-cluster/dns.tf index d9c4d7518f..3353d2dbb6 100644 --- a/modules/beta-autopilot-public-cluster/dns.tf +++ b/modules/beta-autopilot-public-cluster/dns.tf @@ -16,73 +16,3 @@ // This file was automatically generated from a template in ./autogen/main -/****************************************** - Manage kube-dns configmaps - *****************************************/ - -resource "kubernetes_config_map_v1_data" "kube-dns" { - count = local.custom_kube_dns_config && !local.upstream_nameservers_config ? 1 : 0 - - metadata { - name = "kube-dns" - namespace = "kube-system" - } - - data = { - stubDomains = < 0 - upstream_nameservers_config = length(var.upstream_nameservers) > 0 - network_project_id = var.network_project_id != "" ? var.network_project_id : var.project_id - zone_count = length(var.zones) - cluster_type = var.regional ? "regional" : "zonal" + network_project_id = var.network_project_id != "" ? var.network_project_id : var.project_id + zone_count = length(var.zones) + cluster_type = var.regional ? "regional" : "zonal" cluster_subnet_cidr = var.add_cluster_firewall_rules ? data.google_compute_subnetwork.gke_subnetwork[0].ip_cidr_range : null cluster_alias_ranges_cidr = var.add_cluster_firewall_rules ? { for range in toset(data.google_compute_subnetwork.gke_subnetwork[0].secondary_ip_range) : range.range_name => range.ip_cidr_range } : {} diff --git a/modules/beta-autopilot-public-cluster/masq.tf b/modules/beta-autopilot-public-cluster/masq.tf index 65d3cc83c3..617d6afb9f 100644 --- a/modules/beta-autopilot-public-cluster/masq.tf +++ b/modules/beta-autopilot-public-cluster/masq.tf @@ -19,7 +19,7 @@ /****************************************** Create ip-masq-agent confimap *****************************************/ -resource "kubernetes_config_map" "ip-masq-agent" { +resource "kubernetes_config_map" "ip_masq_agent" { count = var.configure_ip_masq ? 1 : 0 metadata { diff --git a/modules/beta-autopilot-public-cluster/moved.tf b/modules/beta-autopilot-public-cluster/moved.tf new file mode 100644 index 0000000000..5535a5721e --- /dev/null +++ b/modules/beta-autopilot-public-cluster/moved.tf @@ -0,0 +1,46 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +moved { + from = kubernetes_config_map.ip-masq-agent + to = kubernetes_config_map.ip_masq_agent +} + +moved { + from = google_project_iam_member.cluster_service_account-nodeService_account + to = google_project_iam_member.cluster_service_account_node_service_account +} + +moved { + from = google_project_iam_member.cluster_service_account-metric_writer + to = google_project_iam_member.cluster_service_account_metric_writer +} + +moved { + from = google_project_iam_member.cluster_service_account-resourceMetadata-writer + to = google_project_iam_member.cluster_service_account_resource_metadata_writer +} + +moved { + from = google_project_iam_member.cluster_service_account-gcr + to = google_project_iam_member.cluster_service_account_gcr +} + +moved { + from = google_project_iam_member.cluster_service_account-artifact-registry + to = google_project_iam_member.cluster_service_account_artifact_registry +} diff --git a/modules/beta-autopilot-public-cluster/sa.tf b/modules/beta-autopilot-public-cluster/sa.tf index 34106c5e62..8082a60312 100644 --- a/modules/beta-autopilot-public-cluster/sa.tf +++ b/modules/beta-autopilot-public-cluster/sa.tf @@ -46,35 +46,35 @@ resource "google_service_account" "cluster_service_account" { display_name = "Terraform-managed service account for cluster ${var.name}" } -resource "google_project_iam_member" "cluster_service_account-nodeService_account" { +resource "google_project_iam_member" "cluster_service_account_node_service_account" { count = var.create_service_account ? 1 : 0 project = google_service_account.cluster_service_account[0].project role = "roles/container.defaultNodeServiceAccount" member = google_service_account.cluster_service_account[0].member } -resource "google_project_iam_member" "cluster_service_account-metric_writer" { +resource "google_project_iam_member" "cluster_service_account_metric_writer" { count = var.create_service_account ? 1 : 0 project = google_service_account.cluster_service_account[0].project role = "roles/monitoring.metricWriter" member = google_service_account.cluster_service_account[0].member } -resource "google_project_iam_member" "cluster_service_account-resourceMetadata-writer" { +resource "google_project_iam_member" "cluster_service_account_resource_metadata_writer" { count = var.create_service_account ? 1 : 0 project = google_service_account.cluster_service_account[0].project role = "roles/stackdriver.resourceMetadata.writer" member = google_service_account.cluster_service_account[0].member } -resource "google_project_iam_member" "cluster_service_account-gcr" { +resource "google_project_iam_member" "cluster_service_account_gcr" { for_each = var.create_service_account && var.grant_registry_access ? toset(local.registry_projects_list) : [] project = each.key role = "roles/storage.objectViewer" member = "serviceAccount:${google_service_account.cluster_service_account[0].email}" } -resource "google_project_iam_member" "cluster_service_account-artifact-registry" { +resource "google_project_iam_member" "cluster_service_account_artifact_registry" { for_each = var.create_service_account && var.grant_registry_access ? toset(local.registry_projects_list) : [] project = each.key role = "roles/artifactregistry.reader" diff --git a/modules/beta-autopilot-public-cluster/variables.tf b/modules/beta-autopilot-public-cluster/variables.tf index 22a05337af..377b5c561c 100644 --- a/modules/beta-autopilot-public-cluster/variables.tf +++ b/modules/beta-autopilot-public-cluster/variables.tf @@ -185,17 +185,6 @@ variable "network_tags" { default = [] } -variable "stub_domains" { - type = map(list(string)) - description = "Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server" - default = {} -} - -variable "upstream_nameservers" { - type = list(string) - description = "If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf" - default = [] -} variable "non_masquerade_cidrs" { type = list(string) diff --git a/modules/beta-autopilot-public-cluster/versions.tf b/modules/beta-autopilot-public-cluster/versions.tf index a2046e5d98..daffa17c4d 100644 --- a/modules/beta-autopilot-public-cluster/versions.tf +++ b/modules/beta-autopilot-public-cluster/versions.tf @@ -39,6 +39,6 @@ terraform { } } provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:beta-autopilot-public-cluster/v33.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:beta-autopilot-public-cluster/v34.0.0" } } diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md index 892f57c723..bcff4546cc 100644 --- a/modules/beta-private-cluster-update-variant/README.md +++ b/modules/beta-private-cluster-update-variant/README.md @@ -233,6 +233,7 @@ Then perform the following commands on the root folder: | http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no | | identity\_namespace | The workload pool to attach all Kubernetes service accounts to. (Default value of `enabled` automatically sets project-based pool `[project_id].svc.id.goog`) | `string` | `"enabled"` | no | | initial\_node\_count | The number of nodes to create in this cluster's default node pool. | `number` | `0` | no | +| insecure\_kubelet\_readonly\_port\_enabled | Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`. | `bool` | `null` | no | | ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | `bool` | `false` | no | | ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | `string` | `"60s"` | no | | ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes | @@ -377,6 +378,7 @@ The node_pools variable takes the following parameters: | gpu_partition_size | Size of partitions to create on the GPU | null | Optional | | image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS_CONTAINERD | Optional | | initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource. Defaults to the value of min_count | " " | Optional | +| insecure_kubelet_readonly_port_enabled | (boolean) Whether or not to enable the insecure Kubelet readonly port. | null | Optional | | key | The key required for the taint | | Required | | logging_variant | The type of logging agent that is deployed by default for newly created node pools in the cluster. Valid values include DEFAULT and MAX_THROUGHPUT. | DEFAULT | Optional | | local_ssd_count | The amount of local SSD disks that will be attached to each cluster node and may be used as a `hostpath` volume or a `local` PersistentVolume. | 0 | Optional | diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf index 74d7cca8db..af766203d9 100644 --- a/modules/beta-private-cluster-update-variant/cluster.tf +++ b/modules/beta-private-cluster-update-variant/cluster.tf @@ -225,10 +225,10 @@ resource "google_container_cluster" "primary" { } dynamic "node_pool_auto_config" { - for_each = var.cluster_autoscaling.enabled && length(var.network_tags) > 0 ? [1] : [] + for_each = var.cluster_autoscaling.enabled && (length(var.network_tags) > 0 || var.add_cluster_firewall_rules) ? [1] : [] content { network_tags { - tags = var.network_tags + tags = var.add_cluster_firewall_rules ? (concat(var.network_tags, [local.cluster_network_tag])) : var.network_tags } } } @@ -453,6 +453,21 @@ resource "google_container_cluster" "primary" { } } + dynamic "kubelet_config" { + for_each = length(setintersection( + keys(var.node_pools[0]), + ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "insecure_kubelet_readonly_port_enabled", "pod_pids_limit"] + )) != 0 || var.insecure_kubelet_readonly_port_enabled != null ? [1] : [] + + content { + cpu_manager_policy = lookup(var.node_pools[0], "cpu_manager_policy", "static") + cpu_cfs_quota = lookup(var.node_pools[0], "cpu_cfs_quota", null) + cpu_cfs_quota_period = lookup(var.node_pools[0], "cpu_cfs_quota_period", null) + insecure_kubelet_readonly_port_enabled = lookup(var.node_pools[0], "insecure_kubelet_readonly_port_enabled", var.insecure_kubelet_readonly_port_enabled) != null ? upper(tostring(lookup(var.node_pools[0], "insecure_kubelet_readonly_port_enabled", var.insecure_kubelet_readonly_port_enabled))) : null + pod_pids_limit = lookup(var.node_pools[0], "pod_pids_limit", null) + } + } + service_account = lookup(var.node_pools[0], "service_account", local.service_account) tags = concat( @@ -581,6 +596,7 @@ resource "google_container_cluster" "primary" { gcfs_config { enabled = var.enable_gcfs } + insecure_kubelet_readonly_port_enabled = var.insecure_kubelet_readonly_port_enabled != null ? upper(tostring(var.insecure_kubelet_readonly_port_enabled)) : null } } @@ -602,6 +618,7 @@ locals { "enable_secure_boot", "enable_integrity_monitoring", "local_ssd_count", + "local_ssd_ephemeral_count", "machine_type", "placement_policy", "max_pods_per_node", @@ -619,6 +636,7 @@ locals { "reservation_affinity_key", "reservation_affinity_values", "enable_confidential_nodes", + "secondary_boot_disk", ] } @@ -629,7 +647,7 @@ locals { resource "random_id" "name" { for_each = merge(local.node_pools, local.windows_node_pools) byte_length = 2 - prefix = format("%s-", lookup(each.value, "name")) + prefix = "${each.key}-" keepers = merge( zipmap( local.force_node_pool_recreation_resources, @@ -913,14 +931,15 @@ resource "google_container_node_pool" "pools" { dynamic "kubelet_config" { for_each = length(setintersection( keys(each.value), - ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "pod_pids_limit"] + ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "insecure_kubelet_readonly_port_enabled", "pod_pids_limit"] )) != 0 ? [1] : [] content { - cpu_manager_policy = lookup(each.value, "cpu_manager_policy", "static") - cpu_cfs_quota = lookup(each.value, "cpu_cfs_quota", null) - cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null) - pod_pids_limit = lookup(each.value, "pod_pids_limit", null) + cpu_manager_policy = lookup(each.value, "cpu_manager_policy", "static") + cpu_cfs_quota = lookup(each.value, "cpu_cfs_quota", null) + cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null) + insecure_kubelet_readonly_port_enabled = lookup(each.value, "insecure_kubelet_readonly_port_enabled", null) != null ? upper(tostring(each.value.insecure_kubelet_readonly_port_enabled)) : null + pod_pids_limit = lookup(each.value, "pod_pids_limit", null) } } @@ -955,7 +974,7 @@ resource "google_container_node_pool" "pools" { } dynamic "confidential_nodes" { - for_each = lookup(each.value, "enable_confidential_nodes", null) != null ? [each.value.confidential_nodes] : [] + for_each = lookup(each.value, "enable_confidential_nodes", null) != null ? [each.value.enable_confidential_nodes] : [] content { enabled = confidential_nodes.value } @@ -1217,14 +1236,15 @@ resource "google_container_node_pool" "windows_pools" { dynamic "kubelet_config" { for_each = length(setintersection( keys(each.value), - ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "pod_pids_limit"] + ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "insecure_kubelet_readonly_port_enabled", "pod_pids_limit"] )) != 0 ? [1] : [] content { - cpu_manager_policy = lookup(each.value, "cpu_manager_policy", "static") - cpu_cfs_quota = lookup(each.value, "cpu_cfs_quota", null) - cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null) - pod_pids_limit = lookup(each.value, "pod_pids_limit", null) + cpu_manager_policy = lookup(each.value, "cpu_manager_policy", "static") + cpu_cfs_quota = lookup(each.value, "cpu_cfs_quota", null) + cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null) + insecure_kubelet_readonly_port_enabled = lookup(each.value, "insecure_kubelet_readonly_port_enabled", null) != null ? upper(tostring(each.value.insecure_kubelet_readonly_port_enabled)) : null + pod_pids_limit = lookup(each.value, "pod_pids_limit", null) } } @@ -1244,7 +1264,7 @@ resource "google_container_node_pool" "windows_pools" { } dynamic "confidential_nodes" { - for_each = lookup(each.value, "enable_confidential_nodes", null) != null ? [each.value.confidential_nodes] : [] + for_each = lookup(each.value, "enable_confidential_nodes", null) != null ? [each.value.enable_confidential_nodes] : [] content { enabled = confidential_nodes.value } diff --git a/modules/beta-private-cluster-update-variant/dns.tf b/modules/beta-private-cluster-update-variant/dns.tf index bf0d05b723..c120c2be92 100644 --- a/modules/beta-private-cluster-update-variant/dns.tf +++ b/modules/beta-private-cluster-update-variant/dns.tf @@ -20,7 +20,7 @@ Manage kube-dns configmaps *****************************************/ -resource "kubernetes_config_map_v1_data" "kube-dns" { +resource "kubernetes_config_map_v1_data" "kube_dns" { count = local.custom_kube_dns_config && !local.upstream_nameservers_config ? 1 : 0 metadata { @@ -42,7 +42,7 @@ EOF ] } -resource "kubernetes_config_map_v1_data" "kube-dns-upstream-namservers" { +resource "kubernetes_config_map_v1_data" "kube_dns_upstream_nameservers" { count = !local.custom_kube_dns_config && local.upstream_nameservers_config ? 1 : 0 metadata { @@ -64,7 +64,7 @@ EOF ] } -resource "kubernetes_config_map_v1_data" "kube-dns-upstream-nameservers-and-stub-domains" { +resource "kubernetes_config_map_v1_data" "kube_dns_upstream_nameservers_and_stub_domains" { count = local.custom_kube_dns_config && local.upstream_nameservers_config ? 1 : 0 metadata { diff --git a/modules/beta-private-cluster-update-variant/masq.tf b/modules/beta-private-cluster-update-variant/masq.tf index b356aee251..cbb862d720 100644 --- a/modules/beta-private-cluster-update-variant/masq.tf +++ b/modules/beta-private-cluster-update-variant/masq.tf @@ -19,7 +19,7 @@ /****************************************** Create ip-masq-agent confimap *****************************************/ -resource "kubernetes_config_map" "ip-masq-agent" { +resource "kubernetes_config_map" "ip_masq_agent" { count = var.configure_ip_masq ? 1 : 0 metadata { diff --git a/modules/beta-private-cluster-update-variant/moved.tf b/modules/beta-private-cluster-update-variant/moved.tf new file mode 100644 index 0000000000..a745d16489 --- /dev/null +++ b/modules/beta-private-cluster-update-variant/moved.tf @@ -0,0 +1,62 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# Updates for kebab to snake case, to match best practices and Google style. +moved { + from = kubernetes_config_map_v1_data.kube-dns + to = kubernetes_config_map_v1_data.kube_dns +} + +# Typo fix and snake case at the same time +moved { + from = kubernetes_config_map_v1_data.kube-dns-upstream-namservers + to = kubernetes_config_map_v1_data.kube_dns_upstream_nameservers +} + +moved { + from = kubernetes_config_map_v1_data.kube-dns-upstream-nameservers-and-stub-domains + to = kubernetes_config_map_v1_data.kube_dns_upstream_nameservers_and_stub_domains +} + +moved { + from = kubernetes_config_map.ip-masq-agent + to = kubernetes_config_map.ip_masq_agent +} + +moved { + from = google_project_iam_member.cluster_service_account-nodeService_account + to = google_project_iam_member.cluster_service_account_node_service_account +} + +moved { + from = google_project_iam_member.cluster_service_account-metric_writer + to = google_project_iam_member.cluster_service_account_metric_writer +} + +moved { + from = google_project_iam_member.cluster_service_account-resourceMetadata-writer + to = google_project_iam_member.cluster_service_account_resource_metadata_writer +} + +moved { + from = google_project_iam_member.cluster_service_account-gcr + to = google_project_iam_member.cluster_service_account_gcr +} + +moved { + from = google_project_iam_member.cluster_service_account-artifact-registry + to = google_project_iam_member.cluster_service_account_artifact_registry +} diff --git a/modules/beta-private-cluster-update-variant/sa.tf b/modules/beta-private-cluster-update-variant/sa.tf index 34106c5e62..8082a60312 100644 --- a/modules/beta-private-cluster-update-variant/sa.tf +++ b/modules/beta-private-cluster-update-variant/sa.tf @@ -46,35 +46,35 @@ resource "google_service_account" "cluster_service_account" { display_name = "Terraform-managed service account for cluster ${var.name}" } -resource "google_project_iam_member" "cluster_service_account-nodeService_account" { +resource "google_project_iam_member" "cluster_service_account_node_service_account" { count = var.create_service_account ? 1 : 0 project = google_service_account.cluster_service_account[0].project role = "roles/container.defaultNodeServiceAccount" member = google_service_account.cluster_service_account[0].member } -resource "google_project_iam_member" "cluster_service_account-metric_writer" { +resource "google_project_iam_member" "cluster_service_account_metric_writer" { count = var.create_service_account ? 1 : 0 project = google_service_account.cluster_service_account[0].project role = "roles/monitoring.metricWriter" member = google_service_account.cluster_service_account[0].member } -resource "google_project_iam_member" "cluster_service_account-resourceMetadata-writer" { +resource "google_project_iam_member" "cluster_service_account_resource_metadata_writer" { count = var.create_service_account ? 1 : 0 project = google_service_account.cluster_service_account[0].project role = "roles/stackdriver.resourceMetadata.writer" member = google_service_account.cluster_service_account[0].member } -resource "google_project_iam_member" "cluster_service_account-gcr" { +resource "google_project_iam_member" "cluster_service_account_gcr" { for_each = var.create_service_account && var.grant_registry_access ? toset(local.registry_projects_list) : [] project = each.key role = "roles/storage.objectViewer" member = "serviceAccount:${google_service_account.cluster_service_account[0].email}" } -resource "google_project_iam_member" "cluster_service_account-artifact-registry" { +resource "google_project_iam_member" "cluster_service_account_artifact_registry" { for_each = var.create_service_account && var.grant_registry_access ? toset(local.registry_projects_list) : [] project = each.key role = "roles/artifactregistry.reader" diff --git a/modules/beta-private-cluster-update-variant/variables.tf b/modules/beta-private-cluster-update-variant/variables.tf index 195db8f643..12bf92a229 100644 --- a/modules/beta-private-cluster-update-variant/variables.tf +++ b/modules/beta-private-cluster-update-variant/variables.tf @@ -108,6 +108,12 @@ variable "service_external_ips" { default = false } +variable "insecure_kubelet_readonly_port_enabled" { + type = bool + description = "Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`." + default = null +} + variable "datapath_provider" { type = string description = "The desired datapath provider for this cluster. By default, `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation. `ADVANCED_DATAPATH` enables Dataplane-V2 feature." diff --git a/modules/beta-private-cluster-update-variant/versions.tf b/modules/beta-private-cluster-update-variant/versions.tf index 9726aee6f5..aec77ac32f 100644 --- a/modules/beta-private-cluster-update-variant/versions.tf +++ b/modules/beta-private-cluster-update-variant/versions.tf @@ -21,11 +21,11 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 5.40.0, < 7" + version = ">= 5.44.2, !=6.0.0, !=6.0.1, !=6.1.0, !=6.2.0, !=6.3.0, !=6.4.0, !=6.5.0, !=6.6.0, < 7" } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.40.0, < 7" + version = ">= 5.44.2, !=6.0.0, !=6.0.1, !=6.1.0, !=6.2.0, !=6.3.0, !=6.4.0, !=6.5.0, !=6.6.0, < 7" } kubernetes = { source = "hashicorp/kubernetes" @@ -37,6 +37,6 @@ terraform { } } provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:beta-private-cluster-update-variant/v33.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:beta-private-cluster-update-variant/v34.0.0" } } diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md index 9fb426c846..cfc7c03338 100644 --- a/modules/beta-private-cluster/README.md +++ b/modules/beta-private-cluster/README.md @@ -211,6 +211,7 @@ Then perform the following commands on the root folder: | http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no | | identity\_namespace | The workload pool to attach all Kubernetes service accounts to. (Default value of `enabled` automatically sets project-based pool `[project_id].svc.id.goog`) | `string` | `"enabled"` | no | | initial\_node\_count | The number of nodes to create in this cluster's default node pool. | `number` | `0` | no | +| insecure\_kubelet\_readonly\_port\_enabled | Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`. | `bool` | `null` | no | | ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | `bool` | `false` | no | | ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | `string` | `"60s"` | no | | ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes | @@ -355,6 +356,7 @@ The node_pools variable takes the following parameters: | gpu_partition_size | Size of partitions to create on the GPU | null | Optional | | image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS_CONTAINERD | Optional | | initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource. Defaults to the value of min_count | " " | Optional | +| insecure_kubelet_readonly_port_enabled | (boolean) Whether or not to enable the insecure Kubelet readonly port. | null | Optional | | key | The key required for the taint | | Required | | logging_variant | The type of logging agent that is deployed by default for newly created node pools in the cluster. Valid values include DEFAULT and MAX_THROUGHPUT. | DEFAULT | Optional | | local_ssd_count | The amount of local SSD disks that will be attached to each cluster node and may be used as a `hostpath` volume or a `local` PersistentVolume. | 0 | Optional | diff --git a/modules/beta-private-cluster/cluster.tf b/modules/beta-private-cluster/cluster.tf index 364aff70b8..6e5313b2d7 100644 --- a/modules/beta-private-cluster/cluster.tf +++ b/modules/beta-private-cluster/cluster.tf @@ -225,10 +225,10 @@ resource "google_container_cluster" "primary" { } dynamic "node_pool_auto_config" { - for_each = var.cluster_autoscaling.enabled && length(var.network_tags) > 0 ? [1] : [] + for_each = var.cluster_autoscaling.enabled && (length(var.network_tags) > 0 || var.add_cluster_firewall_rules) ? [1] : [] content { network_tags { - tags = var.network_tags + tags = var.add_cluster_firewall_rules ? (concat(var.network_tags, [local.cluster_network_tag])) : var.network_tags } } } @@ -453,6 +453,21 @@ resource "google_container_cluster" "primary" { } } + dynamic "kubelet_config" { + for_each = length(setintersection( + keys(var.node_pools[0]), + ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "insecure_kubelet_readonly_port_enabled", "pod_pids_limit"] + )) != 0 || var.insecure_kubelet_readonly_port_enabled != null ? [1] : [] + + content { + cpu_manager_policy = lookup(var.node_pools[0], "cpu_manager_policy", "static") + cpu_cfs_quota = lookup(var.node_pools[0], "cpu_cfs_quota", null) + cpu_cfs_quota_period = lookup(var.node_pools[0], "cpu_cfs_quota_period", null) + insecure_kubelet_readonly_port_enabled = lookup(var.node_pools[0], "insecure_kubelet_readonly_port_enabled", var.insecure_kubelet_readonly_port_enabled) != null ? upper(tostring(lookup(var.node_pools[0], "insecure_kubelet_readonly_port_enabled", var.insecure_kubelet_readonly_port_enabled))) : null + pod_pids_limit = lookup(var.node_pools[0], "pod_pids_limit", null) + } + } + service_account = lookup(var.node_pools[0], "service_account", local.service_account) tags = concat( @@ -581,6 +596,7 @@ resource "google_container_cluster" "primary" { gcfs_config { enabled = var.enable_gcfs } + insecure_kubelet_readonly_port_enabled = var.insecure_kubelet_readonly_port_enabled != null ? upper(tostring(var.insecure_kubelet_readonly_port_enabled)) : null } } @@ -830,14 +846,15 @@ resource "google_container_node_pool" "pools" { dynamic "kubelet_config" { for_each = length(setintersection( keys(each.value), - ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "pod_pids_limit"] + ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "insecure_kubelet_readonly_port_enabled", "pod_pids_limit"] )) != 0 ? [1] : [] content { - cpu_manager_policy = lookup(each.value, "cpu_manager_policy", "static") - cpu_cfs_quota = lookup(each.value, "cpu_cfs_quota", null) - cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null) - pod_pids_limit = lookup(each.value, "pod_pids_limit", null) + cpu_manager_policy = lookup(each.value, "cpu_manager_policy", "static") + cpu_cfs_quota = lookup(each.value, "cpu_cfs_quota", null) + cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null) + insecure_kubelet_readonly_port_enabled = lookup(each.value, "insecure_kubelet_readonly_port_enabled", null) != null ? upper(tostring(each.value.insecure_kubelet_readonly_port_enabled)) : null + pod_pids_limit = lookup(each.value, "pod_pids_limit", null) } } @@ -872,7 +889,7 @@ resource "google_container_node_pool" "pools" { } dynamic "confidential_nodes" { - for_each = lookup(each.value, "enable_confidential_nodes", null) != null ? [each.value.confidential_nodes] : [] + for_each = lookup(each.value, "enable_confidential_nodes", null) != null ? [each.value.enable_confidential_nodes] : [] content { enabled = confidential_nodes.value } @@ -1133,14 +1150,15 @@ resource "google_container_node_pool" "windows_pools" { dynamic "kubelet_config" { for_each = length(setintersection( keys(each.value), - ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "pod_pids_limit"] + ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "insecure_kubelet_readonly_port_enabled", "pod_pids_limit"] )) != 0 ? [1] : [] content { - cpu_manager_policy = lookup(each.value, "cpu_manager_policy", "static") - cpu_cfs_quota = lookup(each.value, "cpu_cfs_quota", null) - cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null) - pod_pids_limit = lookup(each.value, "pod_pids_limit", null) + cpu_manager_policy = lookup(each.value, "cpu_manager_policy", "static") + cpu_cfs_quota = lookup(each.value, "cpu_cfs_quota", null) + cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null) + insecure_kubelet_readonly_port_enabled = lookup(each.value, "insecure_kubelet_readonly_port_enabled", null) != null ? upper(tostring(each.value.insecure_kubelet_readonly_port_enabled)) : null + pod_pids_limit = lookup(each.value, "pod_pids_limit", null) } } @@ -1160,7 +1178,7 @@ resource "google_container_node_pool" "windows_pools" { } dynamic "confidential_nodes" { - for_each = lookup(each.value, "enable_confidential_nodes", null) != null ? [each.value.confidential_nodes] : [] + for_each = lookup(each.value, "enable_confidential_nodes", null) != null ? [each.value.enable_confidential_nodes] : [] content { enabled = confidential_nodes.value } diff --git a/modules/beta-private-cluster/dns.tf b/modules/beta-private-cluster/dns.tf index bf0d05b723..c120c2be92 100644 --- a/modules/beta-private-cluster/dns.tf +++ b/modules/beta-private-cluster/dns.tf @@ -20,7 +20,7 @@ Manage kube-dns configmaps *****************************************/ -resource "kubernetes_config_map_v1_data" "kube-dns" { +resource "kubernetes_config_map_v1_data" "kube_dns" { count = local.custom_kube_dns_config && !local.upstream_nameservers_config ? 1 : 0 metadata { @@ -42,7 +42,7 @@ EOF ] } -resource "kubernetes_config_map_v1_data" "kube-dns-upstream-namservers" { +resource "kubernetes_config_map_v1_data" "kube_dns_upstream_nameservers" { count = !local.custom_kube_dns_config && local.upstream_nameservers_config ? 1 : 0 metadata { @@ -64,7 +64,7 @@ EOF ] } -resource "kubernetes_config_map_v1_data" "kube-dns-upstream-nameservers-and-stub-domains" { +resource "kubernetes_config_map_v1_data" "kube_dns_upstream_nameservers_and_stub_domains" { count = local.custom_kube_dns_config && local.upstream_nameservers_config ? 1 : 0 metadata { diff --git a/modules/beta-private-cluster/masq.tf b/modules/beta-private-cluster/masq.tf index b356aee251..cbb862d720 100644 --- a/modules/beta-private-cluster/masq.tf +++ b/modules/beta-private-cluster/masq.tf @@ -19,7 +19,7 @@ /****************************************** Create ip-masq-agent confimap *****************************************/ -resource "kubernetes_config_map" "ip-masq-agent" { +resource "kubernetes_config_map" "ip_masq_agent" { count = var.configure_ip_masq ? 1 : 0 metadata { diff --git a/modules/beta-private-cluster/moved.tf b/modules/beta-private-cluster/moved.tf new file mode 100644 index 0000000000..a745d16489 --- /dev/null +++ b/modules/beta-private-cluster/moved.tf @@ -0,0 +1,62 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# Updates for kebab to snake case, to match best practices and Google style. +moved { + from = kubernetes_config_map_v1_data.kube-dns + to = kubernetes_config_map_v1_data.kube_dns +} + +# Typo fix and snake case at the same time +moved { + from = kubernetes_config_map_v1_data.kube-dns-upstream-namservers + to = kubernetes_config_map_v1_data.kube_dns_upstream_nameservers +} + +moved { + from = kubernetes_config_map_v1_data.kube-dns-upstream-nameservers-and-stub-domains + to = kubernetes_config_map_v1_data.kube_dns_upstream_nameservers_and_stub_domains +} + +moved { + from = kubernetes_config_map.ip-masq-agent + to = kubernetes_config_map.ip_masq_agent +} + +moved { + from = google_project_iam_member.cluster_service_account-nodeService_account + to = google_project_iam_member.cluster_service_account_node_service_account +} + +moved { + from = google_project_iam_member.cluster_service_account-metric_writer + to = google_project_iam_member.cluster_service_account_metric_writer +} + +moved { + from = google_project_iam_member.cluster_service_account-resourceMetadata-writer + to = google_project_iam_member.cluster_service_account_resource_metadata_writer +} + +moved { + from = google_project_iam_member.cluster_service_account-gcr + to = google_project_iam_member.cluster_service_account_gcr +} + +moved { + from = google_project_iam_member.cluster_service_account-artifact-registry + to = google_project_iam_member.cluster_service_account_artifact_registry +} diff --git a/modules/beta-private-cluster/sa.tf b/modules/beta-private-cluster/sa.tf index 34106c5e62..8082a60312 100644 --- a/modules/beta-private-cluster/sa.tf +++ b/modules/beta-private-cluster/sa.tf @@ -46,35 +46,35 @@ resource "google_service_account" "cluster_service_account" { display_name = "Terraform-managed service account for cluster ${var.name}" } -resource "google_project_iam_member" "cluster_service_account-nodeService_account" { +resource "google_project_iam_member" "cluster_service_account_node_service_account" { count = var.create_service_account ? 1 : 0 project = google_service_account.cluster_service_account[0].project role = "roles/container.defaultNodeServiceAccount" member = google_service_account.cluster_service_account[0].member } -resource "google_project_iam_member" "cluster_service_account-metric_writer" { +resource "google_project_iam_member" "cluster_service_account_metric_writer" { count = var.create_service_account ? 1 : 0 project = google_service_account.cluster_service_account[0].project role = "roles/monitoring.metricWriter" member = google_service_account.cluster_service_account[0].member } -resource "google_project_iam_member" "cluster_service_account-resourceMetadata-writer" { +resource "google_project_iam_member" "cluster_service_account_resource_metadata_writer" { count = var.create_service_account ? 1 : 0 project = google_service_account.cluster_service_account[0].project role = "roles/stackdriver.resourceMetadata.writer" member = google_service_account.cluster_service_account[0].member } -resource "google_project_iam_member" "cluster_service_account-gcr" { +resource "google_project_iam_member" "cluster_service_account_gcr" { for_each = var.create_service_account && var.grant_registry_access ? toset(local.registry_projects_list) : [] project = each.key role = "roles/storage.objectViewer" member = "serviceAccount:${google_service_account.cluster_service_account[0].email}" } -resource "google_project_iam_member" "cluster_service_account-artifact-registry" { +resource "google_project_iam_member" "cluster_service_account_artifact_registry" { for_each = var.create_service_account && var.grant_registry_access ? toset(local.registry_projects_list) : [] project = each.key role = "roles/artifactregistry.reader" diff --git a/modules/beta-private-cluster/variables.tf b/modules/beta-private-cluster/variables.tf index 195db8f643..12bf92a229 100644 --- a/modules/beta-private-cluster/variables.tf +++ b/modules/beta-private-cluster/variables.tf @@ -108,6 +108,12 @@ variable "service_external_ips" { default = false } +variable "insecure_kubelet_readonly_port_enabled" { + type = bool + description = "Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`." + default = null +} + variable "datapath_provider" { type = string description = "The desired datapath provider for this cluster. By default, `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation. `ADVANCED_DATAPATH` enables Dataplane-V2 feature." diff --git a/modules/beta-private-cluster/versions.tf b/modules/beta-private-cluster/versions.tf index a342884f69..be62c69f20 100644 --- a/modules/beta-private-cluster/versions.tf +++ b/modules/beta-private-cluster/versions.tf @@ -21,11 +21,11 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 5.40.0, < 7" + version = ">= 5.44.2, !=6.0.0, !=6.0.1, !=6.1.0, !=6.2.0, !=6.3.0, !=6.4.0, !=6.5.0, !=6.6.0, < 7" } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.40.0, < 7" + version = ">= 5.44.2, !=6.0.0, !=6.0.1, !=6.1.0, !=6.2.0, !=6.3.0, !=6.4.0, !=6.5.0, !=6.6.0, < 7" } kubernetes = { source = "hashicorp/kubernetes" @@ -37,6 +37,6 @@ terraform { } } provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:beta-private-cluster/v33.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:beta-private-cluster/v34.0.0" } } diff --git a/modules/beta-public-cluster-update-variant/README.md b/modules/beta-public-cluster-update-variant/README.md index 1e52d6f676..70a3479b96 100644 --- a/modules/beta-public-cluster-update-variant/README.md +++ b/modules/beta-public-cluster-update-variant/README.md @@ -224,6 +224,7 @@ Then perform the following commands on the root folder: | http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no | | identity\_namespace | The workload pool to attach all Kubernetes service accounts to. (Default value of `enabled` automatically sets project-based pool `[project_id].svc.id.goog`) | `string` | `"enabled"` | no | | initial\_node\_count | The number of nodes to create in this cluster's default node pool. | `number` | `0` | no | +| insecure\_kubelet\_readonly\_port\_enabled | Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`. | `bool` | `null` | no | | ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | `bool` | `false` | no | | ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | `string` | `"60s"` | no | | ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes | @@ -363,6 +364,7 @@ The node_pools variable takes the following parameters: | gpu_partition_size | Size of partitions to create on the GPU | null | Optional | | image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS_CONTAINERD | Optional | | initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource. Defaults to the value of min_count | " " | Optional | +| insecure_kubelet_readonly_port_enabled | (boolean) Whether or not to enable the insecure Kubelet readonly port. | null | Optional | | key | The key required for the taint | | Required | | logging_variant | The type of logging agent that is deployed by default for newly created node pools in the cluster. Valid values include DEFAULT and MAX_THROUGHPUT. | DEFAULT | Optional | | local_ssd_count | The amount of local SSD disks that will be attached to each cluster node and may be used as a `hostpath` volume or a `local` PersistentVolume. | 0 | Optional | diff --git a/modules/beta-public-cluster-update-variant/cluster.tf b/modules/beta-public-cluster-update-variant/cluster.tf index 3d30888ad6..708a350f4d 100644 --- a/modules/beta-public-cluster-update-variant/cluster.tf +++ b/modules/beta-public-cluster-update-variant/cluster.tf @@ -225,10 +225,10 @@ resource "google_container_cluster" "primary" { } dynamic "node_pool_auto_config" { - for_each = var.cluster_autoscaling.enabled && length(var.network_tags) > 0 ? [1] : [] + for_each = var.cluster_autoscaling.enabled && (length(var.network_tags) > 0 || var.add_cluster_firewall_rules) ? [1] : [] content { network_tags { - tags = var.network_tags + tags = var.add_cluster_firewall_rules ? (concat(var.network_tags, [local.cluster_network_tag])) : var.network_tags } } } @@ -453,6 +453,21 @@ resource "google_container_cluster" "primary" { } } + dynamic "kubelet_config" { + for_each = length(setintersection( + keys(var.node_pools[0]), + ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "insecure_kubelet_readonly_port_enabled", "pod_pids_limit"] + )) != 0 || var.insecure_kubelet_readonly_port_enabled != null ? [1] : [] + + content { + cpu_manager_policy = lookup(var.node_pools[0], "cpu_manager_policy", "static") + cpu_cfs_quota = lookup(var.node_pools[0], "cpu_cfs_quota", null) + cpu_cfs_quota_period = lookup(var.node_pools[0], "cpu_cfs_quota_period", null) + insecure_kubelet_readonly_port_enabled = lookup(var.node_pools[0], "insecure_kubelet_readonly_port_enabled", var.insecure_kubelet_readonly_port_enabled) != null ? upper(tostring(lookup(var.node_pools[0], "insecure_kubelet_readonly_port_enabled", var.insecure_kubelet_readonly_port_enabled))) : null + pod_pids_limit = lookup(var.node_pools[0], "pod_pids_limit", null) + } + } + service_account = lookup(var.node_pools[0], "service_account", local.service_account) tags = concat( @@ -560,6 +575,7 @@ resource "google_container_cluster" "primary" { gcfs_config { enabled = var.enable_gcfs } + insecure_kubelet_readonly_port_enabled = var.insecure_kubelet_readonly_port_enabled != null ? upper(tostring(var.insecure_kubelet_readonly_port_enabled)) : null } } @@ -581,6 +597,7 @@ locals { "enable_secure_boot", "enable_integrity_monitoring", "local_ssd_count", + "local_ssd_ephemeral_count", "machine_type", "placement_policy", "max_pods_per_node", @@ -598,6 +615,7 @@ locals { "reservation_affinity_key", "reservation_affinity_values", "enable_confidential_nodes", + "secondary_boot_disk", ] } @@ -608,7 +626,7 @@ locals { resource "random_id" "name" { for_each = merge(local.node_pools, local.windows_node_pools) byte_length = 2 - prefix = format("%s-", lookup(each.value, "name")) + prefix = "${each.key}-" keepers = merge( zipmap( local.force_node_pool_recreation_resources, @@ -892,14 +910,15 @@ resource "google_container_node_pool" "pools" { dynamic "kubelet_config" { for_each = length(setintersection( keys(each.value), - ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "pod_pids_limit"] + ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "insecure_kubelet_readonly_port_enabled", "pod_pids_limit"] )) != 0 ? [1] : [] content { - cpu_manager_policy = lookup(each.value, "cpu_manager_policy", "static") - cpu_cfs_quota = lookup(each.value, "cpu_cfs_quota", null) - cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null) - pod_pids_limit = lookup(each.value, "pod_pids_limit", null) + cpu_manager_policy = lookup(each.value, "cpu_manager_policy", "static") + cpu_cfs_quota = lookup(each.value, "cpu_cfs_quota", null) + cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null) + insecure_kubelet_readonly_port_enabled = lookup(each.value, "insecure_kubelet_readonly_port_enabled", null) != null ? upper(tostring(each.value.insecure_kubelet_readonly_port_enabled)) : null + pod_pids_limit = lookup(each.value, "pod_pids_limit", null) } } @@ -934,7 +953,7 @@ resource "google_container_node_pool" "pools" { } dynamic "confidential_nodes" { - for_each = lookup(each.value, "enable_confidential_nodes", null) != null ? [each.value.confidential_nodes] : [] + for_each = lookup(each.value, "enable_confidential_nodes", null) != null ? [each.value.enable_confidential_nodes] : [] content { enabled = confidential_nodes.value } @@ -1196,14 +1215,15 @@ resource "google_container_node_pool" "windows_pools" { dynamic "kubelet_config" { for_each = length(setintersection( keys(each.value), - ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "pod_pids_limit"] + ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "insecure_kubelet_readonly_port_enabled", "pod_pids_limit"] )) != 0 ? [1] : [] content { - cpu_manager_policy = lookup(each.value, "cpu_manager_policy", "static") - cpu_cfs_quota = lookup(each.value, "cpu_cfs_quota", null) - cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null) - pod_pids_limit = lookup(each.value, "pod_pids_limit", null) + cpu_manager_policy = lookup(each.value, "cpu_manager_policy", "static") + cpu_cfs_quota = lookup(each.value, "cpu_cfs_quota", null) + cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null) + insecure_kubelet_readonly_port_enabled = lookup(each.value, "insecure_kubelet_readonly_port_enabled", null) != null ? upper(tostring(each.value.insecure_kubelet_readonly_port_enabled)) : null + pod_pids_limit = lookup(each.value, "pod_pids_limit", null) } } @@ -1223,7 +1243,7 @@ resource "google_container_node_pool" "windows_pools" { } dynamic "confidential_nodes" { - for_each = lookup(each.value, "enable_confidential_nodes", null) != null ? [each.value.confidential_nodes] : [] + for_each = lookup(each.value, "enable_confidential_nodes", null) != null ? [each.value.enable_confidential_nodes] : [] content { enabled = confidential_nodes.value } diff --git a/modules/beta-public-cluster-update-variant/dns.tf b/modules/beta-public-cluster-update-variant/dns.tf index bf0d05b723..c120c2be92 100644 --- a/modules/beta-public-cluster-update-variant/dns.tf +++ b/modules/beta-public-cluster-update-variant/dns.tf @@ -20,7 +20,7 @@ Manage kube-dns configmaps *****************************************/ -resource "kubernetes_config_map_v1_data" "kube-dns" { +resource "kubernetes_config_map_v1_data" "kube_dns" { count = local.custom_kube_dns_config && !local.upstream_nameservers_config ? 1 : 0 metadata { @@ -42,7 +42,7 @@ EOF ] } -resource "kubernetes_config_map_v1_data" "kube-dns-upstream-namservers" { +resource "kubernetes_config_map_v1_data" "kube_dns_upstream_nameservers" { count = !local.custom_kube_dns_config && local.upstream_nameservers_config ? 1 : 0 metadata { @@ -64,7 +64,7 @@ EOF ] } -resource "kubernetes_config_map_v1_data" "kube-dns-upstream-nameservers-and-stub-domains" { +resource "kubernetes_config_map_v1_data" "kube_dns_upstream_nameservers_and_stub_domains" { count = local.custom_kube_dns_config && local.upstream_nameservers_config ? 1 : 0 metadata { diff --git a/modules/beta-public-cluster-update-variant/masq.tf b/modules/beta-public-cluster-update-variant/masq.tf index b356aee251..cbb862d720 100644 --- a/modules/beta-public-cluster-update-variant/masq.tf +++ b/modules/beta-public-cluster-update-variant/masq.tf @@ -19,7 +19,7 @@ /****************************************** Create ip-masq-agent confimap *****************************************/ -resource "kubernetes_config_map" "ip-masq-agent" { +resource "kubernetes_config_map" "ip_masq_agent" { count = var.configure_ip_masq ? 1 : 0 metadata { diff --git a/modules/beta-public-cluster-update-variant/moved.tf b/modules/beta-public-cluster-update-variant/moved.tf new file mode 100644 index 0000000000..a745d16489 --- /dev/null +++ b/modules/beta-public-cluster-update-variant/moved.tf @@ -0,0 +1,62 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# Updates for kebab to snake case, to match best practices and Google style. +moved { + from = kubernetes_config_map_v1_data.kube-dns + to = kubernetes_config_map_v1_data.kube_dns +} + +# Typo fix and snake case at the same time +moved { + from = kubernetes_config_map_v1_data.kube-dns-upstream-namservers + to = kubernetes_config_map_v1_data.kube_dns_upstream_nameservers +} + +moved { + from = kubernetes_config_map_v1_data.kube-dns-upstream-nameservers-and-stub-domains + to = kubernetes_config_map_v1_data.kube_dns_upstream_nameservers_and_stub_domains +} + +moved { + from = kubernetes_config_map.ip-masq-agent + to = kubernetes_config_map.ip_masq_agent +} + +moved { + from = google_project_iam_member.cluster_service_account-nodeService_account + to = google_project_iam_member.cluster_service_account_node_service_account +} + +moved { + from = google_project_iam_member.cluster_service_account-metric_writer + to = google_project_iam_member.cluster_service_account_metric_writer +} + +moved { + from = google_project_iam_member.cluster_service_account-resourceMetadata-writer + to = google_project_iam_member.cluster_service_account_resource_metadata_writer +} + +moved { + from = google_project_iam_member.cluster_service_account-gcr + to = google_project_iam_member.cluster_service_account_gcr +} + +moved { + from = google_project_iam_member.cluster_service_account-artifact-registry + to = google_project_iam_member.cluster_service_account_artifact_registry +} diff --git a/modules/beta-public-cluster-update-variant/sa.tf b/modules/beta-public-cluster-update-variant/sa.tf index 34106c5e62..8082a60312 100644 --- a/modules/beta-public-cluster-update-variant/sa.tf +++ b/modules/beta-public-cluster-update-variant/sa.tf @@ -46,35 +46,35 @@ resource "google_service_account" "cluster_service_account" { display_name = "Terraform-managed service account for cluster ${var.name}" } -resource "google_project_iam_member" "cluster_service_account-nodeService_account" { +resource "google_project_iam_member" "cluster_service_account_node_service_account" { count = var.create_service_account ? 1 : 0 project = google_service_account.cluster_service_account[0].project role = "roles/container.defaultNodeServiceAccount" member = google_service_account.cluster_service_account[0].member } -resource "google_project_iam_member" "cluster_service_account-metric_writer" { +resource "google_project_iam_member" "cluster_service_account_metric_writer" { count = var.create_service_account ? 1 : 0 project = google_service_account.cluster_service_account[0].project role = "roles/monitoring.metricWriter" member = google_service_account.cluster_service_account[0].member } -resource "google_project_iam_member" "cluster_service_account-resourceMetadata-writer" { +resource "google_project_iam_member" "cluster_service_account_resource_metadata_writer" { count = var.create_service_account ? 1 : 0 project = google_service_account.cluster_service_account[0].project role = "roles/stackdriver.resourceMetadata.writer" member = google_service_account.cluster_service_account[0].member } -resource "google_project_iam_member" "cluster_service_account-gcr" { +resource "google_project_iam_member" "cluster_service_account_gcr" { for_each = var.create_service_account && var.grant_registry_access ? toset(local.registry_projects_list) : [] project = each.key role = "roles/storage.objectViewer" member = "serviceAccount:${google_service_account.cluster_service_account[0].email}" } -resource "google_project_iam_member" "cluster_service_account-artifact-registry" { +resource "google_project_iam_member" "cluster_service_account_artifact_registry" { for_each = var.create_service_account && var.grant_registry_access ? toset(local.registry_projects_list) : [] project = each.key role = "roles/artifactregistry.reader" diff --git a/modules/beta-public-cluster-update-variant/variables.tf b/modules/beta-public-cluster-update-variant/variables.tf index e926a9568c..499075e902 100644 --- a/modules/beta-public-cluster-update-variant/variables.tf +++ b/modules/beta-public-cluster-update-variant/variables.tf @@ -108,6 +108,12 @@ variable "service_external_ips" { default = false } +variable "insecure_kubelet_readonly_port_enabled" { + type = bool + description = "Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`." + default = null +} + variable "datapath_provider" { type = string description = "The desired datapath provider for this cluster. By default, `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation. `ADVANCED_DATAPATH` enables Dataplane-V2 feature." diff --git a/modules/beta-public-cluster-update-variant/versions.tf b/modules/beta-public-cluster-update-variant/versions.tf index 24df34d950..b22acaf61e 100644 --- a/modules/beta-public-cluster-update-variant/versions.tf +++ b/modules/beta-public-cluster-update-variant/versions.tf @@ -21,11 +21,11 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 5.40.0, < 7" + version = ">= 5.44.2, !=6.0.0, !=6.0.1, !=6.1.0, !=6.2.0, !=6.3.0, !=6.4.0, !=6.5.0, !=6.6.0, < 7" } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.40.0, < 7" + version = ">= 5.44.2, !=6.0.0, !=6.0.1, !=6.1.0, !=6.2.0, !=6.3.0, !=6.4.0, !=6.5.0, !=6.6.0, < 7" } kubernetes = { source = "hashicorp/kubernetes" @@ -37,6 +37,6 @@ terraform { } } provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:beta-public-cluster-update-variant/v33.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:beta-public-cluster-update-variant/v34.0.0" } } diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md index b0700b78ea..a4af452331 100644 --- a/modules/beta-public-cluster/README.md +++ b/modules/beta-public-cluster/README.md @@ -202,6 +202,7 @@ Then perform the following commands on the root folder: | http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no | | identity\_namespace | The workload pool to attach all Kubernetes service accounts to. (Default value of `enabled` automatically sets project-based pool `[project_id].svc.id.goog`) | `string` | `"enabled"` | no | | initial\_node\_count | The number of nodes to create in this cluster's default node pool. | `number` | `0` | no | +| insecure\_kubelet\_readonly\_port\_enabled | Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`. | `bool` | `null` | no | | ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | `bool` | `false` | no | | ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | `string` | `"60s"` | no | | ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes | @@ -341,6 +342,7 @@ The node_pools variable takes the following parameters: | gpu_partition_size | Size of partitions to create on the GPU | null | Optional | | image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS_CONTAINERD | Optional | | initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource. Defaults to the value of min_count | " " | Optional | +| insecure_kubelet_readonly_port_enabled | (boolean) Whether or not to enable the insecure Kubelet readonly port. | null | Optional | | key | The key required for the taint | | Required | | logging_variant | The type of logging agent that is deployed by default for newly created node pools in the cluster. Valid values include DEFAULT and MAX_THROUGHPUT. | DEFAULT | Optional | | local_ssd_count | The amount of local SSD disks that will be attached to each cluster node and may be used as a `hostpath` volume or a `local` PersistentVolume. | 0 | Optional | diff --git a/modules/beta-public-cluster/cluster.tf b/modules/beta-public-cluster/cluster.tf index 3112ac56e8..c12791010c 100644 --- a/modules/beta-public-cluster/cluster.tf +++ b/modules/beta-public-cluster/cluster.tf @@ -225,10 +225,10 @@ resource "google_container_cluster" "primary" { } dynamic "node_pool_auto_config" { - for_each = var.cluster_autoscaling.enabled && length(var.network_tags) > 0 ? [1] : [] + for_each = var.cluster_autoscaling.enabled && (length(var.network_tags) > 0 || var.add_cluster_firewall_rules) ? [1] : [] content { network_tags { - tags = var.network_tags + tags = var.add_cluster_firewall_rules ? (concat(var.network_tags, [local.cluster_network_tag])) : var.network_tags } } } @@ -453,6 +453,21 @@ resource "google_container_cluster" "primary" { } } + dynamic "kubelet_config" { + for_each = length(setintersection( + keys(var.node_pools[0]), + ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "insecure_kubelet_readonly_port_enabled", "pod_pids_limit"] + )) != 0 || var.insecure_kubelet_readonly_port_enabled != null ? [1] : [] + + content { + cpu_manager_policy = lookup(var.node_pools[0], "cpu_manager_policy", "static") + cpu_cfs_quota = lookup(var.node_pools[0], "cpu_cfs_quota", null) + cpu_cfs_quota_period = lookup(var.node_pools[0], "cpu_cfs_quota_period", null) + insecure_kubelet_readonly_port_enabled = lookup(var.node_pools[0], "insecure_kubelet_readonly_port_enabled", var.insecure_kubelet_readonly_port_enabled) != null ? upper(tostring(lookup(var.node_pools[0], "insecure_kubelet_readonly_port_enabled", var.insecure_kubelet_readonly_port_enabled))) : null + pod_pids_limit = lookup(var.node_pools[0], "pod_pids_limit", null) + } + } + service_account = lookup(var.node_pools[0], "service_account", local.service_account) tags = concat( @@ -560,6 +575,7 @@ resource "google_container_cluster" "primary" { gcfs_config { enabled = var.enable_gcfs } + insecure_kubelet_readonly_port_enabled = var.insecure_kubelet_readonly_port_enabled != null ? upper(tostring(var.insecure_kubelet_readonly_port_enabled)) : null } } @@ -809,14 +825,15 @@ resource "google_container_node_pool" "pools" { dynamic "kubelet_config" { for_each = length(setintersection( keys(each.value), - ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "pod_pids_limit"] + ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "insecure_kubelet_readonly_port_enabled", "pod_pids_limit"] )) != 0 ? [1] : [] content { - cpu_manager_policy = lookup(each.value, "cpu_manager_policy", "static") - cpu_cfs_quota = lookup(each.value, "cpu_cfs_quota", null) - cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null) - pod_pids_limit = lookup(each.value, "pod_pids_limit", null) + cpu_manager_policy = lookup(each.value, "cpu_manager_policy", "static") + cpu_cfs_quota = lookup(each.value, "cpu_cfs_quota", null) + cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null) + insecure_kubelet_readonly_port_enabled = lookup(each.value, "insecure_kubelet_readonly_port_enabled", null) != null ? upper(tostring(each.value.insecure_kubelet_readonly_port_enabled)) : null + pod_pids_limit = lookup(each.value, "pod_pids_limit", null) } } @@ -851,7 +868,7 @@ resource "google_container_node_pool" "pools" { } dynamic "confidential_nodes" { - for_each = lookup(each.value, "enable_confidential_nodes", null) != null ? [each.value.confidential_nodes] : [] + for_each = lookup(each.value, "enable_confidential_nodes", null) != null ? [each.value.enable_confidential_nodes] : [] content { enabled = confidential_nodes.value } @@ -1112,14 +1129,15 @@ resource "google_container_node_pool" "windows_pools" { dynamic "kubelet_config" { for_each = length(setintersection( keys(each.value), - ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "pod_pids_limit"] + ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "insecure_kubelet_readonly_port_enabled", "pod_pids_limit"] )) != 0 ? [1] : [] content { - cpu_manager_policy = lookup(each.value, "cpu_manager_policy", "static") - cpu_cfs_quota = lookup(each.value, "cpu_cfs_quota", null) - cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null) - pod_pids_limit = lookup(each.value, "pod_pids_limit", null) + cpu_manager_policy = lookup(each.value, "cpu_manager_policy", "static") + cpu_cfs_quota = lookup(each.value, "cpu_cfs_quota", null) + cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null) + insecure_kubelet_readonly_port_enabled = lookup(each.value, "insecure_kubelet_readonly_port_enabled", null) != null ? upper(tostring(each.value.insecure_kubelet_readonly_port_enabled)) : null + pod_pids_limit = lookup(each.value, "pod_pids_limit", null) } } @@ -1139,7 +1157,7 @@ resource "google_container_node_pool" "windows_pools" { } dynamic "confidential_nodes" { - for_each = lookup(each.value, "enable_confidential_nodes", null) != null ? [each.value.confidential_nodes] : [] + for_each = lookup(each.value, "enable_confidential_nodes", null) != null ? [each.value.enable_confidential_nodes] : [] content { enabled = confidential_nodes.value } diff --git a/modules/beta-public-cluster/dns.tf b/modules/beta-public-cluster/dns.tf index bf0d05b723..c120c2be92 100644 --- a/modules/beta-public-cluster/dns.tf +++ b/modules/beta-public-cluster/dns.tf @@ -20,7 +20,7 @@ Manage kube-dns configmaps *****************************************/ -resource "kubernetes_config_map_v1_data" "kube-dns" { +resource "kubernetes_config_map_v1_data" "kube_dns" { count = local.custom_kube_dns_config && !local.upstream_nameservers_config ? 1 : 0 metadata { @@ -42,7 +42,7 @@ EOF ] } -resource "kubernetes_config_map_v1_data" "kube-dns-upstream-namservers" { +resource "kubernetes_config_map_v1_data" "kube_dns_upstream_nameservers" { count = !local.custom_kube_dns_config && local.upstream_nameservers_config ? 1 : 0 metadata { @@ -64,7 +64,7 @@ EOF ] } -resource "kubernetes_config_map_v1_data" "kube-dns-upstream-nameservers-and-stub-domains" { +resource "kubernetes_config_map_v1_data" "kube_dns_upstream_nameservers_and_stub_domains" { count = local.custom_kube_dns_config && local.upstream_nameservers_config ? 1 : 0 metadata { diff --git a/modules/beta-public-cluster/masq.tf b/modules/beta-public-cluster/masq.tf index b356aee251..cbb862d720 100644 --- a/modules/beta-public-cluster/masq.tf +++ b/modules/beta-public-cluster/masq.tf @@ -19,7 +19,7 @@ /****************************************** Create ip-masq-agent confimap *****************************************/ -resource "kubernetes_config_map" "ip-masq-agent" { +resource "kubernetes_config_map" "ip_masq_agent" { count = var.configure_ip_masq ? 1 : 0 metadata { diff --git a/modules/beta-public-cluster/moved.tf b/modules/beta-public-cluster/moved.tf new file mode 100644 index 0000000000..a745d16489 --- /dev/null +++ b/modules/beta-public-cluster/moved.tf @@ -0,0 +1,62 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# Updates for kebab to snake case, to match best practices and Google style. +moved { + from = kubernetes_config_map_v1_data.kube-dns + to = kubernetes_config_map_v1_data.kube_dns +} + +# Typo fix and snake case at the same time +moved { + from = kubernetes_config_map_v1_data.kube-dns-upstream-namservers + to = kubernetes_config_map_v1_data.kube_dns_upstream_nameservers +} + +moved { + from = kubernetes_config_map_v1_data.kube-dns-upstream-nameservers-and-stub-domains + to = kubernetes_config_map_v1_data.kube_dns_upstream_nameservers_and_stub_domains +} + +moved { + from = kubernetes_config_map.ip-masq-agent + to = kubernetes_config_map.ip_masq_agent +} + +moved { + from = google_project_iam_member.cluster_service_account-nodeService_account + to = google_project_iam_member.cluster_service_account_node_service_account +} + +moved { + from = google_project_iam_member.cluster_service_account-metric_writer + to = google_project_iam_member.cluster_service_account_metric_writer +} + +moved { + from = google_project_iam_member.cluster_service_account-resourceMetadata-writer + to = google_project_iam_member.cluster_service_account_resource_metadata_writer +} + +moved { + from = google_project_iam_member.cluster_service_account-gcr + to = google_project_iam_member.cluster_service_account_gcr +} + +moved { + from = google_project_iam_member.cluster_service_account-artifact-registry + to = google_project_iam_member.cluster_service_account_artifact_registry +} diff --git a/modules/beta-public-cluster/sa.tf b/modules/beta-public-cluster/sa.tf index 34106c5e62..8082a60312 100644 --- a/modules/beta-public-cluster/sa.tf +++ b/modules/beta-public-cluster/sa.tf @@ -46,35 +46,35 @@ resource "google_service_account" "cluster_service_account" { display_name = "Terraform-managed service account for cluster ${var.name}" } -resource "google_project_iam_member" "cluster_service_account-nodeService_account" { +resource "google_project_iam_member" "cluster_service_account_node_service_account" { count = var.create_service_account ? 1 : 0 project = google_service_account.cluster_service_account[0].project role = "roles/container.defaultNodeServiceAccount" member = google_service_account.cluster_service_account[0].member } -resource "google_project_iam_member" "cluster_service_account-metric_writer" { +resource "google_project_iam_member" "cluster_service_account_metric_writer" { count = var.create_service_account ? 1 : 0 project = google_service_account.cluster_service_account[0].project role = "roles/monitoring.metricWriter" member = google_service_account.cluster_service_account[0].member } -resource "google_project_iam_member" "cluster_service_account-resourceMetadata-writer" { +resource "google_project_iam_member" "cluster_service_account_resource_metadata_writer" { count = var.create_service_account ? 1 : 0 project = google_service_account.cluster_service_account[0].project role = "roles/stackdriver.resourceMetadata.writer" member = google_service_account.cluster_service_account[0].member } -resource "google_project_iam_member" "cluster_service_account-gcr" { +resource "google_project_iam_member" "cluster_service_account_gcr" { for_each = var.create_service_account && var.grant_registry_access ? toset(local.registry_projects_list) : [] project = each.key role = "roles/storage.objectViewer" member = "serviceAccount:${google_service_account.cluster_service_account[0].email}" } -resource "google_project_iam_member" "cluster_service_account-artifact-registry" { +resource "google_project_iam_member" "cluster_service_account_artifact_registry" { for_each = var.create_service_account && var.grant_registry_access ? toset(local.registry_projects_list) : [] project = each.key role = "roles/artifactregistry.reader" diff --git a/modules/beta-public-cluster/variables.tf b/modules/beta-public-cluster/variables.tf index e926a9568c..499075e902 100644 --- a/modules/beta-public-cluster/variables.tf +++ b/modules/beta-public-cluster/variables.tf @@ -108,6 +108,12 @@ variable "service_external_ips" { default = false } +variable "insecure_kubelet_readonly_port_enabled" { + type = bool + description = "Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`." + default = null +} + variable "datapath_provider" { type = string description = "The desired datapath provider for this cluster. By default, `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation. `ADVANCED_DATAPATH` enables Dataplane-V2 feature." diff --git a/modules/beta-public-cluster/versions.tf b/modules/beta-public-cluster/versions.tf index d62645f541..583e4e538b 100644 --- a/modules/beta-public-cluster/versions.tf +++ b/modules/beta-public-cluster/versions.tf @@ -21,11 +21,11 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 5.40.0, < 7" + version = ">= 5.44.2, !=6.0.0, !=6.0.1, !=6.1.0, !=6.2.0, !=6.3.0, !=6.4.0, !=6.5.0, !=6.6.0, < 7" } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.40.0, < 7" + version = ">= 5.44.2, !=6.0.0, !=6.0.1, !=6.1.0, !=6.2.0, !=6.3.0, !=6.4.0, !=6.5.0, !=6.6.0, < 7" } kubernetes = { source = "hashicorp/kubernetes" @@ -37,6 +37,6 @@ terraform { } } provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:beta-public-cluster/v33.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:beta-public-cluster/v34.0.0" } } diff --git a/modules/binary-authorization/main.tf b/modules/binary-authorization/main.tf index 901aecc5b4..11b7c5fa8e 100644 --- a/modules/binary-authorization/main.tf +++ b/modules/binary-authorization/main.tf @@ -25,7 +25,7 @@ locals { module "project-services" { source = "terraform-google-modules/project-factory/google//modules/project_services" - version = "~> 16.0" + version = "~> 17.0" project_id = var.project_id activate_apis = local.required_enabled_apis diff --git a/modules/binary-authorization/versions.tf b/modules/binary-authorization/versions.tf index 6c11ea4b1b..8f1d801565 100644 --- a/modules/binary-authorization/versions.tf +++ b/modules/binary-authorization/versions.tf @@ -28,6 +28,6 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:binary-authorization/v33.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:binary-authorization/v34.0.0" } } diff --git a/modules/fleet-app-operator-permissions/versions.tf b/modules/fleet-app-operator-permissions/versions.tf index 226b9aa4f8..92243370a9 100644 --- a/modules/fleet-app-operator-permissions/versions.tf +++ b/modules/fleet-app-operator-permissions/versions.tf @@ -33,7 +33,7 @@ terraform { } provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:fleet-app-operator-permissions/v33.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:fleet-app-operator-permissions/v34.0.0" } } diff --git a/modules/fleet-membership/versions.tf b/modules/fleet-membership/versions.tf index 3c35a4ec95..9dfff21d28 100644 --- a/modules/fleet-membership/versions.tf +++ b/modules/fleet-membership/versions.tf @@ -30,6 +30,6 @@ terraform { } provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:hub/v33.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:hub/v34.0.0" } } diff --git a/modules/hub-legacy/versions.tf b/modules/hub-legacy/versions.tf index 76f55f1109..95e3c8a6a5 100644 --- a/modules/hub-legacy/versions.tf +++ b/modules/hub-legacy/versions.tf @@ -28,6 +28,6 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:hub/v33.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:hub/v34.0.0" } } diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md index 3fb72c548c..cb364f11f2 100644 --- a/modules/private-cluster-update-variant/README.md +++ b/modules/private-cluster-update-variant/README.md @@ -221,6 +221,7 @@ Then perform the following commands on the root folder: | http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no | | identity\_namespace | The workload pool to attach all Kubernetes service accounts to. (Default value of `enabled` automatically sets project-based pool `[project_id].svc.id.goog`) | `string` | `"enabled"` | no | | initial\_node\_count | The number of nodes to create in this cluster's default node pool. | `number` | `0` | no | +| insecure\_kubelet\_readonly\_port\_enabled | Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`. | `bool` | `null` | no | | ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | `bool` | `false` | no | | ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | `string` | `"60s"` | no | | ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes | @@ -355,6 +356,7 @@ The node_pools variable takes the following parameters: | gpu_partition_size | Size of partitions to create on the GPU | null | Optional | | image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS_CONTAINERD | Optional | | initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource. Defaults to the value of min_count | " " | Optional | +| insecure_kubelet_readonly_port_enabled | (boolean) Whether or not to enable the insecure Kubelet readonly port. | null | Optional | | key | The key required for the taint | | Required | | logging_variant | The type of logging agent that is deployed by default for newly created node pools in the cluster. Valid values include DEFAULT and MAX_THROUGHPUT. | DEFAULT | Optional | | local_ssd_count | The amount of local SSD disks that will be attached to each cluster node and may be used as a `hostpath` volume or a `local` PersistentVolume. | 0 | Optional | diff --git a/modules/private-cluster-update-variant/cluster.tf b/modules/private-cluster-update-variant/cluster.tf index 5dc2ac967f..d783db4227 100644 --- a/modules/private-cluster-update-variant/cluster.tf +++ b/modules/private-cluster-update-variant/cluster.tf @@ -204,10 +204,10 @@ resource "google_container_cluster" "primary" { } dynamic "node_pool_auto_config" { - for_each = var.cluster_autoscaling.enabled && length(var.network_tags) > 0 ? [1] : [] + for_each = var.cluster_autoscaling.enabled && (length(var.network_tags) > 0 || var.add_cluster_firewall_rules) ? [1] : [] content { network_tags { - tags = var.network_tags + tags = var.add_cluster_firewall_rules ? (concat(var.network_tags, [local.cluster_network_tag])) : var.network_tags } } } @@ -407,6 +407,21 @@ resource "google_container_cluster" "primary" { } } + dynamic "kubelet_config" { + for_each = length(setintersection( + keys(var.node_pools[0]), + ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "insecure_kubelet_readonly_port_enabled", "pod_pids_limit"] + )) != 0 || var.insecure_kubelet_readonly_port_enabled != null ? [1] : [] + + content { + cpu_manager_policy = lookup(var.node_pools[0], "cpu_manager_policy", "static") + cpu_cfs_quota = lookup(var.node_pools[0], "cpu_cfs_quota", null) + cpu_cfs_quota_period = lookup(var.node_pools[0], "cpu_cfs_quota_period", null) + insecure_kubelet_readonly_port_enabled = lookup(var.node_pools[0], "insecure_kubelet_readonly_port_enabled", var.insecure_kubelet_readonly_port_enabled) != null ? upper(tostring(lookup(var.node_pools[0], "insecure_kubelet_readonly_port_enabled", var.insecure_kubelet_readonly_port_enabled))) : null + pod_pids_limit = lookup(var.node_pools[0], "pod_pids_limit", null) + } + } + service_account = lookup(var.node_pools[0], "service_account", local.service_account) tags = concat( @@ -521,6 +536,13 @@ resource "google_container_cluster" "primary" { } } } + + node_pool_defaults { + node_config_defaults { + insecure_kubelet_readonly_port_enabled = var.insecure_kubelet_readonly_port_enabled != null ? upper(tostring(var.insecure_kubelet_readonly_port_enabled)) : null + } + } + } /****************************************** Create Container Cluster node pools @@ -555,6 +577,7 @@ locals { "reservation_affinity_key", "reservation_affinity_values", "enable_confidential_nodes", + "secondary_boot_disk", ] } @@ -565,7 +588,7 @@ locals { resource "random_id" "name" { for_each = merge(local.node_pools, local.windows_node_pools) byte_length = 2 - prefix = format("%s-", lookup(each.value, "name")) + prefix = "${each.key}-" keepers = merge( zipmap( local.force_node_pool_recreation_resources, @@ -843,14 +866,15 @@ resource "google_container_node_pool" "pools" { dynamic "kubelet_config" { for_each = length(setintersection( keys(each.value), - ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "pod_pids_limit"] + ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "insecure_kubelet_readonly_port_enabled", "pod_pids_limit"] )) != 0 ? [1] : [] content { - cpu_manager_policy = lookup(each.value, "cpu_manager_policy", "static") - cpu_cfs_quota = lookup(each.value, "cpu_cfs_quota", null) - cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null) - pod_pids_limit = lookup(each.value, "pod_pids_limit", null) + cpu_manager_policy = lookup(each.value, "cpu_manager_policy", "static") + cpu_cfs_quota = lookup(each.value, "cpu_cfs_quota", null) + cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null) + insecure_kubelet_readonly_port_enabled = lookup(each.value, "insecure_kubelet_readonly_port_enabled", null) != null ? upper(tostring(each.value.insecure_kubelet_readonly_port_enabled)) : null + pod_pids_limit = lookup(each.value, "pod_pids_limit", null) } } @@ -878,7 +902,7 @@ resource "google_container_node_pool" "pools" { } dynamic "confidential_nodes" { - for_each = lookup(each.value, "enable_confidential_nodes", null) != null ? [each.value.confidential_nodes] : [] + for_each = lookup(each.value, "enable_confidential_nodes", null) != null ? [each.value.enable_confidential_nodes] : [] content { enabled = confidential_nodes.value } @@ -1134,14 +1158,15 @@ resource "google_container_node_pool" "windows_pools" { dynamic "kubelet_config" { for_each = length(setintersection( keys(each.value), - ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "pod_pids_limit"] + ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "insecure_kubelet_readonly_port_enabled", "pod_pids_limit"] )) != 0 ? [1] : [] content { - cpu_manager_policy = lookup(each.value, "cpu_manager_policy", "static") - cpu_cfs_quota = lookup(each.value, "cpu_cfs_quota", null) - cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null) - pod_pids_limit = lookup(each.value, "pod_pids_limit", null) + cpu_manager_policy = lookup(each.value, "cpu_manager_policy", "static") + cpu_cfs_quota = lookup(each.value, "cpu_cfs_quota", null) + cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null) + insecure_kubelet_readonly_port_enabled = lookup(each.value, "insecure_kubelet_readonly_port_enabled", null) != null ? upper(tostring(each.value.insecure_kubelet_readonly_port_enabled)) : null + pod_pids_limit = lookup(each.value, "pod_pids_limit", null) } } @@ -1154,7 +1179,7 @@ resource "google_container_node_pool" "windows_pools" { } dynamic "confidential_nodes" { - for_each = lookup(each.value, "enable_confidential_nodes", null) != null ? [each.value.confidential_nodes] : [] + for_each = lookup(each.value, "enable_confidential_nodes", null) != null ? [each.value.enable_confidential_nodes] : [] content { enabled = confidential_nodes.value } diff --git a/modules/private-cluster-update-variant/dns.tf b/modules/private-cluster-update-variant/dns.tf index bf0d05b723..c120c2be92 100644 --- a/modules/private-cluster-update-variant/dns.tf +++ b/modules/private-cluster-update-variant/dns.tf @@ -20,7 +20,7 @@ Manage kube-dns configmaps *****************************************/ -resource "kubernetes_config_map_v1_data" "kube-dns" { +resource "kubernetes_config_map_v1_data" "kube_dns" { count = local.custom_kube_dns_config && !local.upstream_nameservers_config ? 1 : 0 metadata { @@ -42,7 +42,7 @@ EOF ] } -resource "kubernetes_config_map_v1_data" "kube-dns-upstream-namservers" { +resource "kubernetes_config_map_v1_data" "kube_dns_upstream_nameservers" { count = !local.custom_kube_dns_config && local.upstream_nameservers_config ? 1 : 0 metadata { @@ -64,7 +64,7 @@ EOF ] } -resource "kubernetes_config_map_v1_data" "kube-dns-upstream-nameservers-and-stub-domains" { +resource "kubernetes_config_map_v1_data" "kube_dns_upstream_nameservers_and_stub_domains" { count = local.custom_kube_dns_config && local.upstream_nameservers_config ? 1 : 0 metadata { diff --git a/modules/private-cluster-update-variant/masq.tf b/modules/private-cluster-update-variant/masq.tf index b356aee251..cbb862d720 100644 --- a/modules/private-cluster-update-variant/masq.tf +++ b/modules/private-cluster-update-variant/masq.tf @@ -19,7 +19,7 @@ /****************************************** Create ip-masq-agent confimap *****************************************/ -resource "kubernetes_config_map" "ip-masq-agent" { +resource "kubernetes_config_map" "ip_masq_agent" { count = var.configure_ip_masq ? 1 : 0 metadata { diff --git a/modules/private-cluster-update-variant/moved.tf b/modules/private-cluster-update-variant/moved.tf new file mode 100644 index 0000000000..a745d16489 --- /dev/null +++ b/modules/private-cluster-update-variant/moved.tf @@ -0,0 +1,62 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# Updates for kebab to snake case, to match best practices and Google style. +moved { + from = kubernetes_config_map_v1_data.kube-dns + to = kubernetes_config_map_v1_data.kube_dns +} + +# Typo fix and snake case at the same time +moved { + from = kubernetes_config_map_v1_data.kube-dns-upstream-namservers + to = kubernetes_config_map_v1_data.kube_dns_upstream_nameservers +} + +moved { + from = kubernetes_config_map_v1_data.kube-dns-upstream-nameservers-and-stub-domains + to = kubernetes_config_map_v1_data.kube_dns_upstream_nameservers_and_stub_domains +} + +moved { + from = kubernetes_config_map.ip-masq-agent + to = kubernetes_config_map.ip_masq_agent +} + +moved { + from = google_project_iam_member.cluster_service_account-nodeService_account + to = google_project_iam_member.cluster_service_account_node_service_account +} + +moved { + from = google_project_iam_member.cluster_service_account-metric_writer + to = google_project_iam_member.cluster_service_account_metric_writer +} + +moved { + from = google_project_iam_member.cluster_service_account-resourceMetadata-writer + to = google_project_iam_member.cluster_service_account_resource_metadata_writer +} + +moved { + from = google_project_iam_member.cluster_service_account-gcr + to = google_project_iam_member.cluster_service_account_gcr +} + +moved { + from = google_project_iam_member.cluster_service_account-artifact-registry + to = google_project_iam_member.cluster_service_account_artifact_registry +} diff --git a/modules/private-cluster-update-variant/sa.tf b/modules/private-cluster-update-variant/sa.tf index 72d044a4c9..a8923ce1db 100644 --- a/modules/private-cluster-update-variant/sa.tf +++ b/modules/private-cluster-update-variant/sa.tf @@ -46,35 +46,35 @@ resource "google_service_account" "cluster_service_account" { display_name = "Terraform-managed service account for cluster ${var.name}" } -resource "google_project_iam_member" "cluster_service_account-nodeService_account" { +resource "google_project_iam_member" "cluster_service_account_node_service_account" { count = var.create_service_account ? 1 : 0 project = google_service_account.cluster_service_account[0].project role = "roles/container.defaultNodeServiceAccount" member = google_service_account.cluster_service_account[0].member } -resource "google_project_iam_member" "cluster_service_account-metric_writer" { +resource "google_project_iam_member" "cluster_service_account_metric_writer" { count = var.create_service_account ? 1 : 0 project = google_service_account.cluster_service_account[0].project role = "roles/monitoring.metricWriter" member = google_service_account.cluster_service_account[0].member } -resource "google_project_iam_member" "cluster_service_account-resourceMetadata-writer" { +resource "google_project_iam_member" "cluster_service_account_resource_metadata_writer" { count = var.create_service_account ? 1 : 0 project = google_service_account.cluster_service_account[0].project role = "roles/stackdriver.resourceMetadata.writer" member = google_service_account.cluster_service_account[0].member } -resource "google_project_iam_member" "cluster_service_account-gcr" { +resource "google_project_iam_member" "cluster_service_account_gcr" { for_each = var.create_service_account && var.grant_registry_access ? toset(local.registry_projects_list) : [] project = each.key role = "roles/storage.objectViewer" member = "serviceAccount:${google_service_account.cluster_service_account[0].email}" } -resource "google_project_iam_member" "cluster_service_account-artifact-registry" { +resource "google_project_iam_member" "cluster_service_account_artifact_registry" { for_each = var.create_service_account && var.grant_registry_access ? toset(local.registry_projects_list) : [] project = each.key role = "roles/artifactregistry.reader" diff --git a/modules/private-cluster-update-variant/variables.tf b/modules/private-cluster-update-variant/variables.tf index 728b64ad7d..70b2619e6c 100644 --- a/modules/private-cluster-update-variant/variables.tf +++ b/modules/private-cluster-update-variant/variables.tf @@ -108,6 +108,12 @@ variable "service_external_ips" { default = false } +variable "insecure_kubelet_readonly_port_enabled" { + type = bool + description = "Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`." + default = null +} + variable "datapath_provider" { type = string description = "The desired datapath provider for this cluster. By default, `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation. `ADVANCED_DATAPATH` enables Dataplane-V2 feature." diff --git a/modules/private-cluster-update-variant/versions.tf b/modules/private-cluster-update-variant/versions.tf index a91f502a79..f12b7c5070 100644 --- a/modules/private-cluster-update-variant/versions.tf +++ b/modules/private-cluster-update-variant/versions.tf @@ -21,7 +21,7 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 5.40.0, < 7" + version = ">= 5.44.2, !=6.0.0, !=6.0.1, !=6.1.0, !=6.2.0, !=6.3.0, !=6.4.0, !=6.5.0, !=6.6.0, < 7" } kubernetes = { source = "hashicorp/kubernetes" @@ -33,6 +33,6 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:private-cluster-update-variant/v33.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:private-cluster-update-variant/v34.0.0" } } diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md index 6b68e9bff5..96d897cb4b 100644 --- a/modules/private-cluster/README.md +++ b/modules/private-cluster/README.md @@ -199,6 +199,7 @@ Then perform the following commands on the root folder: | http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no | | identity\_namespace | The workload pool to attach all Kubernetes service accounts to. (Default value of `enabled` automatically sets project-based pool `[project_id].svc.id.goog`) | `string` | `"enabled"` | no | | initial\_node\_count | The number of nodes to create in this cluster's default node pool. | `number` | `0` | no | +| insecure\_kubelet\_readonly\_port\_enabled | Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`. | `bool` | `null` | no | | ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | `bool` | `false` | no | | ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | `string` | `"60s"` | no | | ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes | @@ -333,6 +334,7 @@ The node_pools variable takes the following parameters: | gpu_partition_size | Size of partitions to create on the GPU | null | Optional | | image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS_CONTAINERD | Optional | | initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource. Defaults to the value of min_count | " " | Optional | +| insecure_kubelet_readonly_port_enabled | (boolean) Whether or not to enable the insecure Kubelet readonly port. | null | Optional | | key | The key required for the taint | | Required | | logging_variant | The type of logging agent that is deployed by default for newly created node pools in the cluster. Valid values include DEFAULT and MAX_THROUGHPUT. | DEFAULT | Optional | | local_ssd_count | The amount of local SSD disks that will be attached to each cluster node and may be used as a `hostpath` volume or a `local` PersistentVolume. | 0 | Optional | diff --git a/modules/private-cluster/cluster.tf b/modules/private-cluster/cluster.tf index ed33e87b2a..5e892fe978 100644 --- a/modules/private-cluster/cluster.tf +++ b/modules/private-cluster/cluster.tf @@ -204,10 +204,10 @@ resource "google_container_cluster" "primary" { } dynamic "node_pool_auto_config" { - for_each = var.cluster_autoscaling.enabled && length(var.network_tags) > 0 ? [1] : [] + for_each = var.cluster_autoscaling.enabled && (length(var.network_tags) > 0 || var.add_cluster_firewall_rules) ? [1] : [] content { network_tags { - tags = var.network_tags + tags = var.add_cluster_firewall_rules ? (concat(var.network_tags, [local.cluster_network_tag])) : var.network_tags } } } @@ -407,6 +407,21 @@ resource "google_container_cluster" "primary" { } } + dynamic "kubelet_config" { + for_each = length(setintersection( + keys(var.node_pools[0]), + ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "insecure_kubelet_readonly_port_enabled", "pod_pids_limit"] + )) != 0 || var.insecure_kubelet_readonly_port_enabled != null ? [1] : [] + + content { + cpu_manager_policy = lookup(var.node_pools[0], "cpu_manager_policy", "static") + cpu_cfs_quota = lookup(var.node_pools[0], "cpu_cfs_quota", null) + cpu_cfs_quota_period = lookup(var.node_pools[0], "cpu_cfs_quota_period", null) + insecure_kubelet_readonly_port_enabled = lookup(var.node_pools[0], "insecure_kubelet_readonly_port_enabled", var.insecure_kubelet_readonly_port_enabled) != null ? upper(tostring(lookup(var.node_pools[0], "insecure_kubelet_readonly_port_enabled", var.insecure_kubelet_readonly_port_enabled))) : null + pod_pids_limit = lookup(var.node_pools[0], "pod_pids_limit", null) + } + } + service_account = lookup(var.node_pools[0], "service_account", local.service_account) tags = concat( @@ -521,6 +536,13 @@ resource "google_container_cluster" "primary" { } } } + + node_pool_defaults { + node_config_defaults { + insecure_kubelet_readonly_port_enabled = var.insecure_kubelet_readonly_port_enabled != null ? upper(tostring(var.insecure_kubelet_readonly_port_enabled)) : null + } + } + } /****************************************** Create Container Cluster node pools @@ -760,14 +782,15 @@ resource "google_container_node_pool" "pools" { dynamic "kubelet_config" { for_each = length(setintersection( keys(each.value), - ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "pod_pids_limit"] + ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "insecure_kubelet_readonly_port_enabled", "pod_pids_limit"] )) != 0 ? [1] : [] content { - cpu_manager_policy = lookup(each.value, "cpu_manager_policy", "static") - cpu_cfs_quota = lookup(each.value, "cpu_cfs_quota", null) - cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null) - pod_pids_limit = lookup(each.value, "pod_pids_limit", null) + cpu_manager_policy = lookup(each.value, "cpu_manager_policy", "static") + cpu_cfs_quota = lookup(each.value, "cpu_cfs_quota", null) + cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null) + insecure_kubelet_readonly_port_enabled = lookup(each.value, "insecure_kubelet_readonly_port_enabled", null) != null ? upper(tostring(each.value.insecure_kubelet_readonly_port_enabled)) : null + pod_pids_limit = lookup(each.value, "pod_pids_limit", null) } } @@ -795,7 +818,7 @@ resource "google_container_node_pool" "pools" { } dynamic "confidential_nodes" { - for_each = lookup(each.value, "enable_confidential_nodes", null) != null ? [each.value.confidential_nodes] : [] + for_each = lookup(each.value, "enable_confidential_nodes", null) != null ? [each.value.enable_confidential_nodes] : [] content { enabled = confidential_nodes.value } @@ -1050,14 +1073,15 @@ resource "google_container_node_pool" "windows_pools" { dynamic "kubelet_config" { for_each = length(setintersection( keys(each.value), - ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "pod_pids_limit"] + ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "insecure_kubelet_readonly_port_enabled", "pod_pids_limit"] )) != 0 ? [1] : [] content { - cpu_manager_policy = lookup(each.value, "cpu_manager_policy", "static") - cpu_cfs_quota = lookup(each.value, "cpu_cfs_quota", null) - cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null) - pod_pids_limit = lookup(each.value, "pod_pids_limit", null) + cpu_manager_policy = lookup(each.value, "cpu_manager_policy", "static") + cpu_cfs_quota = lookup(each.value, "cpu_cfs_quota", null) + cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null) + insecure_kubelet_readonly_port_enabled = lookup(each.value, "insecure_kubelet_readonly_port_enabled", null) != null ? upper(tostring(each.value.insecure_kubelet_readonly_port_enabled)) : null + pod_pids_limit = lookup(each.value, "pod_pids_limit", null) } } @@ -1070,7 +1094,7 @@ resource "google_container_node_pool" "windows_pools" { } dynamic "confidential_nodes" { - for_each = lookup(each.value, "enable_confidential_nodes", null) != null ? [each.value.confidential_nodes] : [] + for_each = lookup(each.value, "enable_confidential_nodes", null) != null ? [each.value.enable_confidential_nodes] : [] content { enabled = confidential_nodes.value } diff --git a/modules/private-cluster/dns.tf b/modules/private-cluster/dns.tf index bf0d05b723..c120c2be92 100644 --- a/modules/private-cluster/dns.tf +++ b/modules/private-cluster/dns.tf @@ -20,7 +20,7 @@ Manage kube-dns configmaps *****************************************/ -resource "kubernetes_config_map_v1_data" "kube-dns" { +resource "kubernetes_config_map_v1_data" "kube_dns" { count = local.custom_kube_dns_config && !local.upstream_nameservers_config ? 1 : 0 metadata { @@ -42,7 +42,7 @@ EOF ] } -resource "kubernetes_config_map_v1_data" "kube-dns-upstream-namservers" { +resource "kubernetes_config_map_v1_data" "kube_dns_upstream_nameservers" { count = !local.custom_kube_dns_config && local.upstream_nameservers_config ? 1 : 0 metadata { @@ -64,7 +64,7 @@ EOF ] } -resource "kubernetes_config_map_v1_data" "kube-dns-upstream-nameservers-and-stub-domains" { +resource "kubernetes_config_map_v1_data" "kube_dns_upstream_nameservers_and_stub_domains" { count = local.custom_kube_dns_config && local.upstream_nameservers_config ? 1 : 0 metadata { diff --git a/modules/private-cluster/masq.tf b/modules/private-cluster/masq.tf index b356aee251..cbb862d720 100644 --- a/modules/private-cluster/masq.tf +++ b/modules/private-cluster/masq.tf @@ -19,7 +19,7 @@ /****************************************** Create ip-masq-agent confimap *****************************************/ -resource "kubernetes_config_map" "ip-masq-agent" { +resource "kubernetes_config_map" "ip_masq_agent" { count = var.configure_ip_masq ? 1 : 0 metadata { diff --git a/modules/private-cluster/moved.tf b/modules/private-cluster/moved.tf new file mode 100644 index 0000000000..a745d16489 --- /dev/null +++ b/modules/private-cluster/moved.tf @@ -0,0 +1,62 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# Updates for kebab to snake case, to match best practices and Google style. +moved { + from = kubernetes_config_map_v1_data.kube-dns + to = kubernetes_config_map_v1_data.kube_dns +} + +# Typo fix and snake case at the same time +moved { + from = kubernetes_config_map_v1_data.kube-dns-upstream-namservers + to = kubernetes_config_map_v1_data.kube_dns_upstream_nameservers +} + +moved { + from = kubernetes_config_map_v1_data.kube-dns-upstream-nameservers-and-stub-domains + to = kubernetes_config_map_v1_data.kube_dns_upstream_nameservers_and_stub_domains +} + +moved { + from = kubernetes_config_map.ip-masq-agent + to = kubernetes_config_map.ip_masq_agent +} + +moved { + from = google_project_iam_member.cluster_service_account-nodeService_account + to = google_project_iam_member.cluster_service_account_node_service_account +} + +moved { + from = google_project_iam_member.cluster_service_account-metric_writer + to = google_project_iam_member.cluster_service_account_metric_writer +} + +moved { + from = google_project_iam_member.cluster_service_account-resourceMetadata-writer + to = google_project_iam_member.cluster_service_account_resource_metadata_writer +} + +moved { + from = google_project_iam_member.cluster_service_account-gcr + to = google_project_iam_member.cluster_service_account_gcr +} + +moved { + from = google_project_iam_member.cluster_service_account-artifact-registry + to = google_project_iam_member.cluster_service_account_artifact_registry +} diff --git a/modules/private-cluster/sa.tf b/modules/private-cluster/sa.tf index 72d044a4c9..a8923ce1db 100644 --- a/modules/private-cluster/sa.tf +++ b/modules/private-cluster/sa.tf @@ -46,35 +46,35 @@ resource "google_service_account" "cluster_service_account" { display_name = "Terraform-managed service account for cluster ${var.name}" } -resource "google_project_iam_member" "cluster_service_account-nodeService_account" { +resource "google_project_iam_member" "cluster_service_account_node_service_account" { count = var.create_service_account ? 1 : 0 project = google_service_account.cluster_service_account[0].project role = "roles/container.defaultNodeServiceAccount" member = google_service_account.cluster_service_account[0].member } -resource "google_project_iam_member" "cluster_service_account-metric_writer" { +resource "google_project_iam_member" "cluster_service_account_metric_writer" { count = var.create_service_account ? 1 : 0 project = google_service_account.cluster_service_account[0].project role = "roles/monitoring.metricWriter" member = google_service_account.cluster_service_account[0].member } -resource "google_project_iam_member" "cluster_service_account-resourceMetadata-writer" { +resource "google_project_iam_member" "cluster_service_account_resource_metadata_writer" { count = var.create_service_account ? 1 : 0 project = google_service_account.cluster_service_account[0].project role = "roles/stackdriver.resourceMetadata.writer" member = google_service_account.cluster_service_account[0].member } -resource "google_project_iam_member" "cluster_service_account-gcr" { +resource "google_project_iam_member" "cluster_service_account_gcr" { for_each = var.create_service_account && var.grant_registry_access ? toset(local.registry_projects_list) : [] project = each.key role = "roles/storage.objectViewer" member = "serviceAccount:${google_service_account.cluster_service_account[0].email}" } -resource "google_project_iam_member" "cluster_service_account-artifact-registry" { +resource "google_project_iam_member" "cluster_service_account_artifact_registry" { for_each = var.create_service_account && var.grant_registry_access ? toset(local.registry_projects_list) : [] project = each.key role = "roles/artifactregistry.reader" diff --git a/modules/private-cluster/variables.tf b/modules/private-cluster/variables.tf index 728b64ad7d..70b2619e6c 100644 --- a/modules/private-cluster/variables.tf +++ b/modules/private-cluster/variables.tf @@ -108,6 +108,12 @@ variable "service_external_ips" { default = false } +variable "insecure_kubelet_readonly_port_enabled" { + type = bool + description = "Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`." + default = null +} + variable "datapath_provider" { type = string description = "The desired datapath provider for this cluster. By default, `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation. `ADVANCED_DATAPATH` enables Dataplane-V2 feature." diff --git a/modules/private-cluster/versions.tf b/modules/private-cluster/versions.tf index acdcd3eef6..8387c73b70 100644 --- a/modules/private-cluster/versions.tf +++ b/modules/private-cluster/versions.tf @@ -21,7 +21,7 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 5.40.0, < 7" + version = ">= 5.44.2, !=6.0.0, !=6.0.1, !=6.1.0, !=6.2.0, !=6.3.0, !=6.4.0, !=6.5.0, !=6.6.0, < 7" } kubernetes = { source = "hashicorp/kubernetes" @@ -33,6 +33,6 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:private-cluster/v33.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:private-cluster/v34.0.0" } } diff --git a/modules/safer-cluster-update-variant/README.md b/modules/safer-cluster-update-variant/README.md index d2203ddf33..759d11f8b1 100644 --- a/modules/safer-cluster-update-variant/README.md +++ b/modules/safer-cluster-update-variant/README.md @@ -209,8 +209,9 @@ For simplicity, we suggest using `roles/container.admin` and | cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | | cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | | cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | -| compute\_engine\_service\_account | Use the given service account for nodes rather than creating a new dedicated service account. | `string` | `""` | no | +| compute\_engine\_service\_account | Use the given service account for nodes rather than creating a new dedicated service account. If set then also set var.create\_service\_account to false to avoid 'value depends on resource attributes that cannot be determined until apply' errors. | `string` | `""` | no | | config\_connector | Whether ConfigConnector is enabled for this cluster. | `bool` | `false` | no | +| create\_service\_account | Defines if service account specified to run nodes should be created. Explicitly set to false if var.compute\_engine\_service\_account is set to avoid 'value depends on resource attributes that cannot be determined until apply' errors. | `bool` | `true` | no | | database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key\_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key\_name is the name of a CloudKMS key. | `list(object({ state = string, key_name = string }))` | 
[
  {
    "key_name": "",
    "state": "DECRYPTED"
  }
]
| no | | datapath\_provider | The desired datapath provider for this cluster. By default, `ADVANCED_DATAPATH` enables Dataplane-V2 feature. `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation as a fallback since upgrading to V2 requires a cluster re-creation. | `string` | `"ADVANCED_DATAPATH"` | no | | default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | `number` | `110` | no | diff --git a/modules/safer-cluster-update-variant/main.tf b/modules/safer-cluster-update-variant/main.tf index 85f55f3e7b..a13fafe5fe 100644 --- a/modules/safer-cluster-update-variant/main.tf +++ b/modules/safer-cluster-update-variant/main.tf @@ -115,10 +115,15 @@ module "gke" { // All applications should run with an identity defined via Workload Identity anyway. // - Use a service account passed as a parameter to the module, in case the user // wants to maintain control of their service accounts. - create_service_account = var.compute_engine_service_account == "" ? true : false - service_account = var.compute_engine_service_account - registry_project_ids = var.registry_project_ids - grant_registry_access = var.grant_registry_access + service_account = var.compute_engine_service_account + registry_project_ids = var.registry_project_ids + grant_registry_access = var.grant_registry_access + + // If create_service_account is explicitly set to false we short-circuit the + // compute_engine_service_account check to potentially avoid an error (see variables.tf documentation). + // Otherwise if true (the default), we check if compute_engine_service_account is set for backwards compatability + // before the create_service_account variable was added. + create_service_account = var.create_service_account == false ? var.create_service_account : (var.compute_engine_service_account == "" ? true : false) issue_client_certificate = false diff --git a/modules/safer-cluster-update-variant/variables.tf b/modules/safer-cluster-update-variant/variables.tf index 1934526404..02d6f8e526 100644 --- a/modules/safer-cluster-update-variant/variables.tf +++ b/modules/safer-cluster-update-variant/variables.tf @@ -411,10 +411,16 @@ variable "authenticator_security_group" { variable "compute_engine_service_account" { type = string - description = "Use the given service account for nodes rather than creating a new dedicated service account." + description = "Use the given service account for nodes rather than creating a new dedicated service account. If set then also set var.create_service_account to false to avoid 'value depends on resource attributes that cannot be determined until apply' errors." default = "" } +variable "create_service_account" { + type = bool + description = "Defines if service account specified to run nodes should be created. Explicitly set to false if var.compute_engine_service_account is set to avoid 'value depends on resource attributes that cannot be determined until apply' errors." + default = true +} + variable "enable_shielded_nodes" { type = bool description = "Enable Shielded Nodes features on all nodes in this cluster." diff --git a/modules/safer-cluster-update-variant/versions.tf b/modules/safer-cluster-update-variant/versions.tf index 11a26b6144..14c48578d0 100644 --- a/modules/safer-cluster-update-variant/versions.tf +++ b/modules/safer-cluster-update-variant/versions.tf @@ -21,6 +21,6 @@ terraform { required_version = ">=1.3" provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:safer-cluster-update-variant/v33.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:safer-cluster-update-variant/v34.0.0" } } diff --git a/modules/safer-cluster/README.md b/modules/safer-cluster/README.md index d2203ddf33..759d11f8b1 100644 --- a/modules/safer-cluster/README.md +++ b/modules/safer-cluster/README.md @@ -209,8 +209,9 @@ For simplicity, we suggest using `roles/container.admin` and | cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | | cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | | cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | -| compute\_engine\_service\_account | Use the given service account for nodes rather than creating a new dedicated service account. | `string` | `""` | no | +| compute\_engine\_service\_account | Use the given service account for nodes rather than creating a new dedicated service account. If set then also set var.create\_service\_account to false to avoid 'value depends on resource attributes that cannot be determined until apply' errors. | `string` | `""` | no | | config\_connector | Whether ConfigConnector is enabled for this cluster. | `bool` | `false` | no | +| create\_service\_account | Defines if service account specified to run nodes should be created. Explicitly set to false if var.compute\_engine\_service\_account is set to avoid 'value depends on resource attributes that cannot be determined until apply' errors. | `bool` | `true` | no | | database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key\_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key\_name is the name of a CloudKMS key. | `list(object({ state = string, key_name = string }))` | 
[
  {
    "key_name": "",
    "state": "DECRYPTED"
  }
]
| no | | datapath\_provider | The desired datapath provider for this cluster. By default, `ADVANCED_DATAPATH` enables Dataplane-V2 feature. `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation as a fallback since upgrading to V2 requires a cluster re-creation. | `string` | `"ADVANCED_DATAPATH"` | no | | default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | `number` | `110` | no | diff --git a/modules/safer-cluster/main.tf b/modules/safer-cluster/main.tf index 3c67db4830..e113c09a6a 100644 --- a/modules/safer-cluster/main.tf +++ b/modules/safer-cluster/main.tf @@ -115,10 +115,15 @@ module "gke" { // All applications should run with an identity defined via Workload Identity anyway. // - Use a service account passed as a parameter to the module, in case the user // wants to maintain control of their service accounts. - create_service_account = var.compute_engine_service_account == "" ? true : false - service_account = var.compute_engine_service_account - registry_project_ids = var.registry_project_ids - grant_registry_access = var.grant_registry_access + service_account = var.compute_engine_service_account + registry_project_ids = var.registry_project_ids + grant_registry_access = var.grant_registry_access + + // If create_service_account is explicitly set to false we short-circuit the + // compute_engine_service_account check to potentially avoid an error (see variables.tf documentation). + // Otherwise if true (the default), we check if compute_engine_service_account is set for backwards compatability + // before the create_service_account variable was added. + create_service_account = var.create_service_account == false ? var.create_service_account : (var.compute_engine_service_account == "" ? true : false) issue_client_certificate = false diff --git a/modules/safer-cluster/variables.tf b/modules/safer-cluster/variables.tf index 1934526404..02d6f8e526 100644 --- a/modules/safer-cluster/variables.tf +++ b/modules/safer-cluster/variables.tf @@ -411,10 +411,16 @@ variable "authenticator_security_group" { variable "compute_engine_service_account" { type = string - description = "Use the given service account for nodes rather than creating a new dedicated service account." + description = "Use the given service account for nodes rather than creating a new dedicated service account. If set then also set var.create_service_account to false to avoid 'value depends on resource attributes that cannot be determined until apply' errors." default = "" } +variable "create_service_account" { + type = bool + description = "Defines if service account specified to run nodes should be created. Explicitly set to false if var.compute_engine_service_account is set to avoid 'value depends on resource attributes that cannot be determined until apply' errors." + default = true +} + variable "enable_shielded_nodes" { type = bool description = "Enable Shielded Nodes features on all nodes in this cluster." diff --git a/modules/safer-cluster/versions.tf b/modules/safer-cluster/versions.tf index 486ae99f95..52eebd8aad 100644 --- a/modules/safer-cluster/versions.tf +++ b/modules/safer-cluster/versions.tf @@ -21,6 +21,6 @@ terraform { required_version = ">=1.3" provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:safer-cluster/v33.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:safer-cluster/v34.0.0" } } diff --git a/modules/services/main.tf b/modules/services/main.tf index 25daf441b6..e6cb613f18 100644 --- a/modules/services/main.tf +++ b/modules/services/main.tf @@ -16,7 +16,7 @@ module "services" { source = "terraform-google-modules/project-factory/google//modules/project_services" - version = "~> 16.0" + version = "~> 17.0" project_id = var.project_id enable_apis = var.enable_apis diff --git a/modules/services/versions.tf b/modules/services/versions.tf index 7fe495b761..21f0d2d161 100644 --- a/modules/services/versions.tf +++ b/modules/services/versions.tf @@ -19,6 +19,6 @@ terraform { required_version = ">= 0.13.0" provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:services/v33.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:services/v34.0.0" } } diff --git a/modules/workload-identity/README.md b/modules/workload-identity/README.md index d11a8dbb71..d7e374ad6e 100644 --- a/modules/workload-identity/README.md +++ b/modules/workload-identity/README.md @@ -148,6 +148,7 @@ Error: Get "http://localhost/api/v1/namespaces/default/serviceaccounts/your-serv | annotate\_k8s\_sa | Annotate the kubernetes service account with 'iam.gke.io/gcp-service-account' annotation. Valid in cases when an existing SA is used. | `bool` | `true` | no | | automount\_service\_account\_token | Enable automatic mounting of the service account token | `bool` | `false` | no | | cluster\_name | Cluster name. Required if using existing KSA. | `string` | `""` | no | +| gcp\_sa\_create\_ignore\_already\_exists | If set to true, skip service account creation if a service account with the same email already exists. | `bool` | `null` | no | | gcp\_sa\_description | The Service Google service account desciption; if null, will be left out | `string` | `null` | no | | gcp\_sa\_display\_name | The Google service account display name; if null, a default string will be used | `string` | `null` | no | | gcp\_sa\_name | Name for the Google service account; overrides `var.name`. | `string` | `null` | no | diff --git a/modules/workload-identity/main.tf b/modules/workload-identity/main.tf index 2c2d278ae3..eaa9a0b47d 100644 --- a/modules/workload-identity/main.tf +++ b/modules/workload-identity/main.tf @@ -42,10 +42,11 @@ data "google_service_account" "cluster_service_account" { resource "google_service_account" "cluster_service_account" { count = var.use_existing_gcp_sa ? 0 : 1 - account_id = local.gcp_given_name - display_name = coalesce(var.gcp_sa_display_name, substr("GCP SA bound to K8S SA ${local.k8s_sa_project_id}[${local.k8s_given_name}]", 0, 100)) - description = var.gcp_sa_description - project = var.project_id + account_id = local.gcp_given_name + display_name = coalesce(var.gcp_sa_display_name, substr("GCP SA bound to K8S SA ${local.k8s_sa_project_id}[${local.k8s_given_name}]", 0, 100)) + description = var.gcp_sa_description + project = var.project_id + create_ignore_already_exists = var.gcp_sa_create_ignore_already_exists } resource "kubernetes_service_account" "main" { diff --git a/modules/workload-identity/variables.tf b/modules/workload-identity/variables.tf index 32b1c861dc..c773126532 100644 --- a/modules/workload-identity/variables.tf +++ b/modules/workload-identity/variables.tf @@ -137,3 +137,9 @@ variable "gcp_sa_description" { error_message = "The Google service account description must be at most 256 characters" } } + +variable "gcp_sa_create_ignore_already_exists" { + description = "If set to true, skip service account creation if a service account with the same email already exists." + type = bool + default = null +} diff --git a/modules/workload-identity/versions.tf b/modules/workload-identity/versions.tf index 78f6e941d8..35aa3da18a 100644 --- a/modules/workload-identity/versions.tf +++ b/modules/workload-identity/versions.tf @@ -30,6 +30,6 @@ terraform { } provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:workload-identity/v33.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:workload-identity/v34.0.0" } } diff --git a/moved.tf b/moved.tf new file mode 100644 index 0000000000..a745d16489 --- /dev/null +++ b/moved.tf @@ -0,0 +1,62 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# Updates for kebab to snake case, to match best practices and Google style. +moved { + from = kubernetes_config_map_v1_data.kube-dns + to = kubernetes_config_map_v1_data.kube_dns +} + +# Typo fix and snake case at the same time +moved { + from = kubernetes_config_map_v1_data.kube-dns-upstream-namservers + to = kubernetes_config_map_v1_data.kube_dns_upstream_nameservers +} + +moved { + from = kubernetes_config_map_v1_data.kube-dns-upstream-nameservers-and-stub-domains + to = kubernetes_config_map_v1_data.kube_dns_upstream_nameservers_and_stub_domains +} + +moved { + from = kubernetes_config_map.ip-masq-agent + to = kubernetes_config_map.ip_masq_agent +} + +moved { + from = google_project_iam_member.cluster_service_account-nodeService_account + to = google_project_iam_member.cluster_service_account_node_service_account +} + +moved { + from = google_project_iam_member.cluster_service_account-metric_writer + to = google_project_iam_member.cluster_service_account_metric_writer +} + +moved { + from = google_project_iam_member.cluster_service_account-resourceMetadata-writer + to = google_project_iam_member.cluster_service_account_resource_metadata_writer +} + +moved { + from = google_project_iam_member.cluster_service_account-gcr + to = google_project_iam_member.cluster_service_account_gcr +} + +moved { + from = google_project_iam_member.cluster_service_account-artifact-registry + to = google_project_iam_member.cluster_service_account_artifact_registry +} diff --git a/sa.tf b/sa.tf index 72d044a4c9..a8923ce1db 100644 --- a/sa.tf +++ b/sa.tf @@ -46,35 +46,35 @@ resource "google_service_account" "cluster_service_account" { display_name = "Terraform-managed service account for cluster ${var.name}" } -resource "google_project_iam_member" "cluster_service_account-nodeService_account" { +resource "google_project_iam_member" "cluster_service_account_node_service_account" { count = var.create_service_account ? 1 : 0 project = google_service_account.cluster_service_account[0].project role = "roles/container.defaultNodeServiceAccount" member = google_service_account.cluster_service_account[0].member } -resource "google_project_iam_member" "cluster_service_account-metric_writer" { +resource "google_project_iam_member" "cluster_service_account_metric_writer" { count = var.create_service_account ? 1 : 0 project = google_service_account.cluster_service_account[0].project role = "roles/monitoring.metricWriter" member = google_service_account.cluster_service_account[0].member } -resource "google_project_iam_member" "cluster_service_account-resourceMetadata-writer" { +resource "google_project_iam_member" "cluster_service_account_resource_metadata_writer" { count = var.create_service_account ? 1 : 0 project = google_service_account.cluster_service_account[0].project role = "roles/stackdriver.resourceMetadata.writer" member = google_service_account.cluster_service_account[0].member } -resource "google_project_iam_member" "cluster_service_account-gcr" { +resource "google_project_iam_member" "cluster_service_account_gcr" { for_each = var.create_service_account && var.grant_registry_access ? toset(local.registry_projects_list) : [] project = each.key role = "roles/storage.objectViewer" member = "serviceAccount:${google_service_account.cluster_service_account[0].email}" } -resource "google_project_iam_member" "cluster_service_account-artifact-registry" { +resource "google_project_iam_member" "cluster_service_account_artifact_registry" { for_each = var.create_service_account && var.grant_registry_access ? toset(local.registry_projects_list) : [] project = each.key role = "roles/artifactregistry.reader" diff --git a/test/fixtures/simple_regional_cluster_autoscaling/example.tf b/test/fixtures/simple_regional_cluster_autoscaling/example.tf new file mode 100644 index 0000000000..f7651de4fb --- /dev/null +++ b/test/fixtures/simple_regional_cluster_autoscaling/example.tf @@ -0,0 +1,28 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "example" { + source = "../../../examples/simple_regional_cluster_autoscaling" + + project_id = var.project_ids[0] + cluster_name_suffix = "-${random_string.suffix.result}" + region = "us-west1" + network = google_compute_network.main.name + subnetwork = google_compute_subnetwork.main.name + ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name + ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name + compute_engine_service_account = var.compute_engine_service_accounts[0] +} diff --git a/test/fixtures/simple_regional_cluster_autoscaling/network.tf b/test/fixtures/simple_regional_cluster_autoscaling/network.tf new file mode 100644 index 0000000000..03e1e87b16 --- /dev/null +++ b/test/fixtures/simple_regional_cluster_autoscaling/network.tf @@ -0,0 +1,48 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "random_string" "suffix" { + length = 4 + special = false + upper = false +} + +provider "google" { + project = var.project_ids[0] +} + +resource "google_compute_network" "main" { + name = "cft-gke-test-${random_string.suffix.result}" + auto_create_subnetworks = false +} + +resource "google_compute_subnetwork" "main" { + name = "cft-gke-test-${random_string.suffix.result}" + ip_cidr_range = "10.0.0.0/17" + region = "us-west1" + network = google_compute_network.main.self_link + + secondary_ip_range { + range_name = "cft-gke-test-pods-${random_string.suffix.result}" + ip_cidr_range = "192.168.0.0/18" + } + + secondary_ip_range { + range_name = "cft-gke-test-services-${random_string.suffix.result}" + ip_cidr_range = "192.168.64.0/18" + } +} + diff --git a/test/fixtures/simple_regional_cluster_autoscaling/outputs.tf b/test/fixtures/simple_regional_cluster_autoscaling/outputs.tf new file mode 100644 index 0000000000..57b647ad8a --- /dev/null +++ b/test/fixtures/simple_regional_cluster_autoscaling/outputs.tf @@ -0,0 +1,85 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "project_id" { + value = module.example.project_id +} + +output "region" { + value = module.example.region +} + +output "cluster_name" { + description = "Cluster name" + value = module.example.cluster_name +} + +output "network" { + value = google_compute_network.main.name +} + +output "subnetwork" { + value = google_compute_subnetwork.main.name +} + +output "location" { + value = module.example.location +} + +output "ip_range_pods" { + description = "The secondary IP range used for pods" + value = google_compute_subnetwork.main.secondary_ip_range[0].range_name +} + +output "ip_range_services" { + description = "The secondary IP range used for services" + value = google_compute_subnetwork.main.secondary_ip_range[1].range_name +} + +output "zones" { + description = "List of zones in which the cluster resides" + value = module.example.zones +} + +output "master_kubernetes_version" { + description = "The master Kubernetes version" + value = module.example.master_kubernetes_version +} + +output "kubernetes_endpoint" { + sensitive = true + value = module.example.kubernetes_endpoint +} + +output "client_token" { + sensitive = true + value = module.example.client_token +} + +output "ca_certificate" { + description = "The cluster CA certificate" + value = module.example.ca_certificate + sensitive = true +} + +output "service_account" { + description = "The service account to default running nodes as if not overridden in `node_pools`." + value = module.example.service_account +} + +output "registry_project_ids" { + value = var.registry_project_ids +} diff --git a/test/fixtures/simple_regional_cluster_autoscaling/variables.tf b/test/fixtures/simple_regional_cluster_autoscaling/variables.tf new file mode 100644 index 0000000000..a5bf1908f1 --- /dev/null +++ b/test/fixtures/simple_regional_cluster_autoscaling/variables.tf @@ -0,0 +1,47 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_ids" { + type = list(string) + description = "The GCP projects to use for integration tests" +} + +variable "region" { + description = "The GCP region to create and test resources in" + default = "us-central1" +} + +variable "zones" { + type = list(string) + description = "The GCP zones to create and test resources in, for applicable tests" + default = ["us-central1-a", "us-central1-b", "us-central1-c"] +} + +variable "compute_engine_service_accounts" { + type = list(string) + description = "The email addresses of the service account to associate with the GKE cluster" +} + +variable "registry_project_ids" { + description = "Projects to use for granting access to GCR registries, if requested" + type = list(string) +} + +variable "kubernetes_version" { + type = string + description = "The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region." + default = "latest" +} diff --git a/test/integration/autopilot_private_firewalls/autopilot_private_firewalls_test.go b/test/integration/autopilot_private_firewalls/autopilot_private_firewalls_test.go index 8ba773fbc1..db6aca31e5 100644 --- a/test/integration/autopilot_private_firewalls/autopilot_private_firewalls_test.go +++ b/test/integration/autopilot_private_firewalls/autopilot_private_firewalls_test.go @@ -26,8 +26,7 @@ import ( func TestAutopilotPrivateFirewalls(t *testing.T) { bpt := tft.NewTFBlueprintTest(t) bpt.DefineVerify(func(assert *assert.Assertions) { - //Skipping Default Verify as the Verify Stage fails due to change in Client Cert Token - // bpt.DefaultVerify(assert) + bpt.DefaultVerify(assert) projectId := bpt.GetStringOutput("project_id") location := bpt.GetStringOutput("location") clusterName := bpt.GetStringOutput("cluster_name") diff --git a/test/integration/beta_cluster/beta_cluster_test.go b/test/integration/beta_cluster/beta_cluster_test.go index c5429eaee5..ecc481d18f 100755 --- a/test/integration/beta_cluster/beta_cluster_test.go +++ b/test/integration/beta_cluster/beta_cluster_test.go @@ -33,8 +33,7 @@ func TestBetaCluster(t *testing.T) { ) gke.DefineVerify(func(assert *assert.Assertions) { - // Commenting Default Verify due to issue 1478 for location Policy - // gke.DefaultVerify(assert) //disables no changes + gke.DefaultVerify(assert) projectId := gke.GetStringOutput("project_id") location := gke.GetStringOutput("location") diff --git a/test/integration/go.mod b/test/integration/go.mod index 6e6647d6a4..b53973d259 100644 --- a/test/integration/go.mod +++ b/test/integration/go.mod @@ -2,13 +2,13 @@ module github.com/terraform-google-modules/terraform-google-kubernetes-engine/te go 1.22.0 -toolchain go1.22.7 +toolchain go1.22.8 require ( github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test v0.16.2 github.com/gruntwork-io/terratest v0.47.2 github.com/stretchr/testify v1.9.0 - github.com/tidwall/gjson v1.17.3 + github.com/tidwall/gjson v1.18.0 ) require ( diff --git a/test/integration/go.sum b/test/integration/go.sum index 93b5d8b6ea..fd25867d1d 100644 --- a/test/integration/go.sum +++ b/test/integration/go.sum @@ -489,8 +489,8 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/gjson v1.17.3 h1:bwWLZU7icoKRG+C+0PNwIKC6FCJO/Q3p2pZvuP0jN94= -github.com/tidwall/gjson v1.17.3/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= +github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= diff --git a/test/integration/private_zonal_with_networking/testdata/TestPrivateZonalWithNetworking.json b/test/integration/private_zonal_with_networking/testdata/TestPrivateZonalWithNetworking.json index e14afc8565..1f95d1646d 100755 --- a/test/integration/private_zonal_with_networking/testdata/TestPrivateZonalWithNetworking.json +++ b/test/integration/private_zonal_with_networking/testdata/TestPrivateZonalWithNetworking.json @@ -107,6 +107,9 @@ "diskSizeGb": 100, "diskType": "pd-balanced", "imageType": "COS_CONTAINERD", + "kubeletConfig": { + "insecureKubeletReadonlyPortEnabled": false + }, "loggingConfig": { "variantConfig": { "variant": "DEFAULT" diff --git a/test/integration/sandbox_enabled/sandbox_enabled_test.go b/test/integration/sandbox_enabled/sandbox_enabled_test.go index 204876742c..6d7a92f362 100644 --- a/test/integration/sandbox_enabled/sandbox_enabled_test.go +++ b/test/integration/sandbox_enabled/sandbox_enabled_test.go @@ -22,6 +22,7 @@ import ( "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft" "github.com/stretchr/testify/assert" "github.com/terraform-google-modules/terraform-google-kubernetes-engine/test/integration/testutils" + gkeutils "github.com/terraform-google-modules/terraform-google-kubernetes-engine/test/integration/utils" ) func TestSandboxEnabled(t *testing.T) { @@ -32,6 +33,7 @@ func TestSandboxEnabled(t *testing.T) { bpt.DefineVerify(func(assert *assert.Assertions) { //Skipping Default Verify as the Verify Stage fails due to change in Client Cert Token // bpt.DefaultVerify(assert) + gkeutils.TGKEVerify(t, bpt, assert) // Verify Resources projectId := bpt.GetStringOutput("project_id") location := bpt.GetStringOutput("location") diff --git a/test/integration/simple_autopilot_private_non_default_sa/simple_autopilot_private_non_default_sa_test.go b/test/integration/simple_autopilot_private_non_default_sa/simple_autopilot_private_non_default_sa_test.go index e14440edb4..33e6bc5e9e 100644 --- a/test/integration/simple_autopilot_private_non_default_sa/simple_autopilot_private_non_default_sa_test.go +++ b/test/integration/simple_autopilot_private_non_default_sa/simple_autopilot_private_non_default_sa_test.go @@ -33,8 +33,7 @@ func TestSimpleAutopilotPrivateNonDefaultSA(t *testing.T) { ) bpt.DefineVerify(func(assert *assert.Assertions) { - //Skipping Default Verify as the Verify Stage fails due to change in Client Cert Token - // bpt.DefaultVerify(assert) + bpt.DefaultVerify(assert) location := bpt.GetStringOutput("location") clusterName := bpt.GetStringOutput("cluster_name") diff --git a/test/integration/simple_autopilot_public/simple_autopiliot_public_test.go b/test/integration/simple_autopilot_public/simple_autopiliot_public_test.go index 1e0221e0c2..36f82ad91d 100644 --- a/test/integration/simple_autopilot_public/simple_autopiliot_public_test.go +++ b/test/integration/simple_autopilot_public/simple_autopiliot_public_test.go @@ -30,8 +30,7 @@ func TestSimpleAutopilotPublic(t *testing.T) { ) bpt.DefineVerify(func(assert *assert.Assertions) { - //Skipping Default Verify as the Verify Stage fails due to change in Client Cert Token - // bpt.DefaultVerify(assert) + bpt.DefaultVerify(assert) projectId := bpt.GetStringOutput("project_id") location := bpt.GetStringOutput("location") diff --git a/test/integration/simple_regional_cluster_autoscaling/simple_regional_cluster_autoscaling_test.go b/test/integration/simple_regional_cluster_autoscaling/simple_regional_cluster_autoscaling_test.go new file mode 100644 index 0000000000..d0ecfd2170 --- /dev/null +++ b/test/integration/simple_regional_cluster_autoscaling/simple_regional_cluster_autoscaling_test.go @@ -0,0 +1,65 @@ +// Copyright 2022-2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package simple_regional_cluster_autoscaling + +import ( + "testing" + "time" + + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/golden" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft" + "github.com/stretchr/testify/assert" + "github.com/terraform-google-modules/terraform-google-kubernetes-engine/test/integration/testutils" +) + +func TestSimpleRegionalClusterAutoscaling(t *testing.T) { + bpt := tft.NewTFBlueprintTest(t, + tft.WithRetryableTerraformErrors(testutils.RetryableTransientErrors, 3, 2*time.Minute), + ) + + bpt.DefineVerify(func(assert *assert.Assertions) { + projectId := bpt.GetStringOutput("project_id") + location := bpt.GetStringOutput("location") + clusterName := bpt.GetStringOutput("cluster_name") + serviceAccount := bpt.GetStringOutput("service_account") + + op := gcloud.Runf(t, "container clusters describe %s --zone %s --project %s", clusterName, location, projectId) + g := golden.NewOrUpdate(t, op.String(), + golden.WithSanitizer(golden.StringSanitizer(serviceAccount, "SERVICE_ACCOUNT")), + golden.WithSanitizer(golden.StringSanitizer(projectId, "PROJECT_ID")), + golden.WithSanitizer(golden.StringSanitizer(clusterName, "CLUSTER_NAME")), + ) + validateJSONPaths := []string{ + "status", + "location", + "privateClusterConfig.enablePrivateEndpoint", + "privateClusterConfig.enablePrivateNodes", + "addonsConfig", + "databaseEncryption", + "shieldedNodes", + "binaryAuthorization", + "nodePools.autoscaling", + "nodePools.config", + "nodePools.config.tags", + "nodePools.management", + } + for _, pth := range validateJSONPaths { + g.JSONEq(assert, op, pth) + } + + }) + + bpt.Test() +} diff --git a/test/integration/simple_regional_cluster_autoscaling/testdata/TestSimpleRegionalClusterAutoscaling.json b/test/integration/simple_regional_cluster_autoscaling/testdata/TestSimpleRegionalClusterAutoscaling.json new file mode 100644 index 0000000000..a922600484 --- /dev/null +++ b/test/integration/simple_regional_cluster_autoscaling/testdata/TestSimpleRegionalClusterAutoscaling.json @@ -0,0 +1,325 @@ +{ + "addonsConfig": { + "configConnectorConfig": {}, + "dnsCacheConfig": {}, + "gcePersistentDiskCsiDriverConfig": { + "enabled": true + }, + "gcpFilestoreCsiDriverConfig": {}, + "gkeBackupAgentConfig": {}, + "horizontalPodAutoscaling": {}, + "httpLoadBalancing": {}, + "kubernetesDashboard": { + "disabled": true + }, + "networkPolicyConfig": { + "disabled": true + } + }, + "autopilot": {}, + "autoscaling": { + "autoscalingProfile": "BALANCED" + }, + "binaryAuthorization": {}, + "clusterIpv4Cidr": "192.168.0.0/18", + "costManagementConfig": { + "enabled": true + }, + "createTime": "2023-08-25T16:39:57+00:00", + "currentMasterVersion": "1.27.3-gke.100", + "currentNodeCount": 3, + "currentNodeVersion": "1.27.3-gke.100", + "databaseEncryption": { + "currentState": "CURRENT_STATE_DECRYPTED", + "state": "DECRYPTED" + }, + "defaultMaxPodsConstraint": { + "maxPodsPerNode": "110" + }, + "endpoint": "35.238.117.51", + "etag": "a214a2ff-16ea-4e65-8c01-43e7a6cece82", + "id": "8e4011253bcb4fbc943f88ae797f124f0f001ed95cc94b229231d68b8a44e20b", + "initialClusterVersion": "1.27.3-gke.100", + "instanceGroupUrls": [ + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-west1-a/instanceGroupManagers/gke-simple-regional-clus-default-pool-209983a6-grp", + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-west1-c/instanceGroupManagers/gke-simple-regional-clus-default-pool-6094c28c-grp", + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-west1-f/instanceGroupManagers/gke-simple-regional-clus-default-pool-a9225012-grp", + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-west1-a/instanceGroupManagers/gke-simple-regional--default-node-poo-8fb4fd0f-grp", + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-west1-c/instanceGroupManagers/gke-simple-regional--default-node-poo-24ffed9f-grp", + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-west1-f/instanceGroupManagers/gke-simple-regional--default-node-poo-1d2dc357-grp" + ], + "ipAllocationPolicy": { + "clusterIpv4Cidr": "192.168.0.0/18", + "clusterIpv4CidrBlock": "192.168.0.0/18", + "clusterSecondaryRangeName": "cft-gke-test-pods-44kc", + "defaultPodIpv4RangeUtilization": 0.0469, + "podCidrOverprovisionConfig": {}, + "servicesIpv4Cidr": "192.168.64.0/18", + "servicesIpv4CidrBlock": "192.168.64.0/18", + "servicesSecondaryRangeName": "cft-gke-test-services-44kc", + "stackType": "IPV4", + "useIpAliases": true + }, + "labelFingerprint": "a9dc16a7", + "legacyAbac": {}, + "location": "us-west1", + "locations": [ + "us-west1-a", + "us-west1-c", + "us-west1-f" + ], + "loggingConfig": { + "componentConfig": { + "enableComponents": [ + "SYSTEM_COMPONENTS", + "WORKLOADS" + ] + } + }, + "loggingService": "logging.googleapis.com/kubernetes", + "maintenancePolicy": { + "resourceVersion": "ce912209", + "window": { + "dailyMaintenanceWindow": { + "duration": "PT4H0M0S", + "startTime": "05:00" + } + } + }, + "masterAuth": { + "clusterCaCertificate": "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVMRENDQXBTZ0F3SUJBZ0lRQ0J5aEs4ZEtSMTNBeGIrTkxTM1FSREFOQmdrcWhraUc5dzBCQVFzRkFEQXYKTVMwd0t3WURWUVFERXlReE56VTNaR1l4WkMwMFpUY3dMVFF5TVdZdE9XWmpPUzB4TVRZelpXRXdNbUpsWmpRdwpJQmNOTWpNd09ESTFNVFV6T1RVM1doZ1BNakExTXpBNE1UY3hOak01TlRkYU1DOHhMVEFyQmdOVkJBTVRKREUzCk5UZGtaakZrTFRSbE56QXROREl4WmkwNVptTTVMVEV4TmpObFlUQXlZbVZtTkRDQ0FhSXdEUVlKS29aSWh2Y04KQVFFQkJRQURnZ0dQQURDQ0FZb0NnZ0dCQUtDYW1KRWwvT04rUUxIWGh0dVVaVVQ5VHFYMnNoeTZtL2lvQ3d0dgp1TG9LVTYzQzRvS0N1ajQzbXRYMVFTaUFRLzloNHVqV0xucW9PUzdpd2FrWnZHMHpBdjJrZmNuVUtXQnM5MTJRCm40K05pcU1EVWRyd1Fab0NwMTl2MDhGOFBSTTZCWFVIRUNnQUtvanhSeVpyTzkrZDVESndXNFRHUzhJWVNleTUKR2xnUkFyQk5LMXBzb09ZMHhaZGpxTUxDNnpkTEMyT01PaVhOcXVMUGNkUFk0aHFGSDZ0cXFOY09kTllNaWVBUwpwNGM3YS83WFA1cnF0M1NiSlhYbkZmVFFmL1dablJiUWh1SDRxRmpUUUVrdEdieW9rT0tEQ09QS0lGNEVwc1czCmR5M2ZVNkhDaWR2V2JCSHEvV21JVWpRWmxlSE5UWkZack5HVC9tcmF2R0RsWjZFaHkvY2psam5DZi90NXduK1AKb1BOZUVjaWxkZDhmNWpqQTc4TW1ValNiV3Z0c01xY2VKTEQ3eUx4RUhGNEZnOXJ1c2wyZU9SZDBQSFkzYmFOZAp4SUlVcVF0M3JORmc4Q0NoSnA0N2w1T3YzTHQzWTI5SjVTRHo2dkM4c3psUkl3NE1TNVQ0OGQ0T200T0VrcVRuCkFmWGw2TjFETzVGRjJKOHZCeHA4Q2Q3NEd3SURBUUFCbzBJd1FEQU9CZ05WSFE4QkFmOEVCQU1DQWdRd0R3WUQKVlIwVEFRSC9CQVV3QXdFQi96QWRCZ05WSFE0RUZnUVVaa1FWVXdBcmZLZEpBSWZzUFBSdmFraktabll3RFFZSgpLb1pJaHZjTkFRRUxCUUFEZ2dHQkFIenEvaWxoekNMeEJuYkp2c2lsdkthTXRpbXRxSjNqTFh5NkdvYW56U3F0ClY1STZFZ01Jd3Zubml4ZVhOb0lrQnhJbDlGcUVmV1g0eHFGLytQb1JoU1dMZHkyZUk1d1I4WkJzaU9lcDA0ZTgKdW80Nk1tdU93alZkM2pOZExOdW1ZRlRqVzR3TStKUytPNUQ0SUo3YXpPVVF1K2gvd2ViQ1pXSjkxbUcxcUtnRQo4WVM3d3pKYlJiU0lQL2tMdWN6aGM4V2dnL1Z2czBldklaNFY2SHRFeFo1UmFjZ3lMcGhvKzNBQlFjakwvVDFWCjRLQ08zUFAyMmo5ZkRLUXNYdEppMHNmV3lMSmwybERwb1ZHMGJGZ2l1MFJxY05XM2dDTTNZYWs3dVRpelozOWYKU3I1NHU0SzBMQVdXbnRqc0NBR0RkajNFYmw0eFVGWWo5SG1zVURKNm5ncFdmejBHeHJQL29rUm5WV05MZjRJMwpuRlYzb0o4S0ZQRjVhUmQwYXNKZnc2a3VpZTNwZytYanlVRlp5ek51a1ZqcUFXdWFVV1lPUjg5Mnd3T0tnNFUrCnJQTUU3RzhqRFBjNWt0MVZ4dktQY0NOckkxdjgralBTSktDYnhCWFU2bXdRSFRnUGVzVWJuUDkrd1FobWpkdHkKME1JOFlpOWltdWtmTy9JMHpSRkJDdz09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K" + }, + "masterAuthorizedNetworksConfig": { + "gcpPublicCidrsAccessEnabled": true + }, + "monitoringConfig": { + "advancedDatapathObservabilityConfig": { + "enableRelay": false + }, + "componentConfig": { + "enableComponents": [ + "SYSTEM_COMPONENTS" + ] + }, + "managedPrometheusConfig": {} + }, + "monitoringService": "monitoring.googleapis.com/kubernetes", + "name": "CLUSTER_NAME", + "network": "cft-gke-test-44kc", + "networkConfig": { + "defaultSnatStatus": {}, + "network": "projects/PROJECT_ID/global/networks/cft-gke-test-44kc", + "serviceExternalIpsConfig": {}, + "subnetwork": "projects/PROJECT_ID/regions/us-west1/subnetworks/cft-gke-test-44kc" + }, + "nodeConfig": { + "diskSizeGb": 100, + "diskType": "pd-balanced", + "imageType": "COS_CONTAINERD", + "loggingConfig": { + "variantConfig": { + "variant": "DEFAULT" + } + }, + "machineType": "e2-medium", + "metadata": { + "disable-legacy-endpoints": "true" + }, + "oauthScopes": [ + "https://www.googleapis.com/auth/userinfo.email", + "https://www.googleapis.com/auth/cloud-platform" + ], + "serviceAccount": "gke-sa-int-test-p1-e155@PROJECT_ID.iam.gserviceaccount.com", + "shieldedInstanceConfig": { + "enableIntegrityMonitoring": true + }, + "tags": [ + "gke-CLUSTER_NAME", + "gke-CLUSTER_NAME-default-pool" + ], + "windowsNodeConfig": {}, + "workloadMetadataConfig": { + "mode": "GKE_METADATA" + } + }, + "nodePoolAutoConfig": {}, + "nodePoolDefaults": { + "nodeConfigDefaults": { + "loggingConfig": { + "variantConfig": { + "variant": "DEFAULT" + } + } + } + }, + "nodePools": [ + { + "config": { + "diskSizeGb": 100, + "diskType": "pd-balanced", + "imageType": "COS_CONTAINERD", + "loggingConfig": { + "variantConfig": { + "variant": "DEFAULT" + } + }, + "machineType": "e2-medium", + "metadata": { + "disable-legacy-endpoints": "true" + }, + "oauthScopes": [ + "https://www.googleapis.com/auth/userinfo.email", + "https://www.googleapis.com/auth/cloud-platform" + ], + "serviceAccount": "gke-sa-int-test-p1-e155@PROJECT_ID.iam.gserviceaccount.com", + "shieldedInstanceConfig": { + "enableIntegrityMonitoring": true + }, + "tags": [ + "gke-CLUSTER_NAME", + "gke-CLUSTER_NAME-default-pool" + ], + "windowsNodeConfig": {}, + "workloadMetadataConfig": { + "mode": "GKE_METADATA" + } + }, + "etag": "3bb00743-9dbf-4e92-9512-1e64fb8f1b8a", + "instanceGroupUrls": [ + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-west1-a/instanceGroupManagers/gke-simple-regional-clus-default-pool-209983a6-grp", + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-west1-c/instanceGroupManagers/gke-simple-regional-clus-default-pool-6094c28c-grp", + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-west1-f/instanceGroupManagers/gke-simple-regional-clus-default-pool-a9225012-grp" + ], + "locations": [ + "us-west1-a", + "us-west1-c", + "us-west1-f" + ], + "management": { + "autoRepair": true, + "autoUpgrade": true + }, + "maxPodsConstraint": { + "maxPodsPerNode": "110" + }, + "name": "default-pool", + "networkConfig": { + "enablePrivateNodes": false, + "podIpv4CidrBlock": "192.168.0.0/18", + "podIpv4RangeUtilization": 0.0469, + "podRange": "cft-gke-test-pods-44kc" + }, + "podIpv4CidrSize": 24, + "selfLink": "https://container.googleapis.com/v1/projects/PROJECT_ID/locations/us-west1/clusters/CLUSTER_NAME/nodePools/default-pool", + "status": "RUNNING", + "upgradeSettings": { + "maxSurge": 1, + "strategy": "SURGE" + }, + "version": "1.27.3-gke.100" + }, + { + "autoscaling": { + "enabled": true, + "locationPolicy": "BALANCED", + "maxNodeCount": 100, + "minNodeCount": 1 + }, + "config": { + "diskSizeGb": 100, + "diskType": "pd-standard", + "imageType": "COS_CONTAINERD", + "labels": { + "cluster_name": "CLUSTER_NAME", + "node_pool": "default-node-pool" + }, + "loggingConfig": { + "variantConfig": { + "variant": "DEFAULT" + } + }, + "machineType": "e2-medium", + "metadata": { + "cluster_name": "CLUSTER_NAME", + "disable-legacy-endpoints": "true", + "node_pool": "default-node-pool" + }, + "oauthScopes": [ + "https://www.googleapis.com/auth/cloud-platform" + ], + "serviceAccount": "gke-sa-int-test-p1-e155@PROJECT_ID.iam.gserviceaccount.com", + "shieldedInstanceConfig": { + "enableIntegrityMonitoring": true + }, + "tags": [ + "gke-CLUSTER_NAME", + "gke-CLUSTER_NAME-default-node-pool" + ], + "windowsNodeConfig": {}, + "workloadMetadataConfig": { + "mode": "GKE_METADATA" + } + }, + "etag": "c1cb03bd-8b4e-4a06-9c4e-213b87aa86a3", + "initialNodeCount": 1, + "instanceGroupUrls": [ + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-west1-a/instanceGroupManagers/gke-simple-regional--default-node-poo-8fb4fd0f-grp", + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-west1-c/instanceGroupManagers/gke-simple-regional--default-node-poo-24ffed9f-grp", + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-west1-f/instanceGroupManagers/gke-simple-regional--default-node-poo-1d2dc357-grp" + ], + "locations": [ + "us-west1-a", + "us-west1-c", + "us-west1-f" + ], + "management": { + "autoRepair": true, + "autoUpgrade": true + }, + "maxPodsConstraint": { + "maxPodsPerNode": "110" + }, + "name": "default-node-pool", + "networkConfig": { + "enablePrivateNodes": false, + "podIpv4CidrBlock": "192.168.0.0/18", + "podIpv4RangeUtilization": 0.0469, + "podRange": "cft-gke-test-pods-44kc" + }, + "podIpv4CidrSize": 24, + "selfLink": "https://container.googleapis.com/v1/projects/PROJECT_ID/locations/us-west1/clusters/CLUSTER_NAME/nodePools/default-node-pool", + "status": "RUNNING", + "upgradeSettings": { + "maxSurge": 1, + "strategy": "SURGE" + }, + "version": "1.27.3-gke.100" + } + ], + "notificationConfig": { + "pubsub": {} + }, + "privateClusterConfig": { + "privateEndpoint": "10.0.0.2", + "publicEndpoint": "35.238.117.51" + }, + "releaseChannel": { + "channel": "REGULAR" + }, + "securityPostureConfig": { + "mode": "BASIC", + "vulnerabilityMode": "VULNERABILITY_MODE_UNSPECIFIED" + }, + "selfLink": "https://container.googleapis.com/v1/projects/PROJECT_ID/locations/us-west1/clusters/CLUSTER_NAME", + "servicesIpv4Cidr": "192.168.64.0/18", + "shieldedNodes": { + "enabled": true + }, + "status": "RUNNING", + "subnetwork": "cft-gke-test-44kc", + "verticalPodAutoscaling": {}, + "workloadIdentityConfig": { + "workloadPool": "PROJECT_ID.svc.id.goog" + }, + "zone": "us-west1" +} diff --git a/test/integration/simple_zonal/simple_zonal_test.go b/test/integration/simple_zonal/simple_zonal_test.go index cb5c796995..a29082134f 100644 --- a/test/integration/simple_zonal/simple_zonal_test.go +++ b/test/integration/simple_zonal/simple_zonal_test.go @@ -79,10 +79,6 @@ func TestSimpleZonal(t *testing.T) { assert.NoError(err) configkubeNS := testutils.ParseKubectlJSONResult(t, configNameSpace) assert.Contains(configkubeNS.Get("metadata.name").String(), "config-management-system", "Namespace is Functional") - gateKeeperNameSpace, err := k8s.RunKubectlAndGetOutputE(t, &k8sOpts, "get", "ns", "gatekeeper-system", "-o", "json") - assert.NoError(err) - gateKeeperkubeNS := testutils.ParseKubectlJSONResult(t, gateKeeperNameSpace) - assert.Contains(gateKeeperkubeNS.Get("metadata.name").String(), "gatekeeper-system", "Namespace is Functional") }) bpt.Test() diff --git a/test/integration/utils/utils.go b/test/integration/utils/utils.go index c96d8396e2..0640ce7273 100644 --- a/test/integration/utils/utils.go +++ b/test/integration/utils/utils.go @@ -15,10 +15,14 @@ package utils import ( + "slices" + "strings" "testing" "time" "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft" + tfjson "github.com/hashicorp/terraform-json" + "github.com/stretchr/testify/assert" "github.com/terraform-google-modules/terraform-google-kubernetes-engine/test/integration/testutils" ) @@ -32,3 +36,22 @@ func GetTestProjectFromSetup(t *testing.T, idx int) string { } return projectIDs[idx] } + +// TGKEVerify asserts no resource changes exist after apply. +func TGKEVerify(t *testing.T, b *tft.TFBlueprintTest, assert *assert.Assertions) { + TGKEVerifyExemptResources(t, b, assert, []string{}) +} + +// TGKEVerifyExemptResources asserts no resource changes exist after apply except exempt resources: e.g. google_container_cluster.primary +func TGKEVerifyExemptResources(t *testing.T, b *tft.TFBlueprintTest, assert *assert.Assertions, verifyExemptResources []string) { + _, ps := b.PlanAndShow() + for _, r := range ps.ResourceChangesMap { + if slices.ContainsFunc(verifyExemptResources, func(str string) bool { + return strings.HasSuffix(r.Address, str) + }) { + t.Logf("Exempt plan address: %s", r.Address) + continue + } + assert.Equal(tfjson.Actions{tfjson.ActionNoop}, r.Change.Actions, "Plan must be no-op for resource: %s", r.Address) + } +} diff --git a/test/setup/main.tf b/test/setup/main.tf index 6f68dbbc02..b94c404385 100644 --- a/test/setup/main.tf +++ b/test/setup/main.tf @@ -45,7 +45,7 @@ locals { module "gke-project-1" { source = "terraform-google-modules/project-factory/google" - version = "~> 16.0" + version = "~> 17.0" name = "ci-gke-${random_id.random_project_id_suffix.hex}" random_project_id = true @@ -69,7 +69,7 @@ module "gke-project-1" { module "gke-project-2" { source = "terraform-google-modules/project-factory/google" - version = "~> 16.0" + version = "~> 17.0" name = "ci-gke-${random_id.random_project_id_suffix.hex}" random_project_id = true @@ -91,7 +91,7 @@ module "gke-project-2" { # apis as documented https://cloud.google.com/service-mesh/docs/scripted-install/reference#setting_up_your_project module "gke-project-asm" { source = "terraform-google-modules/project-factory/google" - version = "~> 16.0" + version = "~> 17.0" name = "ci-gke-asm-${random_id.random_project_id_suffix.hex}" random_project_id = true @@ -106,7 +106,7 @@ module "gke-project-asm" { module "gke-project-fleet" { source = "terraform-google-modules/project-factory/google" - version = "~> 16.0" + version = "~> 17.0" name = "ci-gke-fleet-${random_id.random_project_id_suffix.hex}" random_project_id = true diff --git a/variables.tf b/variables.tf index 4e415d1aab..7f6f1f2a38 100644 --- a/variables.tf +++ b/variables.tf @@ -108,6 +108,12 @@ variable "service_external_ips" { default = false } +variable "insecure_kubelet_readonly_port_enabled" { + type = bool + description = "Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`." + default = null +} + variable "datapath_provider" { type = string description = "The desired datapath provider for this cluster. By default, `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation. `ADVANCED_DATAPATH` enables Dataplane-V2 feature." diff --git a/versions.tf b/versions.tf index 57f0bb9a16..23aeef283a 100644 --- a/versions.tf +++ b/versions.tf @@ -21,7 +21,7 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 5.40.0, < 7" + version = ">= 5.44.2, !=6.0.0, !=6.0.1, !=6.1.0, !=6.2.0, !=6.3.0, !=6.4.0, !=6.5.0, !=6.6.0, < 7" } kubernetes = { source = "hashicorp/kubernetes" @@ -33,6 +33,6 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine/v33.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine/v34.0.0" } }