diff --git a/.github/renovate.json b/.github/renovate.json index 2a109792b9..9b920dbb1c 100644 --- a/.github/renovate.json +++ b/.github/renovate.json @@ -15,24 +15,24 @@ ], "stabilityDays":0 }, + "separateMajorMinor":false, "packageRules": [ { "matchPaths": ["examples/**", "test/**", ".github/**"], "extends": [":semanticCommitTypeAll(chore)"] }, + { + "matchPaths": ["*", "modules/**"], + "extends": [":semanticCommitTypeAll(fix)"] + }, { "matchDepTypes": ["module"], - "groupName": "TF modules", - "separateMajorMinor":false, - "major": { - "semanticCommitType": "feat!" - } + "groupName": "TF modules" }, { "matchDepTypes": ["require"], "groupName": "GO modules", - "postUpdateOptions": ["gomodTidy"], - "separateMajorMinor":false + "postUpdateOptions": ["gomodTidy"] }, { "matchPackageNames": ["go"], @@ -41,8 +41,7 @@ }, { "matchPackageNames": ["google", "google-beta"], - "groupName": "terraform googles", - "separateMajorMinor": false + "groupName": "terraform googles" } ] } diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index dd2c152bc5..c26bb1d3af 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -21,7 +21,7 @@ jobs: stale: runs-on: ubuntu-latest steps: - - uses: actions/stale@v6 + - uses: actions/stale@v7 with: repo-token: ${{ secrets.GITHUB_TOKEN }} stale-issue-message: 'This issue is stale because it has been open 60 days with no activity. Remove stale label or comment or this will be closed in 7 days' diff --git a/.kitchen.yml b/.kitchen.yml index 7f136310bd..a48a53aca2 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -80,6 +80,13 @@ suites: systems: - name: simple_regional_private backend: local + - name: "simple_regional_with_gateway_api" + driver: + root_module_directory: test/fixtures/simple_regional_with_gateway_api + verifier: + systems: + - name: simple_regional_with_gateway_api + backend: local - name: "simple_regional_with_kubeconfig" driver: root_module_directory: test/fixtures/simple_regional_with_kubeconfig diff --git a/CHANGELOG.md b/CHANGELOG.md index c09d90dd04..f8aaef64af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,47 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 Extending the adopted spec, each change should have a link to its corresponding pull request appended. +## [25.0.0](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v24.1.0...v25.0.0) (2023-02-03) + + +### ⚠ BREAKING CHANGES + +* Promote node sysctl config to GA ([#1536](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1536)) +* enable auto repair and upgrade with cluster autoscaling ([#1530](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1530)) +* support for gateway api for safer cluster variants ([#1523](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1523)) +* promote gke_backup_agent_config to ga ([#1513](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1513)) +* enable private nodes with specified pod ip range ([#1514](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1514)) +* Promote managed_prometheus to GA ([#1505](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1505)) +* support for gateway api ([#1510](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1510)) +* Add option to pass `resource_labels` to NP ([#1508](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1508)) +* promote gce_pd_csi_driver to GA ([#1509](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1509)) +* Set the provided SA when creating autopilot clusters ([#1495](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1495)) + +### Features + +* add all pod_ranges to cluster firewall rules and add missing shadow rules ([#1480](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1480)) ([bcd5e03](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/bcd5e03ff2514c79c906ef5c15cc793aa3c7426f)) +* Add option to pass `resource_labels` to NP ([#1508](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1508)) ([e7566c5](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/e7566c592d6a73aeb56762789aeaea072d2bb6ff)) +* add support for policy bundles and metrics SA ([#1529](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1529)) ([0f63eab](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/0f63eab5163a460d485346e2a37992a1b29dc082)) +* promote gce_pd_csi_driver to GA ([#1509](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1509)) ([ac062f8](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/ac062f81e754c52bc785628fe0b2f0442a126599)) +* promote gke_backup_agent_config to ga ([#1513](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1513)) ([966135f](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/966135f2d201d2d05032f31753c87e4f7d43b00a)) +* Promote managed_prometheus to GA ([#1505](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1505)) ([9c77c6c](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/9c77c6c73c7718cfe793e65b946b5a1048f04f71)) +* Promote node sysctl config to GA ([#1536](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1536)) ([754f4e3](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/754f4e367bd43140f48897fc1d3bfdf9e7d4dfce)) +* Set the provided SA when creating autopilot clusters ([#1495](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1495)) ([d122a55](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/d122a55f82c0625ca88ffb1055d758406d902cd1)) +* support for gateway api ([#1510](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1510)) ([4181276](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/4181276635dd16d334a2787ae10e5f0bba0df253)) +* support for gateway api for safer cluster variants ([#1523](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1523)) ([912da8c](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/912da8c97082d08f4fef96e82bbdf6a76dad006b)) + + +### Bug Fixes + +* auth module avoid TPG v4.49.0 ([#1535](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1535)) ([95c5c11](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/95c5c11e4f5856c408b7a1dffa00fa1acac8a5b2)) +* auth module avoid TPG v4.50.0 ([#1541](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1541)) ([c3e08ea](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/c3e08eaa8418fcf9df54d61692c4c7df97ea6968)) +* avoid TGP v4.49.0 for asm ([#1537](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1537)) ([5d3d54e](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/5d3d54e5a2325443515041342f77c7ecdadba2a1)) +* enable auto repair and upgrade with cluster autoscaling ([#1530](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1530)) ([d59542c](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/d59542c8482709028fda0ec53d7bef8749e6a055)) +* enable private nodes with specified pod ip range ([#1514](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1514)) ([8190439](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/8190439f33b6b525fe930b1bfcdc65762d8652ab)) +* remove datapath provider from Autopilot modules ([#1556](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1556)) ([ea012f5](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/ea012f5c57a79c9de56f529dd9cc2ea9d3673838)) +* support custom service account for autopilot ([#1550](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1550)) ([52e25ab](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/52e25ab2c929777fddada40c0c0a03ac03ae75ec)) +* Update variable validation description ([#1518](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1518)) ([d985879](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/d985879b4bb8d995dca04bed9cdfa541914ffa69)) + ## [24.1.0](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v24.0.0...v24.1.0) (2022-12-14) diff --git a/CODEOWNERS b/CODEOWNERS index 7230a0edf8..17739239b6 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1 +1 @@ -* @terraform-google-modules/cft-admins @Jberlinsky +* @terraform-google-modules/cft-admins @Jberlinsky @ericyz diff --git a/README.md b/README.md index de217ce126..2212da8113 100644 --- a/README.md +++ b/README.md @@ -131,7 +131,7 @@ Then perform the following commands on the root folder: | add\_master\_webhook\_firewall\_rules | Create master\_webhook firewall rules for ports defined in `firewall_inbound_ports` | `bool` | `false` | no | | add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | | authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled       = bool
    min_cpu_cores = number
    max_cpu_cores = number
    min_memory_gb = number
    max_memory_gb = number
    gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number }))
  })
| 
{
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled       = bool
    min_cpu_cores = number
    max_cpu_cores = number
    min_memory_gb = number
    max_memory_gb = number
    gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair   = bool
    auto_upgrade  = bool
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | | cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | | cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | | cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | @@ -148,6 +148,7 @@ Then perform the following commands on the root folder: | dns\_cache | The status of the NodeLocal DNSCache addon. | `bool` | `false` | no | | enable\_binary\_authorization | Enable BinAuthZ Admission controller | `bool` | `false` | no | | enable\_cost\_allocation | Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery | `bool` | `false` | no | +| enable\_kubernetes\_alpha | Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days. | `bool` | `false` | no | | enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `bool` | `false` | no | | enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | `bool` | `true` | no | | enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | `bool` | `true` | no | @@ -155,6 +156,9 @@ Then perform the following commands on the root folder: | filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | | firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | | firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | +| gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | +| gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | +| gke\_backup\_agent\_config | Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | | http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no | @@ -166,12 +170,15 @@ Then perform the following commands on the root folder: | ip\_range\_services | The _name_ of the secondary subnet range to use for services | `string` | n/a | yes | | issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | | kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | `string` | `"latest"` | no | +| logging\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | | logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | `string` | `"logging.googleapis.com/kubernetes"` | no | | maintenance\_end\_time | Time window specified for recurring maintenance operations in RFC3339 format | `string` | `""` | no | | maintenance\_exclusions | List of maintenance exclusions. A cluster can have up to three | `list(object({ name = string, start_time = string, end_time = string, exclusion_scope = string }))` | `[]` | no | | maintenance\_recurrence | Frequency of the recurring maintenance window in RFC5545 format. | `string` | `""` | no | | maintenance\_start\_time | Time window specified for daily or recurring maintenance operations in RFC3339 format | `string` | `"05:00"` | no | | master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | +| monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `false` | no | +| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration. | `list(string)` | `[]` | no | | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | | name | The name of the cluster (required) | `string` | n/a | yes | | network | The VPC network to host the cluster in (required) | `string` | n/a | yes | @@ -181,8 +188,10 @@ Then perform the following commands on the root folder: | node\_metadata | Specifies how node metadata is exposed to the workload running on the node | `string` | `"GKE_METADATA"` | no | | node\_pools | List of maps containing node pools | `list(map(any))` | 
[
  {
    "name": "default-node-pool"
  }
]
| no | | node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_linux\_node\_configs\_sysctls | Map of maps containing linux node config sysctls by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | | node\_pools\_metadata | Map of maps containing node metadata by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | | node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | `map(list(string))` | 
{
  "all": [
    "https://www.googleapis.com/auth/cloud-platform"
  ],
  "default-node-pool": []
}
| no | +| node\_pools\_resource\_labels | Map of maps containing resource labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | | node\_pools\_tags | Map of lists containing node network tags by node-pool name | `map(list(string))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | | node\_pools\_taints | Map of lists containing node taints by node-pool name | `map(list(object({ key = string, value = string, effect = string })))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | | non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | `list(string)` | 
[
  "10.0.0.0/8",
  "172.16.0.0/12",
  "192.168.0.0/16"
]
| no | @@ -196,6 +205,7 @@ Then perform the following commands on the root folder: | resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. | `string` | `""` | no | | service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | +| shadow\_firewall\_rules\_log\_config | The log\_config for shadow firewall rules. You can set this variable to `null` to disable logging. | 
object({
    metadata = string
  })
| 
{
  "metadata": "INCLUDE_ALL_METADATA"
}
| no | | shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | | skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | `bool` | `false` | no | | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | @@ -212,6 +222,7 @@ Then perform the following commands on the root folder: | ca\_certificate | Cluster ca certificate (base64 encoded) | | cluster\_id | Cluster ID | | endpoint | Cluster endpoint | +| gateway\_api\_channel | The gateway api channel of this cluster. | | horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | | http\_load\_balancing\_enabled | Whether http load balancing enabled | | identity\_namespace | Workload Identity pool | @@ -278,6 +289,7 @@ The node_pools variable takes the following parameters: | tags | The list of instance tags applied to all nodes | | Required | | value | The value for the taint | | Required | | version | The Kubernetes version for the nodes in this pool. Should only be set if auto_upgrade is false | " " | Optional | +| location_policy | [Location policy](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_node_pool#location_policy) specifies the algorithm used when scaling-up the node pool. Location policy is supported only in 1.24.1+ clusters. | " " | Optional | ## windows_node_pools variable The windows_node_pools variable takes the same parameters as [node_pools](#node\_pools-variable) but is reserved for provisioning Windows based node pools only. This variable is introduced to satisfy a [specific requirement](https://cloud.google.com/kubernetes-engine/docs/how-to/creating-a-cluster-windows#create_a_cluster_and_node_pools) for the presence of at least one linux based node pool in the cluster before a windows based node pool can be created. @@ -298,8 +310,8 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog #### Kubectl - [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x #### Terraform and Plugins -- [Terraform](https://www.terraform.io/downloads.html) 0.12 -- [Terraform Provider for GCP][terraform-provider-google] v3.41 +- [Terraform](https://www.terraform.io/downloads.html) 0.13+ +- [Terraform Provider for GCP][terraform-provider-google] v4.51 #### gcloud Some submodules use the [terraform-google-gcloud](https://github.com/terraform-google-modules/terraform-google-gcloud) module. By default, this module assumes you already have gcloud installed in your $PATH. See the [module](https://github.com/terraform-google-modules/terraform-google-gcloud#downloading) documentation for more information. diff --git a/autogen/main/README.md b/autogen/main/README.md index 8a68f4d3b5..40182f7c34 100644 --- a/autogen/main/README.md +++ b/autogen/main/README.md @@ -218,7 +218,7 @@ The node_pools variable takes the following parameters: | name | The name of the node pool | | Required | {% if beta_cluster %} | placement_policy | Placement type to set for nodes in a node pool. Can be set as [COMPACT](https://cloud.google.com/kubernetes-engine/docs/how-to/compact-placement#overview) if desired | Optional | -| pod_range | The ID of the secondary range for pod IPs. | | Optional | +| pod_range | The name of the secondary range for pod IPs. | | Optional | {% endif %} | node_count | The number of nodes in the nodepool when autoscaling is false. Otherwise defaults to 1. Only valid for non-autoscaling clusters | | Required | | node_locations | The list of zones in which the cluster's nodes are located. Nodes must be in the region of their regional cluster or in the same region as their cluster's zone for zonal clusters. Defaults to cluster level node locations if nothing is specified | " " | Optional | @@ -232,6 +232,7 @@ The node_pools variable takes the following parameters: | tags | The list of instance tags applied to all nodes | | Required | | value | The value for the taint | | Required | | version | The Kubernetes version for the nodes in this pool. Should only be set if auto_upgrade is false | " " | Optional | +| location_policy | [Location policy](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_node_pool#location_policy) specifies the algorithm used when scaling-up the node pool. Location policy is supported only in 1.24.1+ clusters. | " " | Optional | ## windows_node_pools variable The windows_node_pools variable takes the same parameters as [node_pools](#node\_pools-variable) but is reserved for provisioning Windows based node pools only. This variable is introduced to satisfy a [specific requirement](https://cloud.google.com/kubernetes-engine/docs/how-to/creating-a-cluster-windows#create_a_cluster_and_node_pools) for the presence of at least one linux based node pool in the cluster before a windows based node pool can be created. @@ -253,11 +254,11 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog #### Kubectl - [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x #### Terraform and Plugins -- [Terraform](https://www.terraform.io/downloads.html) 0.12 +- [Terraform](https://www.terraform.io/downloads.html) 0.13+ {% if beta_cluster %} -- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v3.41 +- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v4.51 {% else %} -- [Terraform Provider for GCP][terraform-provider-google] v3.41 +- [Terraform Provider for GCP][terraform-provider-google] v4.51 {% endif %} #### gcloud Some submodules use the [terraform-google-gcloud](https://github.com/terraform-google-modules/terraform-google-gcloud) module. By default, this module assumes you already have gcloud installed in your $PATH. diff --git a/autogen/main/cluster.tf.tmpl b/autogen/main/cluster.tf.tmpl index 3abf6b0f80..5d2b3cc9ff 100644 --- a/autogen/main/cluster.tf.tmpl +++ b/autogen/main/cluster.tf.tmpl @@ -53,6 +53,15 @@ resource "google_container_cluster" "primary" { channel = release_channel.value.channel } } + + dynamic "gateway_api_config" { + for_each = local.gateway_api_config + + content { + channel = gateway_api_config.value.channel + } + } + dynamic "cost_management_config" { for_each = var.enable_cost_allocation ? [1] : [] content { @@ -83,8 +92,14 @@ resource "google_container_cluster" "primary" { type = var.cluster_telemetry_type } } +{% endif %} +{% if autopilot_cluster != true %} # only one of logging/monitoring_service or logging/monitoring_config can be specified - logging_service = local.cluster_telemetry_type_is_set || local.logmon_config_is_set ? null : var.logging_service + {% if beta_cluster %} + logging_service = local.cluster_telemetry_type_is_set || local.logmon_config_is_set ? null : var.logging_service + {% else %} + logging_service = local.logmon_config_is_set ? null : var.logging_service + {% endif %} dynamic "logging_config" { for_each = length(var.logging_enabled_components) > 0 ? [1] : [] @@ -92,12 +107,16 @@ resource "google_container_cluster" "primary" { enable_components = var.logging_enabled_components } } + {% if beta_cluster %} monitoring_service = local.cluster_telemetry_type_is_set || local.logmon_config_is_set ? null : var.monitoring_service + {% else %} + monitoring_service = local.logmon_config_is_set ? null : var.monitoring_service + {% endif %} dynamic "monitoring_config" { for_each = length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus ? [1] : [] content { - enable_components = length(var.monitoring_enabled_components) > 0 ? var.monitoring_enabled_components : null + enable_components = length(var.monitoring_enabled_components) > 0 ? var.monitoring_enabled_components : [] dynamic "managed_prometheus" { for_each = var.monitoring_enable_managed_prometheus ? [1] : [] @@ -108,22 +127,6 @@ resource "google_container_cluster" "primary" { } } } -{% else %} - logging_service = var.logging_service - monitoring_service = var.monitoring_service - {% if beta_cluster %} - dynamic "monitoring_config" { - for_each = var.monitoring_enable_managed_prometheus ? [1] : [] - - content { - managed_prometheus { - enabled = var.monitoring_enable_managed_prometheus - } - } - } - {% endif %} -{% endif %} - {% if autopilot_cluster != true %} cluster_autoscaling { enabled = var.cluster_autoscaling.enabled dynamic "auto_provisioning_defaults" { @@ -132,6 +135,12 @@ resource "google_container_cluster" "primary" { content { service_account = local.service_account oauth_scopes = local.node_pools_oauth_scopes["all"] + + management { + auto_repair = lookup(var.cluster_autoscaling, "auto_repair", true) + auto_upgrade = lookup(var.cluster_autoscaling, "auto_upgrade",true) + } + {% if beta_cluster %} min_cpu_platform = lookup(var.node_pools[0], "min_cpu_platform", "") {% endif %} @@ -149,6 +158,17 @@ resource "google_container_cluster" "primary" { } } } +{% endif %} + {% if autopilot_cluster == true %} + cluster_autoscaling { + dynamic "auto_provisioning_defaults" { + for_each = (var.create_service_account || var.service_account != "") ? [1] : [] + + content { + service_account = local.service_account + } + } + } {% endif %} vertical_pod_autoscaling { enabled = var.enable_vertical_pod_autoscaling @@ -164,9 +184,10 @@ resource "google_container_cluster" "primary" { } } + enable_kubernetes_alpha = var.enable_kubernetes_alpha + {% if beta_cluster %} enable_intranode_visibility = var.enable_intranode_visibility - enable_kubernetes_alpha = var.enable_kubernetes_alpha enable_tpu = var.enable_tpu dynamic "pod_security_policy_config" { @@ -224,7 +245,6 @@ resource "google_container_cluster" "primary" { disabled = !var.horizontal_pod_autoscaling } - {% if autopilot_cluster != true %} network_policy_config { disabled = !var.network_policy @@ -237,6 +257,22 @@ resource "google_container_cluster" "primary" { gcp_filestore_csi_driver_config { enabled = var.filestore_csi_driver } + + dynamic "gce_persistent_disk_csi_driver_config" { + for_each = local.cluster_gce_pd_csi_config + + content { + enabled = gce_persistent_disk_csi_driver_config.value.enabled + } + } + + dynamic "gke_backup_agent_config" { + for_each = local.gke_backup_agent_config + + content { + enabled = gke_backup_agent_config.value.enabled + } + } {% endif %} {% if beta_cluster and autopilot_cluster != true %} @@ -253,14 +289,6 @@ resource "google_container_cluster" "primary" { } } - dynamic "gce_persistent_disk_csi_driver_config" { - for_each = local.cluster_gce_pd_csi_config - - content { - enabled = gce_persistent_disk_csi_driver_config.value.enabled - } - } - kalm_config { enabled = var.kalm_config } @@ -268,18 +296,12 @@ resource "google_container_cluster" "primary" { config_connector_config { enabled = var.config_connector } - - dynamic "gke_backup_agent_config" { - for_each = local.gke_backup_agent_config - - content { - enabled = gke_backup_agent_config.value.enabled - } - } {% endif %} } - + {% if autopilot_cluster != true %} + datapath_provider = var.datapath_provider + {% endif %} {% if beta_cluster %} networking_mode = "VPC_NATIVE" @@ -649,7 +671,10 @@ resource "google_container_node_pool" "windows_pools" { dynamic "network_config" { for_each = length(lookup(each.value, "pod_range", "")) > 0 ? [each.value] : [] content { - pod_range = lookup(network_config.value, "pod_range", null) + pod_range = lookup(network_config.value, "pod_range", null) + {% if private_cluster %} + enable_private_nodes = var.enable_private_nodes + {% endif %} } } {% endif %} @@ -686,6 +711,10 @@ resource "google_container_node_pool" "windows_pools" { local.node_pools_labels["all"], local.node_pools_labels[each.value["name"]], ) + resource_labels = merge( + local.node_pools_resource_labels["all"], + local.node_pools_resource_labels[each.value["name"]], + ) metadata = merge( lookup(lookup(local.node_pools_metadata, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_metadata, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, @@ -776,6 +805,7 @@ resource "google_container_node_pool" "windows_pools" { cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null) } } + {% endif %} {% if i == 0 %} dynamic "linux_node_config" { @@ -792,7 +822,6 @@ resource "google_container_node_pool" "windows_pools" { } } {% endif %} - {% endif %} boot_disk_kms_key = lookup(each.value, "boot_disk_kms_key", "") diff --git a/autogen/main/firewall.tf.tmpl b/autogen/main/firewall.tf.tmpl index aa0f413166..64d4df3bda 100644 --- a/autogen/main/firewall.tf.tmpl +++ b/autogen/main/firewall.tf.tmpl @@ -34,11 +34,12 @@ resource "google_compute_firewall" "intra_egress" { direction = "EGRESS" target_tags = [local.cluster_network_tag] - destination_ranges = [ + destination_ranges = concat([ local.cluster_endpoint_for_nodes, local.cluster_subnet_cidr, - local.cluster_alias_ranges_cidr[var.ip_range_pods], - ] + ], + local.pod_all_ip_ranges + ) # Allow all possible protocols allow { protocol = "tcp" } @@ -143,7 +144,7 @@ resource "google_compute_firewall" "shadow_allow_pods" { priority = var.shadow_firewall_rules_priority direction = "INGRESS" - source_ranges = [local.cluster_alias_ranges_cidr[var.ip_range_pods]] + source_ranges = local.pod_all_ip_ranges target_tags = [local.cluster_network_tag] # Allow all possible protocols @@ -154,8 +155,11 @@ resource "google_compute_firewall" "shadow_allow_pods" { allow { protocol = "esp" } allow { protocol = "ah" } - log_config { - metadata = "INCLUDE_ALL_METADATA" + dynamic "log_config" { + for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config] + content { + metadata = log_config.value.metadata + } } } @@ -177,8 +181,11 @@ resource "google_compute_firewall" "shadow_allow_master" { ports = ["10250", "443"] } - log_config { - metadata = "INCLUDE_ALL_METADATA" + dynamic "log_config" { + for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config] + content { + metadata = log_config.value.metadata + } } } @@ -209,7 +216,63 @@ resource "google_compute_firewall" "shadow_allow_nodes" { ports = ["1-65535"] } - log_config { - metadata = "INCLUDE_ALL_METADATA" + dynamic "log_config" { + for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config] + content { + metadata = log_config.value.metadata + } + } +} + +resource "google_compute_firewall" "shadow_allow_inkubelet" { + count = var.add_shadow_firewall_rules ? 1 : 0 + + name = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-inkubelet" + description = "Managed by terraform GKE module: A shadow firewall rule to match the default rule allowing worker nodes & pods communication to kubelet." + project = local.network_project_id + network = var.network + priority = var.shadow_firewall_rules_priority - 1 # rule created by GKE robot have prio 999 + direction = "INGRESS" + + source_ranges = local.pod_all_ip_ranges + source_tags = [local.cluster_network_tag] + target_tags = [local.cluster_network_tag] + + allow { + protocol = "tcp" + ports = ["10255"] + } + + dynamic "log_config" { + for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config] + content { + metadata = log_config.value.metadata + } + } +} + +resource "google_compute_firewall" "shadow_deny_exkubelet" { + count = var.add_shadow_firewall_rules ? 1 : 0 + + name = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-exkubelet" + description = "Managed by terraform GKE module: A shadow firewall rule to match the default deny rule to kubelet." + project = local.network_project_id + network = var.network + priority = var.shadow_firewall_rules_priority # rule created by GKE robot have prio 1000 + direction = "INGRESS" + + source_ranges = ["0.0.0.0/0"] + target_tags = [local.cluster_network_tag] + + deny { + protocol = "tcp" + ports = ["10255"] + } + + dynamic "log_config" { + for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config] + content { + metadata = log_config.value.metadata + } } } diff --git a/autogen/main/main.tf.tmpl b/autogen/main/main.tf.tmpl index 77239915b3..5f7b1df381 100644 --- a/autogen/main/main.tf.tmpl +++ b/autogen/main/main.tf.tmpl @@ -57,6 +57,7 @@ locals { {% endif %} release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] + gateway_api_config = var.gateway_api_channel != null ? [{ channel : var.gateway_api_channel }] : [] {% if autopilot_cluster != true %} autoscaling_resource_limits = var.cluster_autoscaling.enabled ? concat([{ @@ -67,7 +68,7 @@ locals { resource_type = "memory" minimum = var.cluster_autoscaling.min_memory_gb maximum = var.cluster_autoscaling.max_memory_gb - }], var.cluster_autoscaling.gpu_resources) : [] + }], var.cluster_autoscaling.gpu_resources) : [] {% endif %} @@ -86,6 +87,11 @@ locals { cluster_subnet_cidr = var.add_cluster_firewall_rules ? data.google_compute_subnetwork.gke_subnetwork[0].ip_cidr_range : null cluster_alias_ranges_cidr = var.add_cluster_firewall_rules ? { for range in toset(data.google_compute_subnetwork.gke_subnetwork[0].secondary_ip_range) : range.range_name => range.ip_cidr_range } : {} +{% if autopilot_cluster != true %} + pod_all_ip_ranges = var.add_cluster_firewall_rules ? compact(concat([local.cluster_alias_ranges_cidr[var.ip_range_pods]], [for k, v in merge(local.node_pools, local.windows_node_pools): local.cluster_alias_ranges_cidr[v.pod_range] if length(lookup(v, "pod_range", "")) > 0] )) : [] +{% else %} + pod_all_ip_ranges = var.add_cluster_firewall_rules ? [local.cluster_alias_ranges_cidr[var.ip_range_pods]] : [] +{% endif %} {% if autopilot_cluster != true %} cluster_network_policy = var.network_policy ? [{ @@ -95,6 +101,9 @@ locals { enabled = false provider = null }] + cluster_gce_pd_csi_config = var.gce_pd_csi_driver ? [{ enabled = true }] : [{ enabled = false }] + logmon_config_is_set = length(var.logging_enabled_components) > 0 || length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus + gke_backup_agent_config = var.gke_backup_agent_config ? [{ enabled = true }] : [{ enabled = false }] {% endif %} {% if beta_cluster and autopilot_cluster != true %} cluster_cloudrun_config_load_balancer_config = (var.cloudrun && var.cloudrun_load_balancer_type != "") ? { @@ -109,9 +118,6 @@ locals { ) ] : [] cluster_cloudrun_enabled = var.cloudrun - cluster_gce_pd_csi_config = var.gce_pd_csi_driver ? [{ enabled = true }] : [{ enabled = false }] - gke_backup_agent_config = var.gke_backup_agent_config ? [{ enabled = true }] : [{ enabled = false }] - logmon_config_is_set = length(var.logging_enabled_components) > 0 || length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus {% endif %} cluster_authenticator_security_group = var.authenticator_security_group == null ? [] : [{ diff --git a/autogen/main/outputs.tf.tmpl b/autogen/main/outputs.tf.tmpl index 9c002d0fdf..d22e8bd087 100644 --- a/autogen/main/outputs.tf.tmpl +++ b/autogen/main/outputs.tf.tmpl @@ -158,6 +158,11 @@ output "release_channel" { value = var.release_channel } +output "gateway_api_channel" { + description = "The gateway api channel of this cluster." + value = var.gateway_api_channel +} + output "identity_namespace" { description = "Workload Identity pool" value = length(local.cluster_workload_identity_config) > 0 ? local.cluster_workload_identity_config[0].workload_pool : null diff --git a/autogen/main/variables.tf.tmpl b/autogen/main/variables.tf.tmpl index 554bbd87ac..188f90c356 100644 --- a/autogen/main/variables.tf.tmpl +++ b/autogen/main/variables.tf.tmpl @@ -171,6 +171,16 @@ variable "node_pools_labels" { } } +variable "node_pools_resource_labels" { + type = map(map(string)) + description = "Map of maps containing resource labels by node-pool name" + + default = { + all = {} + default-node-pool = {} + } +} + variable "node_pools_metadata" { type = map(map(string)) description = "Map of maps containing node metadata by node-pool name" @@ -181,7 +191,6 @@ variable "node_pools_metadata" { default-node-pool = {} } } -{% if beta_cluster %} variable "node_pools_linux_node_configs_sysctls" { type = map(map(string)) @@ -194,7 +203,6 @@ variable "node_pools_linux_node_configs_sysctls" { } } {% endif %} -{% endif %} variable "enable_cost_allocation" { type = bool @@ -231,6 +239,8 @@ variable "cluster_autoscaling" { min_memory_gb = number max_memory_gb = number gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) + auto_repair = bool + auto_upgrade = bool }) default = { enabled = false @@ -242,6 +252,8 @@ variable "cluster_autoscaling" { max_memory_gb = 0 min_memory_gb = 0 gpu_resources = [] + auto_repair = true + auto_upgrade = true } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } @@ -444,6 +456,12 @@ variable "release_channel" { default = null } +variable "gateway_api_channel" { + type = string + description = "The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`." + default = null +} + variable "add_cluster_firewall_rules" { type = bool description = "Create additional firewall rules" @@ -475,9 +493,23 @@ variable "add_shadow_firewall_rules" { } variable "shadow_firewall_rules_priority" { - type = number + type = number description = "The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000." - default = 999 + default = 999 + validation { + condition = var.shadow_firewall_rules_priority < 1000 + error_message = "The shadow firewall rule priority must be lower than auto-created one(1000)." + } +} + +variable "shadow_firewall_rules_log_config" { + type = object({ + metadata = string + }) + description = "The log_config for shadow firewall rules. You can set this variable to `null` to disable logging." + default = { + metadata = "INCLUDE_ALL_METADATA" + } } {% if beta_cluster %} @@ -581,7 +613,7 @@ variable "node_metadata" { validation { condition = contains(["GKE_METADATA", "GCE_METADATA", "UNSPECIFIED", "GKE_METADATA_SERVER", "EXPOSE"], var.node_metadata) - error_message = "The node_metadata value must be one of GKE_METADATA, GCE_METADATA or UNSPECIFIED." + error_message = "The node_metadata value must be one of GKE_METADATA, GCE_METADATA, UNSPECIFIED, GKE_METADATA_SERVER or EXPOSE." } } {% endif %} @@ -605,6 +637,18 @@ variable "cluster_dns_domain" { default = "" } +variable "gce_pd_csi_driver" { + type = bool + description = "Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver." + default = true +} + +variable "gke_backup_agent_config" { + type = bool + description = "Whether Backup for GKE agent is enabled for this cluster." + default = false +} + {% endif %} variable "timeouts" { type = map(string) @@ -615,33 +659,33 @@ variable "timeouts" { error_message = "Only create, update, delete timeouts can be specified." } } -{% if beta_cluster %} - {% if autopilot_cluster != true %} -variable "enable_kubernetes_alpha" { +{% if autopilot_cluster != true %} +variable "monitoring_enable_managed_prometheus" { type = bool - description = "Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days." + description = "Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled." default = false } -variable "logging_enabled_components" { +variable "monitoring_enabled_components" { type = list(string) - description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." + description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration." default = [] } -variable "monitoring_enabled_components" { +variable "logging_enabled_components" { type = list(string) - description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration." + description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." default = [] } - {% endif %} -variable "monitoring_enable_managed_prometheus" { +variable "enable_kubernetes_alpha" { type = bool - description = "(Beta) Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled." + description = "Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days." default = false } +{% endif %} +{% if beta_cluster %} {% if autopilot_cluster != true %} variable "istio" { @@ -667,12 +711,6 @@ variable "config_connector" { default = false } -variable "gke_backup_agent_config" { - type = bool - description = "(Beta) Whether Backup for GKE agent is enabled for this cluster." - default = false -} - variable "cloudrun" { description = "(Beta) Enable CloudRun addon" default = false @@ -713,11 +751,5 @@ variable "enable_identity_service" { description = "Enable the Identity Service component, which allows customers to use external identity providers with the K8S API." default = false } - -variable "gce_pd_csi_driver" { - type = bool - description = "(Beta) Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver." - default = false -} {% endif %} {% endif %} diff --git a/autogen/main/variables_defaults.tf.tmpl b/autogen/main/variables_defaults.tf.tmpl index 1706713eb9..3aa635a362 100644 --- a/autogen/main/variables_defaults.tf.tmpl +++ b/autogen/main/variables_defaults.tf.tmpl @@ -35,6 +35,20 @@ locals { var.node_pools_labels ) + node_pools_resource_labels = merge( + { all = {} }, + { default-node-pool = {} }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : {}] + ), + zipmap( + [for node_pool in var.windows_node_pools : node_pool["name"]], + [for node_pool in var.windows_node_pools : {}] + ), + var.node_pools_resource_labels + ) + node_pools_metadata = merge( { all = {} }, { default-node-pool = {} }, @@ -90,7 +104,6 @@ locals { ), var.node_pools_oauth_scopes ) - {% if beta_cluster %} node_pools_linux_node_configs_sysctls = merge( { all = {} }, @@ -101,6 +114,5 @@ locals { ), var.node_pools_linux_node_configs_sysctls ) - {% endif %} } {% endif %} diff --git a/autogen/main/versions.tf.tmpl b/autogen/main/versions.tf.tmpl index a85658bdee..2f3b3861d2 100644 --- a/autogen/main/versions.tf.tmpl +++ b/autogen/main/versions.tf.tmpl @@ -24,7 +24,7 @@ terraform { required_providers { google-beta = { source = "hashicorp/google-beta" - version = ">= 4.42.0, < 5.0" + version = ">= 4.51.0, < 5.0" } kubernetes = { source = "hashicorp/kubernetes" @@ -32,13 +32,13 @@ terraform { } } provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine{% if module_registry_name %}:{{ module_registry_name }}{% endif %}/v24.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine{% if module_registry_name %}:{{ module_registry_name }}{% endif %}/v25.0.0" } {% else %} required_providers { google = { source = "hashicorp/google" - version = ">= 4.36.0, < 5.0" + version = ">= 4.51.0, < 5.0" } kubernetes = { source = "hashicorp/kubernetes" @@ -46,7 +46,7 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine{% if module_registry_name %}:{{ module_registry_name }}{% endif %}/v24.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine{% if module_registry_name %}:{{ module_registry_name }}{% endif %}/v25.0.0" } {% endif %} } diff --git a/autogen/safer-cluster/main.tf.tmpl b/autogen/safer-cluster/main.tf.tmpl index 112c287399..eef4673c83 100644 --- a/autogen/safer-cluster/main.tf.tmpl +++ b/autogen/safer-cluster/main.tf.tmpl @@ -41,7 +41,8 @@ module "gke" { // the master upgrades. // // https://cloud.google.com/kubernetes-engine/versioning-and-upgrades - release_channel = var.release_channel + release_channel = var.release_channel + gateway_api_channel = var.gateway_api_channel master_authorized_networks = var.master_authorized_networks @@ -86,12 +87,13 @@ module "gke" { // If removing the default node pool, initial_node_count should be at least 1. initial_node_count = (var.initial_node_count == 0) ? 1 : var.initial_node_count - node_pools = var.node_pools - windows_node_pools = var.windows_node_pools - node_pools_labels = var.node_pools_labels - node_pools_metadata = var.node_pools_metadata - node_pools_taints = var.node_pools_taints - node_pools_tags = var.node_pools_tags + node_pools = var.node_pools + windows_node_pools = var.windows_node_pools + node_pools_labels = var.node_pools_labels + node_pools_resource_labels = var.node_pools_resource_labels + node_pools_metadata = var.node_pools_metadata + node_pools_taints = var.node_pools_taints + node_pools_tags = var.node_pools_tags node_pools_oauth_scopes = var.node_pools_oauth_scopes diff --git a/autogen/safer-cluster/variables.tf.tmpl b/autogen/safer-cluster/variables.tf.tmpl index 9fd29370f8..3820987fd3 100644 --- a/autogen/safer-cluster/variables.tf.tmpl +++ b/autogen/safer-cluster/variables.tf.tmpl @@ -77,6 +77,12 @@ variable "release_channel" { default = "REGULAR" } +variable "gateway_api_channel" { + type = string + description = "The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`." + default = null +} + variable "master_authorized_networks" { type = list(object({ cidr_block = string, display_name = string })) description = "List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists)." @@ -168,6 +174,16 @@ variable "node_pools_labels" { } } +variable "node_pools_resource_labels" { + type = map(map(string)) + description = "Map of maps containing resource labels by node-pool name" + + default = { + all = {} + default-node-pool = {} + } +} + variable "node_pools_metadata" { type = map(map(string)) description = "Map of maps containing node metadata by node-pool name" @@ -217,6 +233,8 @@ variable "cluster_autoscaling" { min_memory_gb = number max_memory_gb = number gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) + auto_repair = bool + auto_upgrade = bool }) default = { enabled = false @@ -226,6 +244,8 @@ variable "cluster_autoscaling" { max_memory_gb = 0 min_memory_gb = 0 gpu_resources = [] + auto_repair = true + auto_upgrade = true } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } diff --git a/autogen/safer-cluster/versions.tf.tmpl b/autogen/safer-cluster/versions.tf.tmpl index c62c293176..97b4d9e1d1 100644 --- a/autogen/safer-cluster/versions.tf.tmpl +++ b/autogen/safer-cluster/versions.tf.tmpl @@ -23,6 +23,6 @@ terraform { required_version = ">=0.13" provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine{% if module_registry_name %}:{{ module_registry_name }}{% endif %}/v24.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine{% if module_registry_name %}:{{ module_registry_name }}{% endif %}/v25.0.0" } } diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index ddeb878600..f9cc9a6f5b 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -121,6 +121,21 @@ steps: - verify simple-regional-with-kubeconfig-local name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy simple-regional-with-kubeconfig-local'] +- id: converge simple-regional-with-gateway-api-local + waitFor: + - create all + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge simple-regional-with-gateway-api-local'] +- id: verify simple-regional-with-gateway-api-local + waitFor: + - converge simple-regional-with-gateway-api-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify simple-regional-with-gateway-api-local'] +- id: destroy simple-regional-with-gateway-api-local + waitFor: + - verify simple-regional-with-gateway-api-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy simple-regional-with-gateway-api-local'] - id: converge simple-regional-with-networking-local waitFor: - create all @@ -136,22 +151,21 @@ steps: - verify simple-regional-with-networking-local name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy simple-regional-with-networking-local'] -# TODO(bharathkkb): https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1431 -# - id: converge simple-zonal-local -# waitFor: -# - create all -# name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' -# args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge simple-zonal-local'] -# - id: verify simple-zonal-local -# waitFor: -# - converge simple-zonal-local -# name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' -# args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify simple-zonal-local'] -# - id: destroy simple-zonal-local -# waitFor: -# - verify simple-zonal-local -# name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' -# args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy simple-zonal-local'] +- id: converge simple-zonal-local + waitFor: + - create all + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge simple-zonal-local'] +- id: verify simple-zonal-local + waitFor: + - converge simple-zonal-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify simple-zonal-local'] +- id: destroy simple-zonal-local + waitFor: + - verify simple-zonal-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy simple-zonal-local'] - id: converge simple-zonal-private-local waitFor: - create all @@ -402,9 +416,26 @@ steps: - verify private-zonal-with-networking name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'cft test run TestPrivateZonalWithNetworking --stage teardown --verbose --test-dir test/integration'] - - - +- id: init simple-autopilot-private-non-default-sa + waitFor: + - prepare + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestSimpleAutopilotPrivateNonDefaultSA --stage init --verbose'] +- id: apply simple-autopilot-private-non-default-sa + waitFor: + - init simple-autopilot-private-non-default-sa + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestSimpleAutopilotPrivateNonDefaultSA --stage apply --verbose'] +- id: verify simple-autopilot-private-non-default-sa + waitFor: + - apply simple-autopilot-private-non-default-sa + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestSimpleAutopilotPrivateNonDefaultSA --stage verify --verbose'] +- id: teardown simple-autopilot-private-non-default-sa + waitFor: + - verify simple-autopilot-private-non-default-sa + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestSimpleAutopilotPrivateNonDefaultSA --stage teardown --verbose'] tags: - 'ci' - 'integration' diff --git a/cluster.tf b/cluster.tf index 8b9e802489..49520613ee 100644 --- a/cluster.tf +++ b/cluster.tf @@ -47,6 +47,15 @@ resource "google_container_cluster" "primary" { channel = release_channel.value.channel } } + + dynamic "gateway_api_config" { + for_each = local.gateway_api_config + + content { + channel = gateway_api_config.value.channel + } + } + dynamic "cost_management_config" { for_each = var.enable_cost_allocation ? [1] : [] content { @@ -62,8 +71,31 @@ resource "google_container_cluster" "primary" { min_master_version = var.release_channel == null || var.release_channel == "UNSPECIFIED" ? local.master_version : null - logging_service = var.logging_service - monitoring_service = var.monitoring_service + # only one of logging/monitoring_service or logging/monitoring_config can be specified + logging_service = local.logmon_config_is_set ? null : var.logging_service + dynamic "logging_config" { + for_each = length(var.logging_enabled_components) > 0 ? [1] : [] + + content { + enable_components = var.logging_enabled_components + } + } + monitoring_service = local.logmon_config_is_set ? null : var.monitoring_service + dynamic "monitoring_config" { + for_each = length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus ? [1] : [] + + content { + enable_components = length(var.monitoring_enabled_components) > 0 ? var.monitoring_enabled_components : [] + + dynamic "managed_prometheus" { + for_each = var.monitoring_enable_managed_prometheus ? [1] : [] + + content { + enabled = var.monitoring_enable_managed_prometheus + } + } + } + } cluster_autoscaling { enabled = var.cluster_autoscaling.enabled dynamic "auto_provisioning_defaults" { @@ -72,6 +104,12 @@ resource "google_container_cluster" "primary" { content { service_account = local.service_account oauth_scopes = local.node_pools_oauth_scopes["all"] + + management { + auto_repair = lookup(var.cluster_autoscaling, "auto_repair", true) + auto_upgrade = lookup(var.cluster_autoscaling, "auto_upgrade", true) + } + } } dynamic "resource_limits" { @@ -96,6 +134,8 @@ resource "google_container_cluster" "primary" { } } + enable_kubernetes_alpha = var.enable_kubernetes_alpha + dynamic "master_authorized_networks_config" { for_each = local.master_authorized_networks_config content { @@ -131,7 +171,6 @@ resource "google_container_cluster" "primary" { disabled = !var.horizontal_pod_autoscaling } - network_policy_config { disabled = !var.network_policy } @@ -143,6 +182,22 @@ resource "google_container_cluster" "primary" { gcp_filestore_csi_driver_config { enabled = var.filestore_csi_driver } + + dynamic "gce_persistent_disk_csi_driver_config" { + for_each = local.cluster_gce_pd_csi_config + + content { + enabled = gce_persistent_disk_csi_driver_config.value.enabled + } + } + + dynamic "gke_backup_agent_config" { + for_each = local.gke_backup_agent_config + + content { + enabled = gke_backup_agent_config.value.enabled + } + } } datapath_provider = var.datapath_provider @@ -377,6 +432,10 @@ resource "google_container_node_pool" "pools" { local.node_pools_labels["all"], local.node_pools_labels[each.value["name"]], ) + resource_labels = merge( + local.node_pools_resource_labels["all"], + local.node_pools_resource_labels[each.value["name"]], + ) metadata = merge( lookup(lookup(local.node_pools_metadata, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_metadata, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, @@ -440,6 +499,20 @@ resource "google_container_node_pool" "pools" { } + dynamic "linux_node_config" { + for_each = length(merge( + local.node_pools_linux_node_configs_sysctls["all"], + local.node_pools_linux_node_configs_sysctls[each.value["name"]] + )) != 0 ? [1] : [] + + content { + sysctls = merge( + local.node_pools_linux_node_configs_sysctls["all"], + local.node_pools_linux_node_configs_sysctls[each.value["name"]] + ) + } + } + boot_disk_kms_key = lookup(each.value, "boot_disk_kms_key", "") shielded_instance_config { @@ -531,6 +604,10 @@ resource "google_container_node_pool" "windows_pools" { local.node_pools_labels["all"], local.node_pools_labels[each.value["name"]], ) + resource_labels = merge( + local.node_pools_resource_labels["all"], + local.node_pools_resource_labels[each.value["name"]], + ) metadata = merge( lookup(lookup(local.node_pools_metadata, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_metadata, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, @@ -594,6 +671,7 @@ resource "google_container_node_pool" "windows_pools" { } + boot_disk_kms_key = lookup(each.value, "boot_disk_kms_key", "") shielded_instance_config { diff --git a/docs/private_clusters.md b/docs/private_clusters.md index 24a995d1b0..2f3b57fbbb 100644 --- a/docs/private_clusters.md +++ b/docs/private_clusters.md @@ -20,6 +20,16 @@ If you are using these features with a private cluster, you will need to either: If you are going to isolate your GKE private clusters from internet access you could check [this guide](https://medium.com/google-cloud/completely-private-gke-clusters-with-no-internet-connectivity-945fffae1ccd) and the associated [repo](https://github.com/andreyk-code/no-inet-gke-cluster). +## Discontiguous multi-Pod CIDR +If you are going to use [discontiguous multi-Pod CIDR](https://cloud.google.com/kubernetes-engine/docs/how-to/multi-pod-cidr) it can happen that GKE robot will not update `gke-[cluster-name]-[cluster-hash]-all` and other firewall rules automatically when you add a new node pool (as stated in [documentation](https://cloud.google.com/kubernetes-engine/docs/how-to/multi-pod-cidr#modified_firewall_rule)). You can prevent this from happening, by using a workaround with shadow firewall rules: +``` +module "gke" { + ... + add_shadow_firewall_rules = true + shadow_firewall_rules_log_config = null # to save some $ on logs +} +``` + ## Troubleshooting ### Master Authorized Network diff --git a/docs/upgrading_to_v25.0.md b/docs/upgrading_to_v25.0.md new file mode 100644 index 0000000000..0d25eac759 --- /dev/null +++ b/docs/upgrading_to_v25.0.md @@ -0,0 +1,42 @@ +# Upgrading to v25.0 +The v25.0 release of *kubernetes-engine* is a backwards incompatible +release. + +### gce_pd_csi_driver is GA and enabled by default + +`gce_pd_csi_driver` is now supported in GA modules and defaults to true. To opt out, set `gce_pd_csi_driver` to `false`. + +```diff + module "gke" { +- source = "terraform-google-modules/kubernetes-engine" +- version = "~> 24.0" ++ source = "terraform-google-modules/kubernetes-engine" ++ version = "~> 25.0" +... ++ gce_pd_csi_driver = false +} +``` + +### Use the created service account when creating autopilot clusters + +When `create_service_account` is `true` pass the created service account to the `cluster_autoscaling` -> `auto_provisioning_defaults` block +for the `beta-autopilot-private-cluster` / `beta-autopilot-public-cluster` modules. + +This will mean that the `Nodes` will use the created service account, where previously the default service account was erronously used instead. + +To opt out, set `create_service_account` to `false` + +```diff + module "gke" { +- source = "terraform-google-modules/kubernetes-engine" +- version = "~> 24.0" ++ source = "terraform-google-modules/kubernetes-engine" ++ version = "~> 25.0" +... ++ create_service_account = false +} +``` + +### Minimum Google Provider versions + +Minimum Google Provider versions have been updated to `4.51.0`. diff --git a/examples/deploy_service/main.tf b/examples/deploy_service/main.tf index cf86b6969a..f32ced721d 100644 --- a/examples/deploy_service/main.tf +++ b/examples/deploy_service/main.tf @@ -52,7 +52,7 @@ resource "kubernetes_pod" "nginx-example" { spec { container { - image = "nginx:1.23.2" + image = "nginx:1.23.3" name = "nginx-example" } } diff --git a/examples/node_pool/README.md b/examples/node_pool/README.md index 48d27a1e9c..b032cd2fa5 100644 --- a/examples/node_pool/README.md +++ b/examples/node_pool/README.md @@ -7,7 +7,7 @@ This example illustrates how to create a cluster with multiple custom node-pool | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources = list(object({
      resource_type = string
      minimum       = number
      maximum       = number
    }))
  })
| 
{
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources = list(object({
      resource_type = string
      minimum       = number
      maximum       = number
    }))
    auto_repair  = bool
    auto_upgrade = bool
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | | cluster\_name\_suffix | A suffix to append to the default cluster name | `string` | `""` | no | | compute\_engine\_service\_account | Service account to associate to the nodes in the cluster | `any` | n/a | yes | | ip\_range\_pods | The secondary ip range to use for pods | `any` | n/a | yes | diff --git a/examples/node_pool/variables.tf b/examples/node_pool/variables.tf index ac76aa1a05..616acc56e8 100644 --- a/examples/node_pool/variables.tf +++ b/examples/node_pool/variables.tf @@ -65,6 +65,8 @@ variable "cluster_autoscaling" { minimum = number maximum = number })) + auto_repair = bool + auto_upgrade = bool }) default = { enabled = false @@ -74,6 +76,8 @@ variable "cluster_autoscaling" { max_memory_gb = 0 min_memory_gb = 0 gpu_resources = [] + auto_repair = true + auto_upgrade = true } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } diff --git a/examples/simple_autopilot_private/outputs.tf b/examples/simple_autopilot_private/outputs.tf index 32f80d64c1..550609144b 100644 --- a/examples/simple_autopilot_private/outputs.tf +++ b/examples/simple_autopilot_private/outputs.tf @@ -35,6 +35,7 @@ output "master_kubernetes_version" { } output "ca_certificate" { + sensitive = true description = "The cluster ca certificate (base64 encoded)" value = module.gke.ca_certificate } diff --git a/examples/simple_autopilot_private_non_default_sa/README.md b/examples/simple_autopilot_private_non_default_sa/README.md new file mode 100644 index 0000000000..9256580e8d --- /dev/null +++ b/examples/simple_autopilot_private_non_default_sa/README.md @@ -0,0 +1,33 @@ +# Simple Regional Autopilot Cluster + +This example illustrates how to create a simple autopilot cluster with beta features and +not using the default service account. + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| project\_id | The project ID to host the cluster in | `any` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| cluster\_name | Cluster name | +| kubernetes\_endpoint | The cluster endpoint | +| location | n/a | +| master\_kubernetes\_version | Kubernetes version of the master | +| network\_name | The name of the VPC being created | +| region | The region in which the cluster resides | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| subnet\_names | The names of the subnet being created | +| zones | List of zones in which the cluster resides | + + + +To provision this example, run the following from within this directory: +- `terraform init` to get the plugins +- `terraform plan` to see the infrastructure plan +- `terraform apply` to apply the infrastructure build +- `terraform destroy` to destroy the built infrastructure diff --git a/examples/simple_autopilot_private_non_default_sa/main.tf b/examples/simple_autopilot_private_non_default_sa/main.tf new file mode 100644 index 0000000000..e8368eb02b --- /dev/null +++ b/examples/simple_autopilot_private_non_default_sa/main.tf @@ -0,0 +1,59 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + cluster_type = "simple-ap-private-non-default-sa" + network_name = "${local.cluster_type}-network" + subnet_name = "${local.cluster_type}-subnet" + master_auth_subnetwork = "${local.cluster_type}-master-subnet" + pods_range_name = "ip-range-pods-${local.cluster_type}" + svc_range_name = "ip-range-svc-${local.cluster_type}" + subnet_names = [for subnet_self_link in module.gcp-network.subnets_self_links : split("/", subnet_self_link)[length(split("/", subnet_self_link)) - 1]] +} + + +data "google_client_config" "default" {} + +provider "kubernetes" { + host = "https://${module.gke.endpoint}" + token = data.google_client_config.default.access_token + cluster_ca_certificate = base64decode(module.gke.ca_certificate) +} + +module "gke" { + source = "../../modules/beta-autopilot-private-cluster/" + project_id = var.project_id + name = "${local.cluster_type}-cluster" + regional = true + region = "us-central1" + network = module.gcp-network.network_name + subnetwork = local.subnet_names[index(module.gcp-network.subnets_names, local.subnet_name)] + ip_range_pods = local.pods_range_name + ip_range_services = local.svc_range_name + release_channel = "REGULAR" + enable_vertical_pod_autoscaling = true + enable_private_endpoint = true + enable_private_nodes = true + master_ipv4_cidr_block = "172.16.0.0/28" + datapath_provider = "ADVANCED_DATAPATH" + + master_authorized_networks = [ + { + cidr_block = "10.60.0.0/17" + display_name = "VPC" + }, + ] +} diff --git a/examples/simple_autopilot_private_non_default_sa/network.tf b/examples/simple_autopilot_private_non_default_sa/network.tf new file mode 100644 index 0000000000..9f3689370c --- /dev/null +++ b/examples/simple_autopilot_private_non_default_sa/network.tf @@ -0,0 +1,50 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "gcp-network" { + source = "terraform-google-modules/network/google" + version = ">= 4.0.1" + + project_id = var.project_id + network_name = local.network_name + + subnets = [ + { + subnet_name = local.subnet_name + subnet_ip = "10.0.0.0/17" + subnet_region = "us-central1" + subnet_private_access = true + }, + { + subnet_name = local.master_auth_subnetwork + subnet_ip = "10.60.0.0/17" + subnet_region = "us-central1" + }, + ] + + secondary_ranges = { + (local.subnet_name) = [ + { + range_name = local.pods_range_name + ip_cidr_range = "192.168.0.0/18" + }, + { + range_name = local.svc_range_name + ip_cidr_range = "192.168.64.0/18" + }, + ] + } +} diff --git a/examples/simple_autopilot_private_non_default_sa/outputs.tf b/examples/simple_autopilot_private_non_default_sa/outputs.tf new file mode 100644 index 0000000000..cfe5ee17f1 --- /dev/null +++ b/examples/simple_autopilot_private_non_default_sa/outputs.tf @@ -0,0 +1,60 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "kubernetes_endpoint" { + description = "The cluster endpoint" + sensitive = true + value = module.gke.endpoint +} + +output "cluster_name" { + description = "Cluster name" + value = module.gke.name +} + +output "location" { + value = module.gke.location +} + +output "master_kubernetes_version" { + description = "Kubernetes version of the master" + value = module.gke.master_version +} + +output "service_account" { + description = "The service account to default running nodes as if not overridden in `node_pools`." + value = module.gke.service_account +} + +output "network_name" { + description = "The name of the VPC being created" + value = module.gcp-network.network_name +} + +output "subnet_names" { + description = "The names of the subnet being created" + value = module.gcp-network.subnets_names +} + +output "region" { + description = "The region in which the cluster resides" + value = module.gke.region +} + +output "zones" { + description = "List of zones in which the cluster resides" + value = module.gke.zones +} diff --git a/examples/simple_autopilot_private_non_default_sa/variables.tf b/examples/simple_autopilot_private_non_default_sa/variables.tf new file mode 100644 index 0000000000..80f4b3cf3b --- /dev/null +++ b/examples/simple_autopilot_private_non_default_sa/variables.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "The project ID to host the cluster in" +} diff --git a/examples/simple_autopilot_private_non_default_sa/versions.tf b/examples/simple_autopilot_private_non_default_sa/versions.tf new file mode 100644 index 0000000000..210a187481 --- /dev/null +++ b/examples/simple_autopilot_private_non_default_sa/versions.tf @@ -0,0 +1,28 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = "~> 4.0" + } + kubernetes = { + source = "hashicorp/kubernetes" + } + } + required_version = ">= 0.13" +} diff --git a/examples/simple_regional_with_gateway_api/README.md b/examples/simple_regional_with_gateway_api/README.md new file mode 100644 index 0000000000..ca82e650ce --- /dev/null +++ b/examples/simple_regional_with_gateway_api/README.md @@ -0,0 +1,47 @@ +# Simple Regional Cluster + +This example illustrates how to create a simple cluster. + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| cluster\_name\_suffix | A suffix to append to the default cluster name | `string` | `""` | no | +| compute\_engine\_service\_account | Service account to associate to the nodes in the cluster | `any` | n/a | yes | +| enable\_binary\_authorization | Enable BinAuthZ Admission controller | `bool` | `false` | no | +| gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | +| ip\_range\_pods | The secondary ip range to use for pods | `any` | n/a | yes | +| ip\_range\_services | The secondary ip range to use for services | `any` | n/a | yes | +| network | The VPC network to host the cluster in | `any` | n/a | yes | +| project\_id | The project ID to host the cluster in | `any` | n/a | yes | +| region | The region to host the cluster in | `any` | n/a | yes | +| skip\_provisioners | Flag to skip local-exec provisioners | `bool` | `false` | no | +| subnetwork | The subnetwork to host the cluster in | `any` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| ca\_certificate | n/a | +| client\_token | n/a | +| cluster\_name | Cluster name | +| ip\_range\_pods | The secondary IP range used for pods | +| ip\_range\_services | The secondary IP range used for services | +| kubernetes\_endpoint | n/a | +| location | n/a | +| master\_kubernetes\_version | The master Kubernetes version | +| network | n/a | +| project\_id | n/a | +| region | n/a | +| service\_account | The default service account used for running nodes. | +| subnetwork | n/a | +| zones | List of zones in which the cluster resides | + + + +To provision this example, run the following from within this directory: +- `terraform init` to get the plugins +- `terraform plan` to see the infrastructure plan +- `terraform apply` to apply the infrastructure build +- `terraform destroy` to destroy the built infrastructure diff --git a/examples/simple_regional_with_gateway_api/main.tf b/examples/simple_regional_with_gateway_api/main.tf new file mode 100644 index 0000000000..96e9c126c2 --- /dev/null +++ b/examples/simple_regional_with_gateway_api/main.tf @@ -0,0 +1,45 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + cluster_type = "simple-regional-gatewayapi" +} + +data "google_client_config" "default" {} + +provider "kubernetes" { + host = "https://${module.gke.endpoint}" + token = data.google_client_config.default.access_token + cluster_ca_certificate = base64decode(module.gke.ca_certificate) +} + +module "gke" { + source = "../../" + project_id = var.project_id + name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" + regional = true + region = var.region + network = var.network + subnetwork = var.subnetwork + ip_range_pods = var.ip_range_pods + ip_range_services = var.ip_range_services + create_service_account = false + service_account = var.compute_engine_service_account + enable_cost_allocation = true + enable_binary_authorization = var.enable_binary_authorization + skip_provisioners = var.skip_provisioners + gateway_api_channel = var.gateway_api_channel +} diff --git a/examples/simple_regional_with_gateway_api/outputs.tf b/examples/simple_regional_with_gateway_api/outputs.tf new file mode 100644 index 0000000000..01a13147c2 --- /dev/null +++ b/examples/simple_regional_with_gateway_api/outputs.tf @@ -0,0 +1,35 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "kubernetes_endpoint" { + sensitive = true + value = module.gke.endpoint +} + +output "client_token" { + sensitive = true + value = base64encode(data.google_client_config.default.access_token) +} + +output "ca_certificate" { + value = module.gke.ca_certificate +} + +output "service_account" { + description = "The default service account used for running nodes." + value = module.gke.service_account +} + diff --git a/examples/simple_regional_with_gateway_api/test_outputs.tf b/examples/simple_regional_with_gateway_api/test_outputs.tf new file mode 120000 index 0000000000..17b34213ba --- /dev/null +++ b/examples/simple_regional_with_gateway_api/test_outputs.tf @@ -0,0 +1 @@ +../../test/fixtures/all_examples/test_outputs.tf \ No newline at end of file diff --git a/examples/simple_regional_with_gateway_api/variables.tf b/examples/simple_regional_with_gateway_api/variables.tf new file mode 100644 index 0000000000..90ba0ea0ff --- /dev/null +++ b/examples/simple_regional_with_gateway_api/variables.tf @@ -0,0 +1,65 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "The project ID to host the cluster in" +} + +variable "cluster_name_suffix" { + description = "A suffix to append to the default cluster name" + default = "" +} + +variable "region" { + description = "The region to host the cluster in" +} + +variable "network" { + description = "The VPC network to host the cluster in" +} + +variable "subnetwork" { + description = "The subnetwork to host the cluster in" +} + +variable "ip_range_pods" { + description = "The secondary ip range to use for pods" +} + +variable "ip_range_services" { + description = "The secondary ip range to use for services" +} + +variable "compute_engine_service_account" { + description = "Service account to associate to the nodes in the cluster" +} + +variable "skip_provisioners" { + type = bool + description = "Flag to skip local-exec provisioners" + default = false +} + +variable "enable_binary_authorization" { + description = "Enable BinAuthZ Admission controller" + default = false +} + +variable "gateway_api_channel" { + type = string + description = "The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`." + default = null +} diff --git a/examples/simple_regional_with_gateway_api/versions.tf b/examples/simple_regional_with_gateway_api/versions.tf new file mode 100644 index 0000000000..e8fbb1aadd --- /dev/null +++ b/examples/simple_regional_with_gateway_api/versions.tf @@ -0,0 +1,28 @@ +/** + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = "~> 4.0" + } + kubernetes = { + source = "hashicorp/kubernetes" + } + } + required_version = ">= 0.13" +} diff --git a/examples/simple_zonal_with_acm/README.md b/examples/simple_zonal_with_acm/README.md index 2607c45b8b..9c8c4fe2e5 100644 --- a/examples/simple_zonal_with_acm/README.md +++ b/examples/simple_zonal_with_acm/README.md @@ -1,6 +1,6 @@ # Simple Zonal Cluster -This example illustrates how to create a simple cluster and install [Anthos Config Management](https://cloud.google.com/anthos-config-management/docs/). +This example illustrates how to create a simple cluster and install [Anthos Config Management](https://cloud.google.com/anthos-config-management/docs/)'s [Config Sync](https://cloud.google.com/anthos-config-management/docs/config-sync-overview) and [Policy Controller](https://cloud.google.com/anthos-config-management/docs/concepts/policy-controller) with the [Policy Essentials v2022 policy bundle](https://cloud.google.com/anthos-config-management/docs/how-to/using-policy-essentials-v2022). It incorporates the standard cluster module and the [ACM install module](../../modules/acm). @@ -27,13 +27,19 @@ After applying the Terraform configuration, you can run the following commands t kubectl describe ns shipping-dev ``` +4. You can also use `kubectl` to view any policy violations on the cluster: + + ``` + kubectl get constraint -l policycontroller.gke.io/bundleName=policy-essentials-v2022 -o json | jq -cC '.items[]| [.metadata.name,.status.totalViolations]' + ``` + ## Inputs | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | cluster\_name\_suffix | A suffix to append to the default cluster name | `string` | `""` | no | -| project\_id | The project ID to host the cluster in | `any` | n/a | yes | +| project\_id | The project ID to host the cluster in | `string` | n/a | yes | | region | The region to host the cluster in | `string` | `"us-central1"` | no | | zone | The zone to host the cluster in | `string` | `"us-central1-a"` | no | diff --git a/examples/simple_zonal_with_acm/acm.tf b/examples/simple_zonal_with_acm/acm.tf index b1ea3225bc..c72df471e0 100644 --- a/examples/simple_zonal_with_acm/acm.tf +++ b/examples/simple_zonal_with_acm/acm.tf @@ -20,9 +20,13 @@ module "acm" { location = module.gke.location cluster_name = module.gke.name - sync_repo = "git@github.com:GoogleCloudPlatform/csp-config-management.git" + sync_repo = "git@github.com:GoogleCloudPlatform/anthos-config-management-samples.git" sync_branch = "1.0.0" policy_dir = "foo-corp" secret_type = "ssh" + + policy_bundles = ["https://github.com/GoogleCloudPlatform/acm-policy-controller-library/bundles/policy-essentials-v2022#e4094aacb91a35b0219f6f4cf6a31580e85b3c28"] + + create_metrics_gcp_sa = true } diff --git a/examples/simple_zonal_with_acm/variables.tf b/examples/simple_zonal_with_acm/variables.tf index c02931ccd9..722b32e674 100644 --- a/examples/simple_zonal_with_acm/variables.tf +++ b/examples/simple_zonal_with_acm/variables.tf @@ -16,15 +16,18 @@ variable "project_id" { description = "The project ID to host the cluster in" + type = string } variable "cluster_name_suffix" { description = "A suffix to append to the default cluster name" + type = string default = "" } variable "region" { description = "The region to host the cluster in" + type = string default = "us-central1" } diff --git a/examples/simple_zonal_with_acm/versions.tf b/examples/simple_zonal_with_acm/versions.tf index e8fbb1aadd..591f742201 100644 --- a/examples/simple_zonal_with_acm/versions.tf +++ b/examples/simple_zonal_with_acm/versions.tf @@ -21,7 +21,12 @@ terraform { version = "~> 4.0" } kubernetes = { - source = "hashicorp/kubernetes" + source = "hashicorp/kubernetes" + version = "~> 2.10" + } + random = { + source = "hashicorp/random" + version = ">= 2.1" } } required_version = ">= 0.13" diff --git a/firewall.tf b/firewall.tf index 8586855e89..a754fda5c6 100644 --- a/firewall.tf +++ b/firewall.tf @@ -34,11 +34,12 @@ resource "google_compute_firewall" "intra_egress" { direction = "EGRESS" target_tags = [local.cluster_network_tag] - destination_ranges = [ + destination_ranges = concat([ local.cluster_endpoint_for_nodes, local.cluster_subnet_cidr, - local.cluster_alias_ranges_cidr[var.ip_range_pods], - ] + ], + local.pod_all_ip_ranges + ) # Allow all possible protocols allow { protocol = "tcp" } @@ -99,7 +100,7 @@ resource "google_compute_firewall" "shadow_allow_pods" { priority = var.shadow_firewall_rules_priority direction = "INGRESS" - source_ranges = [local.cluster_alias_ranges_cidr[var.ip_range_pods]] + source_ranges = local.pod_all_ip_ranges target_tags = [local.cluster_network_tag] # Allow all possible protocols @@ -110,8 +111,11 @@ resource "google_compute_firewall" "shadow_allow_pods" { allow { protocol = "esp" } allow { protocol = "ah" } - log_config { - metadata = "INCLUDE_ALL_METADATA" + dynamic "log_config" { + for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config] + content { + metadata = log_config.value.metadata + } } } @@ -133,8 +137,11 @@ resource "google_compute_firewall" "shadow_allow_master" { ports = ["10250", "443"] } - log_config { - metadata = "INCLUDE_ALL_METADATA" + dynamic "log_config" { + for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config] + content { + metadata = log_config.value.metadata + } } } @@ -165,7 +172,63 @@ resource "google_compute_firewall" "shadow_allow_nodes" { ports = ["1-65535"] } - log_config { - metadata = "INCLUDE_ALL_METADATA" + dynamic "log_config" { + for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config] + content { + metadata = log_config.value.metadata + } + } +} + +resource "google_compute_firewall" "shadow_allow_inkubelet" { + count = var.add_shadow_firewall_rules ? 1 : 0 + + name = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-inkubelet" + description = "Managed by terraform GKE module: A shadow firewall rule to match the default rule allowing worker nodes & pods communication to kubelet." + project = local.network_project_id + network = var.network + priority = var.shadow_firewall_rules_priority - 1 # rule created by GKE robot have prio 999 + direction = "INGRESS" + + source_ranges = local.pod_all_ip_ranges + source_tags = [local.cluster_network_tag] + target_tags = [local.cluster_network_tag] + + allow { + protocol = "tcp" + ports = ["10255"] + } + + dynamic "log_config" { + for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config] + content { + metadata = log_config.value.metadata + } + } +} + +resource "google_compute_firewall" "shadow_deny_exkubelet" { + count = var.add_shadow_firewall_rules ? 1 : 0 + + name = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-exkubelet" + description = "Managed by terraform GKE module: A shadow firewall rule to match the default deny rule to kubelet." + project = local.network_project_id + network = var.network + priority = var.shadow_firewall_rules_priority # rule created by GKE robot have prio 1000 + direction = "INGRESS" + + source_ranges = ["0.0.0.0/0"] + target_tags = [local.cluster_network_tag] + + deny { + protocol = "tcp" + ports = ["10255"] + } + + dynamic "log_config" { + for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config] + content { + metadata = log_config.value.metadata + } } } diff --git a/main.tf b/main.tf index 83caf0fbda..48bf66213a 100644 --- a/main.tf +++ b/main.tf @@ -50,7 +50,8 @@ locals { windows_node_pool_names = [for np in toset(var.windows_node_pools) : np.name] windows_node_pools = zipmap(local.windows_node_pool_names, tolist(toset(var.windows_node_pools))) - release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] + release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] + gateway_api_config = var.gateway_api_channel != null ? [{ channel : var.gateway_api_channel }] : [] autoscaling_resource_limits = var.cluster_autoscaling.enabled ? concat([{ resource_type = "cpu" @@ -73,6 +74,7 @@ locals { cluster_subnet_cidr = var.add_cluster_firewall_rules ? data.google_compute_subnetwork.gke_subnetwork[0].ip_cidr_range : null cluster_alias_ranges_cidr = var.add_cluster_firewall_rules ? { for range in toset(data.google_compute_subnetwork.gke_subnetwork[0].secondary_ip_range) : range.range_name => range.ip_cidr_range } : {} + pod_all_ip_ranges = var.add_cluster_firewall_rules ? compact(concat([local.cluster_alias_ranges_cidr[var.ip_range_pods]], [for k, v in merge(local.node_pools, local.windows_node_pools) : local.cluster_alias_ranges_cidr[v.pod_range] if length(lookup(v, "pod_range", "")) > 0])) : [] cluster_network_policy = var.network_policy ? [{ enabled = true @@ -81,6 +83,9 @@ locals { enabled = false provider = null }] + cluster_gce_pd_csi_config = var.gce_pd_csi_driver ? [{ enabled = true }] : [{ enabled = false }] + logmon_config_is_set = length(var.logging_enabled_components) > 0 || length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus + gke_backup_agent_config = var.gke_backup_agent_config ? [{ enabled = true }] : [{ enabled = false }] cluster_authenticator_security_group = var.authenticator_security_group == null ? [] : [{ security_group = var.authenticator_security_group diff --git a/modules/acm/README.md b/modules/acm/README.md index de785bccd5..4766079de0 100644 --- a/modules/acm/README.md +++ b/modules/acm/README.md @@ -3,10 +3,12 @@ This module installs [Anthos Config Management](https://cloud.google.com/anthos-config-management/docs/) (ACM) in a Kubernetes cluster. Specifically, this module automates the following steps for [installing ACM](https://cloud.google.com/anthos-config-management/docs/how-to/installing): -1. Enabling the ACM feature on the fleet. +1. Enabling the ACM feature on the fleet 2. Registering the cluster to the fleet 3. Optionally, generating an SSH key for accessing Git and providing it to the Operator 4. Configuring the ACM feature on your cluster +5. Optionally, installing ACM Policy Controller [Policy Bundle(s)](https://cloud.google.com/anthos-config-management/docs/concepts/policy-controller-bundles) +6. Optionally, create and configure a Google Cloud Service Account for writing ACM metrics to Cloud Monitoring ## Fleet feature Only the first cluster in a fleet should activate the ACM fleet feature. @@ -32,9 +34,14 @@ module "acm" { project_id = "my-project-id" cluster_name = "my-cluster-name" location = module.gke.location - sync_repo = "git@github.com:GoogleCloudPlatform/csp-config-management.git" + sync_repo = "git@github.com:GoogleCloudPlatform/anthos-config-management-samples.git" sync_branch = "1.0.0" policy_dir = "foo-corp" + + # ACM Policy Controller Policy Essentials Policy Bundle: https://cloud.google.com/anthos-config-management/docs/how-to/using-policy-essentials-v2022 + policy_bundles = ["https://github.com/GoogleCloudPlatform/acm-policy-controller-library/bundles/policy-essentials-v2022#e4094aacb91a35b0219f6f4cf6a31580e85b3c28"] + + create_metrics_gcp_sa = true } ``` @@ -67,7 +74,9 @@ data "google_client_config" "default" {} | cluster\_membership\_id | The cluster membership ID. If unset, one will be autogenerated. | `string` | `""` | no | | cluster\_name | GCP cluster Name used to reach cluster and which becomes the cluster name in the Config Sync kubernetes custom resource. | `string` | n/a | yes | | configmanagement\_version | Version of ACM. | `string` | `""` | no | +| create\_metrics\_gcp\_sa | Create a Google service account for ACM metrics writing | `bool` | `false` | no | | create\_ssh\_key | Controls whether a key will be generated for Git authentication | `bool` | `true` | no | +| enable\_config\_sync | Whether to enable the ACM Config Sync on the cluster | `bool` | `true` | no | | enable\_fleet\_feature | Whether to enable the ACM feature on the fleet. | `bool` | `true` | no | | enable\_fleet\_registration | Whether to create a new membership. | `bool` | `true` | no | | enable\_log\_denies | Whether to enable logging of all denies and dryrun failures for ACM Policy Controller. | `bool` | `false` | no | @@ -77,19 +86,22 @@ data "google_client_config" "default" {} | https\_proxy | URL for the HTTPS proxy to be used when communicating with the Git repo. | `string` | `null` | no | | install\_template\_library | Whether to install the default Policy Controller template library | `bool` | `true` | no | | location | GCP location used to reach cluster. | `string` | n/a | yes | +| metrics\_gcp\_sa\_name | The name of the Google service account for ACM metrics writing | `string` | `"acm-metrics-writer"` | no | +| policy\_bundles | A list of Policy Controller policy bundles git urls (example: https://github.com/GoogleCloudPlatform/acm-policy-controller-library.git/bundles/policy-essentials-v2022) to install on the cluster. | `list(string)` | `[]` | no | | policy\_dir | Subfolder containing configs in ACM Git repo. If un-set, uses Config Management default. | `string` | `""` | no | | project\_id | GCP project\_id used to reach cluster. | `string` | n/a | yes | | secret\_type | git authentication secret type, is passed through to ConfigManagement spec.git.secretType. Overriden to value 'ssh' if `create_ssh_key` is true | `string` | `"ssh"` | no | | source\_format | Configures a non-hierarchical repo if set to 'unstructured'. Uses [ACM defaults](https://cloud.google.com/anthos-config-management/docs/how-to/installing#configuring-config-management-operator) when unset. | `string` | `""` | no | | ssh\_auth\_key | Key for Git authentication. Overrides 'create\_ssh\_key' variable. Can be set using 'file(path/to/file)'-function. | `string` | `null` | no | | sync\_branch | ACM repo Git branch. If un-set, uses Config Management default. | `string` | `""` | no | -| sync\_repo | ACM Git repo address | `string` | n/a | yes | +| sync\_repo | ACM Git repo address | `string` | `""` | no | | sync\_revision | ACM repo Git revision. If un-set, uses Config Management default. | `string` | `""` | no | ## Outputs | Name | Description | |------|-------------| +| acm\_metrics\_writer\_sa | The ACM metrics writer Service Account | | configmanagement\_version | Version of ACM installed. | | git\_creds\_public | Public key of SSH keypair to allow the Anthos Config Management Operator to authenticate to your Git repository. | | wait | An output to use when you want to depend on cmd finishing | diff --git a/modules/acm/creds.tf b/modules/acm/creds.tf index 45a6748949..b4b2f96c98 100644 --- a/modules/acm/creds.tf +++ b/modules/acm/creds.tf @@ -14,6 +14,11 @@ * limitations under the License. */ +locals { + # GCP service account ids must be <= 30 chars matching regex ^[a-z](?:[-a-z0-9]{4,28}[a-z0-9])$ + service_account_name = trimsuffix(substr(var.metrics_gcp_sa_name, 0, 30), "-") +} + resource "tls_private_key" "k8sop_creds" { count = var.create_ssh_key ? 1 : 0 algorithm = "RSA" @@ -22,10 +27,92 @@ resource "tls_private_key" "k8sop_creds" { # Wait for the ACM operator to create the namespace resource "time_sleep" "wait_acm" { - count = (var.create_ssh_key == true || var.ssh_auth_key != null) ? 1 : 0 + count = (var.create_ssh_key == true || var.ssh_auth_key != null || var.enable_policy_controller || var.enable_config_sync) ? 1 : 0 + depends_on = [google_gke_hub_feature_membership.main] + + create_duration = "300s" +} + +resource "google_service_account_iam_binding" "config-management-monitoring-iam" { + count = var.enable_config_sync && var.create_metrics_gcp_sa ? 1 : 0 + service_account_id = google_service_account.acm_metrics_writer_sa[0].name + role = "roles/iam.workloadIdentityUser" + + members = ["serviceAccount:${var.project_id}.svc.id.goog[config-management-monitoring/default]"] + depends_on = [google_gke_hub_feature_membership.main] +} + +resource "google_service_account_iam_binding" "gatekeeper-system-iam" { + count = var.enable_policy_controller && var.create_metrics_gcp_sa ? 1 : 0 + service_account_id = google_service_account.acm_metrics_writer_sa[0].name + role = "roles/iam.workloadIdentityUser" + + members = ["serviceAccount:${var.project_id}.svc.id.goog[gatekeeper-system/gatekeeper-admin]"] + + depends_on = [google_gke_hub_feature_membership.main] +} + +module "annotate-sa-config-management-monitoring" { + source = "terraform-google-modules/gcloud/google//modules/kubectl-wrapper" + version = "~> 3.1" + + count = var.enable_config_sync && var.create_metrics_gcp_sa ? 1 : 0 + skip_download = true + cluster_name = var.cluster_name + cluster_location = var.location + project_id = var.project_id + + kubectl_create_command = "kubectl annotate --overwrite sa -n config-management-monitoring default iam.gke.io/gcp-service-account=${google_service_account.acm_metrics_writer_sa[0].email}" + kubectl_destroy_command = "kubectl annotate sa -n config-management-monitoring default iam.gke.io/gcp-service-account-" + + module_depends_on = time_sleep.wait_acm +} + +module "annotate-sa-gatekeeper-system" { + source = "terraform-google-modules/gcloud/google//modules/kubectl-wrapper" + version = "~> 3.1" + + count = var.enable_policy_controller && var.create_metrics_gcp_sa ? 1 : 0 + skip_download = true + cluster_name = var.cluster_name + cluster_location = var.location + project_id = var.project_id + + kubectl_create_command = "kubectl annotate --overwrite sa -n gatekeeper-system gatekeeper-admin iam.gke.io/gcp-service-account=${google_service_account.acm_metrics_writer_sa[0].email}" + kubectl_destroy_command = "kubectl annotate sa -n gatekeeper-system gatekeeper-admin iam.gke.io/gcp-service-account-" + + module_depends_on = time_sleep.wait_acm +} + +module "annotate-sa-gatekeeper-system-restart" { + source = "terraform-google-modules/gcloud/google//modules/kubectl-wrapper" + version = "~> 3.1" + + count = var.enable_policy_controller && var.create_metrics_gcp_sa ? 1 : 0 + skip_download = true + cluster_name = var.cluster_name + cluster_location = var.location + project_id = var.project_id + + kubectl_create_command = "kubectl rollout restart deployment gatekeeper-controller-manager -n gatekeeper-system" + kubectl_destroy_command = "" + + module_depends_on = module.annotate-sa-gatekeeper-system +} + +resource "google_service_account" "acm_metrics_writer_sa" { + count = var.create_metrics_gcp_sa ? 1 : 0 + + display_name = "ACM Metrics Writer SA" + account_id = local.service_account_name + project = var.project_id +} - create_duration = "60s" +resource "google_project_iam_member" "acm_metrics_writer_sa_role" { + project = var.project_id + role = "roles/monitoring.metricWriter" + member = "serviceAccount:${google_service_account.acm_metrics_writer_sa[0].email}" } resource "kubernetes_secret_v1" "creds" { @@ -38,6 +125,6 @@ resource "kubernetes_secret_v1" "creds" { } data = { - "${local.k8sop_creds_secret_key}" = local.private_key + (local.k8sop_creds_secret_key) = local.private_key } } diff --git a/modules/acm/feature.tf b/modules/acm/feature.tf index 3bb977388a..101bac31ee 100644 --- a/modules/acm/feature.tf +++ b/modules/acm/feature.tf @@ -38,16 +38,20 @@ resource "google_gke_hub_feature_membership" "main" { configmanagement { version = var.configmanagement_version - config_sync { - source_format = var.source_format != "" ? var.source_format : null + dynamic "config_sync" { + for_each = var.enable_config_sync ? [{ enabled = true }] : [] - git { - sync_repo = var.sync_repo - policy_dir = var.policy_dir != "" ? var.policy_dir : null - sync_branch = var.sync_branch != "" ? var.sync_branch : null - sync_rev = var.sync_revision != "" ? var.sync_revision : null - secret_type = var.secret_type - https_proxy = var.https_proxy + content { + source_format = var.source_format != "" ? var.source_format : null + + git { + sync_repo = var.sync_repo + policy_dir = var.policy_dir != "" ? var.policy_dir : null + sync_branch = var.sync_branch != "" ? var.sync_branch : null + sync_rev = var.sync_revision != "" ? var.sync_revision : null + secret_type = var.secret_type + https_proxy = var.https_proxy + } } } diff --git a/modules/acm/outputs.tf b/modules/acm/outputs.tf index 063d5f171f..3e9594cee5 100644 --- a/modules/acm/outputs.tf +++ b/modules/acm/outputs.tf @@ -16,7 +16,7 @@ output "git_creds_public" { description = "Public key of SSH keypair to allow the Anthos Config Management Operator to authenticate to your Git repository." - value = var.create_ssh_key ? coalesce(tls_private_key.k8sop_creds.*.public_key_openssh...) : null + value = var.create_ssh_key ? coalesce(tls_private_key.k8sop_creds[*].public_key_openssh...) : null } output "configmanagement_version" { @@ -31,3 +31,8 @@ output "wait" { google_gke_hub_feature_membership.main ] } + +output "acm_metrics_writer_sa" { + description = "The ACM metrics writer Service Account" + value = google_service_account.acm_metrics_writer_sa[0].email +} diff --git a/modules/acm/policy_bundles.tf b/modules/acm/policy_bundles.tf new file mode 100644 index 0000000000..b03987e867 --- /dev/null +++ b/modules/acm/policy_bundles.tf @@ -0,0 +1,29 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "policy_bundles" { + source = "terraform-google-modules/gcloud/google//modules/kubectl-wrapper" + version = "~> 3.1" + + for_each = toset(var.policy_bundles) + project_id = var.project_id + cluster_name = var.cluster_name + cluster_location = var.location + kubectl_create_command = "kubectl apply -k ${each.key}" + kubectl_destroy_command = "kubectl delete -k ${each.key}" + + module_depends_on = [time_sleep.wait_acm] +} diff --git a/modules/acm/variables.tf b/modules/acm/variables.tf index f9bf8a41f5..dfb4f6d2fd 100644 --- a/modules/acm/variables.tf +++ b/modules/acm/variables.tf @@ -57,6 +57,7 @@ variable "configmanagement_version" { variable "sync_repo" { description = "ACM Git repo address" type = string + default = "" } variable "sync_branch" { @@ -108,6 +109,12 @@ variable "ssh_auth_key" { default = null } +variable "enable_config_sync" { + description = "Whether to enable the ACM Config Sync on the cluster" + type = bool + default = true +} + # Policy Controller config variable "enable_policy_controller" { description = "Whether to enable the ACM Policy Controller on the cluster" @@ -139,3 +146,21 @@ variable "enable_referential_rules" { type = bool default = true } + +variable "policy_bundles" { + description = "A list of Policy Controller policy bundles git urls (example: https://github.com/GoogleCloudPlatform/acm-policy-controller-library.git/bundles/policy-essentials-v2022) to install on the cluster." + type = list(string) + default = [] +} + +variable "create_metrics_gcp_sa" { + description = "Create a Google service account for ACM metrics writing" + type = bool + default = false +} + +variable "metrics_gcp_sa_name" { + description = "The name of the Google service account for ACM metrics writing" + type = string + default = "acm-metrics-writer" +} diff --git a/modules/acm/versions.tf b/modules/acm/versions.tf index ff75bf4a7b..52a50a3e52 100644 --- a/modules/acm/versions.tf +++ b/modules/acm/versions.tf @@ -19,11 +19,11 @@ terraform { required_version = ">= 0.13.0" provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:acm/v24.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:acm/v25.0.0" } provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:acm/v24.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:acm/v25.0.0" } required_providers { diff --git a/modules/asm/versions.tf b/modules/asm/versions.tf index 507c9a9988..815c188c68 100644 --- a/modules/asm/versions.tf +++ b/modules/asm/versions.tf @@ -23,13 +23,18 @@ terraform { source = "hashicorp/kubernetes" version = "~> 2.0" } + google = { + source = "hashicorp/google" + # Avoid v25.0.0 for https://github.com/hashicorp/terraform-provider-google/issues/13507 + version = ">= 4.47.0, != 4.49.0, != 4.50.0, < 5.0" + } } provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:asm/v24.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:asm/v25.0.0" } provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:asm/v24.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:asm/v25.0.0" } } diff --git a/modules/auth/versions.tf b/modules/auth/versions.tf index 7c95017d7c..c02d12ea49 100644 --- a/modules/auth/versions.tf +++ b/modules/auth/versions.tf @@ -17,8 +17,15 @@ terraform { required_version = ">= 0.13.0" + required_providers { + google = { + source = "hashicorp/google" + # Avoid v25.0.0 for https://github.com/hashicorp/terraform-provider-google/issues/13507 + version = ">= 4.47.0, != 4.49.0, != 4.50.0, < 5.0" + } + } provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:auth/v24.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:auth/v25.0.0" } } diff --git a/modules/beta-autopilot-private-cluster/README.md b/modules/beta-autopilot-private-cluster/README.md index edc894d1fe..db7068e0f2 100644 --- a/modules/beta-autopilot-private-cluster/README.md +++ b/modules/beta-autopilot-private-cluster/README.md @@ -92,6 +92,7 @@ Then perform the following commands on the root folder: | enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | `bool` | `false` | no | | firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | | firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | +| gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | | http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no | @@ -110,7 +111,6 @@ Then perform the following commands on the root folder: | master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | | master\_global\_access\_enabled | Whether the cluster master is accessible globally (from any region) or only within the same region as the private endpoint. | `bool` | `true` | no | | master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | `string` | `"10.0.0.0/28"` | no | -| monitoring\_enable\_managed\_prometheus | (Beta) Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `false` | no | | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | | name | The name of the cluster (required) | `string` | n/a | yes | | network | The VPC network to host the cluster in (required) | `string` | n/a | yes | @@ -125,6 +125,7 @@ Then perform the following commands on the root folder: | resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. | `string` | `""` | no | | service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | +| shadow\_firewall\_rules\_log\_config | The log\_config for shadow firewall rules. You can set this variable to `null` to disable logging. | 
object({
    metadata = string
  })
| 
{
  "metadata": "INCLUDE_ALL_METADATA"
}
| no | | shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | | skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | `bool` | `false` | no | | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | @@ -142,6 +143,7 @@ Then perform the following commands on the root folder: | cluster\_id | Cluster ID | | dns\_cache\_enabled | Whether DNS Cache enabled | | endpoint | Cluster endpoint | +| gateway\_api\_channel | The gateway api channel of this cluster. | | horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | | http\_load\_balancing\_enabled | Whether http load balancing enabled | | identity\_namespace | Workload Identity pool | @@ -184,8 +186,8 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog #### Kubectl - [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x #### Terraform and Plugins -- [Terraform](https://www.terraform.io/downloads.html) 0.12 -- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v3.41 +- [Terraform](https://www.terraform.io/downloads.html) 0.13+ +- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v4.51 #### gcloud Some submodules use the [terraform-google-gcloud](https://github.com/terraform-google-modules/terraform-google-gcloud) module. By default, this module assumes you already have gcloud installed in your $PATH. See the [module](https://github.com/terraform-google-modules/terraform-google-gcloud#downloading) documentation for more information. diff --git a/modules/beta-autopilot-private-cluster/cluster.tf b/modules/beta-autopilot-private-cluster/cluster.tf index d61138267d..2c4a80cb1a 100644 --- a/modules/beta-autopilot-private-cluster/cluster.tf +++ b/modules/beta-autopilot-private-cluster/cluster.tf @@ -39,6 +39,15 @@ resource "google_container_cluster" "primary" { channel = release_channel.value.channel } } + + dynamic "gateway_api_config" { + for_each = local.gateway_api_config + + content { + channel = gateway_api_config.value.channel + } + } + dynamic "cost_management_config" { for_each = var.enable_cost_allocation ? [1] : [] content { @@ -60,14 +69,12 @@ resource "google_container_cluster" "primary" { min_master_version = var.release_channel == null || var.release_channel == "UNSPECIFIED" ? local.master_version : null - logging_service = var.logging_service - monitoring_service = var.monitoring_service - dynamic "monitoring_config" { - for_each = var.monitoring_enable_managed_prometheus ? [1] : [] + cluster_autoscaling { + dynamic "auto_provisioning_defaults" { + for_each = (var.create_service_account || var.service_account != "") ? [1] : [] - content { - managed_prometheus { - enabled = var.monitoring_enable_managed_prometheus + content { + service_account = local.service_account } } } @@ -110,11 +117,8 @@ resource "google_container_cluster" "primary" { disabled = !var.horizontal_pod_autoscaling } - } - datapath_provider = var.datapath_provider - networking_mode = "VPC_NATIVE" ip_allocation_policy { cluster_secondary_range_name = var.ip_range_pods diff --git a/modules/beta-autopilot-private-cluster/firewall.tf b/modules/beta-autopilot-private-cluster/firewall.tf index 14c3e54cb0..96eecab803 100644 --- a/modules/beta-autopilot-private-cluster/firewall.tf +++ b/modules/beta-autopilot-private-cluster/firewall.tf @@ -34,11 +34,12 @@ resource "google_compute_firewall" "intra_egress" { direction = "EGRESS" target_tags = [local.cluster_network_tag] - destination_ranges = [ + destination_ranges = concat([ local.cluster_endpoint_for_nodes, local.cluster_subnet_cidr, - local.cluster_alias_ranges_cidr[var.ip_range_pods], - ] + ], + local.pod_all_ip_ranges + ) # Allow all possible protocols allow { protocol = "tcp" } @@ -126,7 +127,7 @@ resource "google_compute_firewall" "shadow_allow_pods" { priority = var.shadow_firewall_rules_priority direction = "INGRESS" - source_ranges = [local.cluster_alias_ranges_cidr[var.ip_range_pods]] + source_ranges = local.pod_all_ip_ranges target_tags = [local.cluster_network_tag] # Allow all possible protocols @@ -137,8 +138,11 @@ resource "google_compute_firewall" "shadow_allow_pods" { allow { protocol = "esp" } allow { protocol = "ah" } - log_config { - metadata = "INCLUDE_ALL_METADATA" + dynamic "log_config" { + for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config] + content { + metadata = log_config.value.metadata + } } } @@ -160,8 +164,11 @@ resource "google_compute_firewall" "shadow_allow_master" { ports = ["10250", "443"] } - log_config { - metadata = "INCLUDE_ALL_METADATA" + dynamic "log_config" { + for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config] + content { + metadata = log_config.value.metadata + } } } @@ -192,7 +199,63 @@ resource "google_compute_firewall" "shadow_allow_nodes" { ports = ["1-65535"] } - log_config { - metadata = "INCLUDE_ALL_METADATA" + dynamic "log_config" { + for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config] + content { + metadata = log_config.value.metadata + } + } +} + +resource "google_compute_firewall" "shadow_allow_inkubelet" { + count = var.add_shadow_firewall_rules ? 1 : 0 + + name = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-inkubelet" + description = "Managed by terraform GKE module: A shadow firewall rule to match the default rule allowing worker nodes & pods communication to kubelet." + project = local.network_project_id + network = var.network + priority = var.shadow_firewall_rules_priority - 1 # rule created by GKE robot have prio 999 + direction = "INGRESS" + + source_ranges = local.pod_all_ip_ranges + source_tags = [local.cluster_network_tag] + target_tags = [local.cluster_network_tag] + + allow { + protocol = "tcp" + ports = ["10255"] + } + + dynamic "log_config" { + for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config] + content { + metadata = log_config.value.metadata + } + } +} + +resource "google_compute_firewall" "shadow_deny_exkubelet" { + count = var.add_shadow_firewall_rules ? 1 : 0 + + name = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-exkubelet" + description = "Managed by terraform GKE module: A shadow firewall rule to match the default deny rule to kubelet." + project = local.network_project_id + network = var.network + priority = var.shadow_firewall_rules_priority # rule created by GKE robot have prio 1000 + direction = "INGRESS" + + source_ranges = ["0.0.0.0/0"] + target_tags = [local.cluster_network_tag] + + deny { + protocol = "tcp" + ports = ["10255"] + } + + dynamic "log_config" { + for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config] + content { + metadata = log_config.value.metadata + } } } diff --git a/modules/beta-autopilot-private-cluster/main.tf b/modules/beta-autopilot-private-cluster/main.tf index 613be8b4ff..8e1964224c 100644 --- a/modules/beta-autopilot-private-cluster/main.tf +++ b/modules/beta-autopilot-private-cluster/main.tf @@ -45,7 +45,8 @@ locals { master_version_zonal = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.zone.latest_master_version master_version = var.regional ? local.master_version_regional : local.master_version_zonal - release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] + release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] + gateway_api_config = var.gateway_api_channel != null ? [{ channel : var.gateway_api_channel }] : [] @@ -60,6 +61,7 @@ locals { cluster_subnet_cidr = var.add_cluster_firewall_rules ? data.google_compute_subnetwork.gke_subnetwork[0].ip_cidr_range : null cluster_alias_ranges_cidr = var.add_cluster_firewall_rules ? { for range in toset(data.google_compute_subnetwork.gke_subnetwork[0].secondary_ip_range) : range.range_name => range.ip_cidr_range } : {} + pod_all_ip_ranges = var.add_cluster_firewall_rules ? [local.cluster_alias_ranges_cidr[var.ip_range_pods]] : [] cluster_authenticator_security_group = var.authenticator_security_group == null ? [] : [{ diff --git a/modules/beta-autopilot-private-cluster/outputs.tf b/modules/beta-autopilot-private-cluster/outputs.tf index c0a333f39c..a56e4b4faf 100644 --- a/modules/beta-autopilot-private-cluster/outputs.tf +++ b/modules/beta-autopilot-private-cluster/outputs.tf @@ -129,6 +129,11 @@ output "release_channel" { value = var.release_channel } +output "gateway_api_channel" { + description = "The gateway api channel of this cluster." + value = var.gateway_api_channel +} + output "identity_namespace" { description = "Workload Identity pool" value = length(local.cluster_workload_identity_config) > 0 ? local.cluster_workload_identity_config[0].workload_pool : null diff --git a/modules/beta-autopilot-private-cluster/variables.tf b/modules/beta-autopilot-private-cluster/variables.tf index 88267be7d8..0a96ecb63c 100644 --- a/modules/beta-autopilot-private-cluster/variables.tf +++ b/modules/beta-autopilot-private-cluster/variables.tf @@ -323,6 +323,12 @@ variable "release_channel" { default = null } +variable "gateway_api_channel" { + type = string + description = "The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`." + default = null +} + variable "add_cluster_firewall_rules" { type = bool description = "Create additional firewall rules" @@ -357,6 +363,20 @@ variable "shadow_firewall_rules_priority" { type = number description = "The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000." default = 999 + validation { + condition = var.shadow_firewall_rules_priority < 1000 + error_message = "The shadow firewall rule priority must be lower than auto-created one(1000)." + } +} + +variable "shadow_firewall_rules_log_config" { + type = object({ + metadata = string + }) + description = "The log_config for shadow firewall rules. You can set this variable to `null` to disable logging." + default = { + metadata = "INCLUDE_ALL_METADATA" + } } variable "enable_confidential_nodes" { @@ -403,8 +423,3 @@ variable "timeouts" { } } -variable "monitoring_enable_managed_prometheus" { - type = bool - description = "(Beta) Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled." - default = false -} diff --git a/modules/beta-autopilot-private-cluster/versions.tf b/modules/beta-autopilot-private-cluster/versions.tf index 3d0c10df5a..e116c00b89 100644 --- a/modules/beta-autopilot-private-cluster/versions.tf +++ b/modules/beta-autopilot-private-cluster/versions.tf @@ -21,7 +21,7 @@ terraform { required_providers { google-beta = { source = "hashicorp/google-beta" - version = ">= 4.42.0, < 5.0" + version = ">= 4.51.0, < 5.0" } kubernetes = { source = "hashicorp/kubernetes" @@ -29,6 +29,6 @@ terraform { } } provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:beta-autopilot-private-cluster/v24.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:beta-autopilot-private-cluster/v25.0.0" } } diff --git a/modules/beta-autopilot-public-cluster/README.md b/modules/beta-autopilot-public-cluster/README.md index 14c6eb91f3..265743bfce 100644 --- a/modules/beta-autopilot-public-cluster/README.md +++ b/modules/beta-autopilot-public-cluster/README.md @@ -83,6 +83,7 @@ Then perform the following commands on the root folder: | enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | `bool` | `false` | no | | firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | | firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | +| gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | | http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no | @@ -99,7 +100,6 @@ Then perform the following commands on the root folder: | maintenance\_recurrence | Frequency of the recurring maintenance window in RFC5545 format. | `string` | `""` | no | | maintenance\_start\_time | Time window specified for daily or recurring maintenance operations in RFC3339 format | `string` | `"05:00"` | no | | master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | -| monitoring\_enable\_managed\_prometheus | (Beta) Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `false` | no | | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | | name | The name of the cluster (required) | `string` | n/a | yes | | network | The VPC network to host the cluster in (required) | `string` | n/a | yes | @@ -114,6 +114,7 @@ Then perform the following commands on the root folder: | resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. | `string` | `""` | no | | service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | +| shadow\_firewall\_rules\_log\_config | The log\_config for shadow firewall rules. You can set this variable to `null` to disable logging. | 
object({
    metadata = string
  })
| 
{
  "metadata": "INCLUDE_ALL_METADATA"
}
| no | | shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | | skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | `bool` | `false` | no | | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | @@ -131,6 +132,7 @@ Then perform the following commands on the root folder: | cluster\_id | Cluster ID | | dns\_cache\_enabled | Whether DNS Cache enabled | | endpoint | Cluster endpoint | +| gateway\_api\_channel | The gateway api channel of this cluster. | | horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | | http\_load\_balancing\_enabled | Whether http load balancing enabled | | identity\_namespace | Workload Identity pool | @@ -171,8 +173,8 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog #### Kubectl - [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x #### Terraform and Plugins -- [Terraform](https://www.terraform.io/downloads.html) 0.12 -- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v3.41 +- [Terraform](https://www.terraform.io/downloads.html) 0.13+ +- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v4.51 #### gcloud Some submodules use the [terraform-google-gcloud](https://github.com/terraform-google-modules/terraform-google-gcloud) module. By default, this module assumes you already have gcloud installed in your $PATH. See the [module](https://github.com/terraform-google-modules/terraform-google-gcloud#downloading) documentation for more information. diff --git a/modules/beta-autopilot-public-cluster/cluster.tf b/modules/beta-autopilot-public-cluster/cluster.tf index aa78edc45b..a24d5020e7 100644 --- a/modules/beta-autopilot-public-cluster/cluster.tf +++ b/modules/beta-autopilot-public-cluster/cluster.tf @@ -39,6 +39,15 @@ resource "google_container_cluster" "primary" { channel = release_channel.value.channel } } + + dynamic "gateway_api_config" { + for_each = local.gateway_api_config + + content { + channel = gateway_api_config.value.channel + } + } + dynamic "cost_management_config" { for_each = var.enable_cost_allocation ? [1] : [] content { @@ -60,14 +69,12 @@ resource "google_container_cluster" "primary" { min_master_version = var.release_channel == null || var.release_channel == "UNSPECIFIED" ? local.master_version : null - logging_service = var.logging_service - monitoring_service = var.monitoring_service - dynamic "monitoring_config" { - for_each = var.monitoring_enable_managed_prometheus ? [1] : [] + cluster_autoscaling { + dynamic "auto_provisioning_defaults" { + for_each = (var.create_service_account || var.service_account != "") ? [1] : [] - content { - managed_prometheus { - enabled = var.monitoring_enable_managed_prometheus + content { + service_account = local.service_account } } } @@ -110,11 +117,8 @@ resource "google_container_cluster" "primary" { disabled = !var.horizontal_pod_autoscaling } - } - datapath_provider = var.datapath_provider - networking_mode = "VPC_NATIVE" ip_allocation_policy { cluster_secondary_range_name = var.ip_range_pods diff --git a/modules/beta-autopilot-public-cluster/firewall.tf b/modules/beta-autopilot-public-cluster/firewall.tf index 6b71404b79..df15a02367 100644 --- a/modules/beta-autopilot-public-cluster/firewall.tf +++ b/modules/beta-autopilot-public-cluster/firewall.tf @@ -34,11 +34,12 @@ resource "google_compute_firewall" "intra_egress" { direction = "EGRESS" target_tags = [local.cluster_network_tag] - destination_ranges = [ + destination_ranges = concat([ local.cluster_endpoint_for_nodes, local.cluster_subnet_cidr, - local.cluster_alias_ranges_cidr[var.ip_range_pods], - ] + ], + local.pod_all_ip_ranges + ) # Allow all possible protocols allow { protocol = "tcp" } @@ -135,7 +136,7 @@ resource "google_compute_firewall" "shadow_allow_pods" { priority = var.shadow_firewall_rules_priority direction = "INGRESS" - source_ranges = [local.cluster_alias_ranges_cidr[var.ip_range_pods]] + source_ranges = local.pod_all_ip_ranges target_tags = [local.cluster_network_tag] # Allow all possible protocols @@ -146,8 +147,11 @@ resource "google_compute_firewall" "shadow_allow_pods" { allow { protocol = "esp" } allow { protocol = "ah" } - log_config { - metadata = "INCLUDE_ALL_METADATA" + dynamic "log_config" { + for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config] + content { + metadata = log_config.value.metadata + } } } @@ -169,8 +173,11 @@ resource "google_compute_firewall" "shadow_allow_master" { ports = ["10250", "443"] } - log_config { - metadata = "INCLUDE_ALL_METADATA" + dynamic "log_config" { + for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config] + content { + metadata = log_config.value.metadata + } } } @@ -201,7 +208,63 @@ resource "google_compute_firewall" "shadow_allow_nodes" { ports = ["1-65535"] } - log_config { - metadata = "INCLUDE_ALL_METADATA" + dynamic "log_config" { + for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config] + content { + metadata = log_config.value.metadata + } + } +} + +resource "google_compute_firewall" "shadow_allow_inkubelet" { + count = var.add_shadow_firewall_rules ? 1 : 0 + + name = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-inkubelet" + description = "Managed by terraform GKE module: A shadow firewall rule to match the default rule allowing worker nodes & pods communication to kubelet." + project = local.network_project_id + network = var.network + priority = var.shadow_firewall_rules_priority - 1 # rule created by GKE robot have prio 999 + direction = "INGRESS" + + source_ranges = local.pod_all_ip_ranges + source_tags = [local.cluster_network_tag] + target_tags = [local.cluster_network_tag] + + allow { + protocol = "tcp" + ports = ["10255"] + } + + dynamic "log_config" { + for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config] + content { + metadata = log_config.value.metadata + } + } +} + +resource "google_compute_firewall" "shadow_deny_exkubelet" { + count = var.add_shadow_firewall_rules ? 1 : 0 + + name = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-exkubelet" + description = "Managed by terraform GKE module: A shadow firewall rule to match the default deny rule to kubelet." + project = local.network_project_id + network = var.network + priority = var.shadow_firewall_rules_priority # rule created by GKE robot have prio 1000 + direction = "INGRESS" + + source_ranges = ["0.0.0.0/0"] + target_tags = [local.cluster_network_tag] + + deny { + protocol = "tcp" + ports = ["10255"] + } + + dynamic "log_config" { + for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config] + content { + metadata = log_config.value.metadata + } } } diff --git a/modules/beta-autopilot-public-cluster/main.tf b/modules/beta-autopilot-public-cluster/main.tf index 1e69f09336..6ec8dbcd69 100644 --- a/modules/beta-autopilot-public-cluster/main.tf +++ b/modules/beta-autopilot-public-cluster/main.tf @@ -45,7 +45,8 @@ locals { master_version_zonal = var.kubernetes_version != "latest" ? var.kubernetes_version : data.google_container_engine_versions.zone.latest_master_version master_version = var.regional ? local.master_version_regional : local.master_version_zonal - release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] + release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] + gateway_api_config = var.gateway_api_channel != null ? [{ channel : var.gateway_api_channel }] : [] @@ -60,6 +61,7 @@ locals { cluster_subnet_cidr = var.add_cluster_firewall_rules ? data.google_compute_subnetwork.gke_subnetwork[0].ip_cidr_range : null cluster_alias_ranges_cidr = var.add_cluster_firewall_rules ? { for range in toset(data.google_compute_subnetwork.gke_subnetwork[0].secondary_ip_range) : range.range_name => range.ip_cidr_range } : {} + pod_all_ip_ranges = var.add_cluster_firewall_rules ? [local.cluster_alias_ranges_cidr[var.ip_range_pods]] : [] cluster_authenticator_security_group = var.authenticator_security_group == null ? [] : [{ diff --git a/modules/beta-autopilot-public-cluster/outputs.tf b/modules/beta-autopilot-public-cluster/outputs.tf index 5ce92e94da..533f818844 100644 --- a/modules/beta-autopilot-public-cluster/outputs.tf +++ b/modules/beta-autopilot-public-cluster/outputs.tf @@ -129,6 +129,11 @@ output "release_channel" { value = var.release_channel } +output "gateway_api_channel" { + description = "The gateway api channel of this cluster." + value = var.gateway_api_channel +} + output "identity_namespace" { description = "Workload Identity pool" value = length(local.cluster_workload_identity_config) > 0 ? local.cluster_workload_identity_config[0].workload_pool : null diff --git a/modules/beta-autopilot-public-cluster/variables.tf b/modules/beta-autopilot-public-cluster/variables.tf index e465277f65..dd0abfccfb 100644 --- a/modules/beta-autopilot-public-cluster/variables.tf +++ b/modules/beta-autopilot-public-cluster/variables.tf @@ -293,6 +293,12 @@ variable "release_channel" { default = null } +variable "gateway_api_channel" { + type = string + description = "The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`." + default = null +} + variable "add_cluster_firewall_rules" { type = bool description = "Create additional firewall rules" @@ -327,6 +333,20 @@ variable "shadow_firewall_rules_priority" { type = number description = "The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000." default = 999 + validation { + condition = var.shadow_firewall_rules_priority < 1000 + error_message = "The shadow firewall rule priority must be lower than auto-created one(1000)." + } +} + +variable "shadow_firewall_rules_log_config" { + type = object({ + metadata = string + }) + description = "The log_config for shadow firewall rules. You can set this variable to `null` to disable logging." + default = { + metadata = "INCLUDE_ALL_METADATA" + } } variable "enable_confidential_nodes" { @@ -373,8 +393,3 @@ variable "timeouts" { } } -variable "monitoring_enable_managed_prometheus" { - type = bool - description = "(Beta) Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled." - default = false -} diff --git a/modules/beta-autopilot-public-cluster/versions.tf b/modules/beta-autopilot-public-cluster/versions.tf index a12bef4ece..b6e1f53519 100644 --- a/modules/beta-autopilot-public-cluster/versions.tf +++ b/modules/beta-autopilot-public-cluster/versions.tf @@ -21,7 +21,7 @@ terraform { required_providers { google-beta = { source = "hashicorp/google-beta" - version = ">= 4.42.0, < 5.0" + version = ">= 4.51.0, < 5.0" } kubernetes = { source = "hashicorp/kubernetes" @@ -29,6 +29,6 @@ terraform { } } provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:beta-autopilot-public-cluster/v24.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:beta-autopilot-public-cluster/v25.0.0" } } diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md index 80b7d787a4..d084f718f8 100644 --- a/modules/beta-private-cluster-update-variant/README.md +++ b/modules/beta-private-cluster-update-variant/README.md @@ -165,7 +165,7 @@ Then perform the following commands on the root folder: | authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | | cloudrun | (Beta) Enable CloudRun addon | `bool` | `false` | no | | cloudrun\_load\_balancer\_type | (Beta) Configure the Cloud Run load balancer type. External by default. Set to `LOAD_BALANCER_TYPE_INTERNAL` to configure as an internal load balancer. | `string` | `""` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources       = list(object({ resource_type = string, minimum = number, maximum = number }))
  })
| 
{
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources       = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair         = bool
    auto_upgrade        = bool
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | | cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | | cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | | cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | @@ -201,8 +201,9 @@ Then perform the following commands on the root folder: | filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | | firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | | firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | -| gce\_pd\_csi\_driver | (Beta) Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `false` | no | -| gke\_backup\_agent\_config | (Beta) Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | +| gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | +| gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | +| gke\_backup\_agent\_config | Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | | http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no | @@ -226,7 +227,7 @@ Then perform the following commands on the root folder: | master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | | master\_global\_access\_enabled | Whether the cluster master is accessible globally (from any region) or only within the same region as the private endpoint. | `bool` | `true` | no | | master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | `string` | `"10.0.0.0/28"` | no | -| monitoring\_enable\_managed\_prometheus | (Beta) Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `false` | no | +| monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `false` | no | | monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration. | `list(string)` | `[]` | no | | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | | name | The name of the cluster (required) | `string` | n/a | yes | @@ -240,6 +241,7 @@ Then perform the following commands on the root folder: | node\_pools\_linux\_node\_configs\_sysctls | Map of maps containing linux node config sysctls by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | | node\_pools\_metadata | Map of maps containing node metadata by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | | node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | `map(list(string))` | 
{
  "all": [
    "https://www.googleapis.com/auth/cloud-platform"
  ],
  "default-node-pool": []
}
| no | +| node\_pools\_resource\_labels | Map of maps containing resource labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | | node\_pools\_tags | Map of lists containing node network tags by node-pool name | `map(list(string))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | | node\_pools\_taints | Map of lists containing node taints by node-pool name | `map(list(object({ key = string, value = string, effect = string })))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | | non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | `list(string)` | 
[
  "10.0.0.0/8",
  "172.16.0.0/12",
  "192.168.0.0/16"
]
| no | @@ -254,6 +256,7 @@ Then perform the following commands on the root folder: | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it). | `bool` | `false` | no | | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. | `string` | `""` | no | | service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | +| shadow\_firewall\_rules\_log\_config | The log\_config for shadow firewall rules. You can set this variable to `null` to disable logging. | 
object({
    metadata = string
  })
| 
{
  "metadata": "INCLUDE_ALL_METADATA"
}
| no | | shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | | skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | `bool` | `false` | no | | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | @@ -272,6 +275,7 @@ Then perform the following commands on the root folder: | cluster\_id | Cluster ID | | dns\_cache\_enabled | Whether DNS Cache enabled | | endpoint | Cluster endpoint | +| gateway\_api\_channel | The gateway api channel of this cluster. | | horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | | http\_load\_balancing\_enabled | Whether http load balancing enabled | | identity\_namespace | Workload Identity pool | @@ -342,7 +346,7 @@ The node_pools variable takes the following parameters: | min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count. Should be used when autoscaling is true | 1 | Optional | | name | The name of the node pool | | Required | | placement_policy | Placement type to set for nodes in a node pool. Can be set as [COMPACT](https://cloud.google.com/kubernetes-engine/docs/how-to/compact-placement#overview) if desired | Optional | -| pod_range | The ID of the secondary range for pod IPs. | | Optional | +| pod_range | The name of the secondary range for pod IPs. | | Optional | | node_count | The number of nodes in the nodepool when autoscaling is false. Otherwise defaults to 1. Only valid for non-autoscaling clusters | | Required | | node_locations | The list of zones in which the cluster's nodes are located. Nodes must be in the region of their regional cluster or in the same region as their cluster's zone for zonal clusters. Defaults to cluster level node locations if nothing is specified | " " | Optional | | node_metadata | Options to expose the node metadata to the workload running on the node | | Optional | @@ -353,6 +357,7 @@ The node_pools variable takes the following parameters: | tags | The list of instance tags applied to all nodes | | Required | | value | The value for the taint | | Required | | version | The Kubernetes version for the nodes in this pool. Should only be set if auto_upgrade is false | " " | Optional | +| location_policy | [Location policy](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_node_pool#location_policy) specifies the algorithm used when scaling-up the node pool. Location policy is supported only in 1.24.1+ clusters. | " " | Optional | ## windows_node_pools variable The windows_node_pools variable takes the same parameters as [node_pools](#node\_pools-variable) but is reserved for provisioning Windows based node pools only. This variable is introduced to satisfy a [specific requirement](https://cloud.google.com/kubernetes-engine/docs/how-to/creating-a-cluster-windows#create_a_cluster_and_node_pools) for the presence of at least one linux based node pool in the cluster before a windows based node pool can be created. @@ -373,8 +378,8 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog #### Kubectl - [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x #### Terraform and Plugins -- [Terraform](https://www.terraform.io/downloads.html) 0.12 -- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v3.41 +- [Terraform](https://www.terraform.io/downloads.html) 0.13+ +- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v4.51 #### gcloud Some submodules use the [terraform-google-gcloud](https://github.com/terraform-google-modules/terraform-google-gcloud) module. By default, this module assumes you already have gcloud installed in your $PATH. See the [module](https://github.com/terraform-google-modules/terraform-google-gcloud#downloading) documentation for more information. diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf index 65ad656374..5eb7470a18 100644 --- a/modules/beta-private-cluster-update-variant/cluster.tf +++ b/modules/beta-private-cluster-update-variant/cluster.tf @@ -47,6 +47,15 @@ resource "google_container_cluster" "primary" { channel = release_channel.value.channel } } + + dynamic "gateway_api_config" { + for_each = local.gateway_api_config + + content { + channel = gateway_api_config.value.channel + } + } + dynamic "cost_management_config" { for_each = var.enable_cost_allocation ? [1] : [] content { @@ -88,7 +97,7 @@ resource "google_container_cluster" "primary" { for_each = length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus ? [1] : [] content { - enable_components = length(var.monitoring_enabled_components) > 0 ? var.monitoring_enabled_components : null + enable_components = length(var.monitoring_enabled_components) > 0 ? var.monitoring_enabled_components : [] dynamic "managed_prometheus" { for_each = var.monitoring_enable_managed_prometheus ? [1] : [] @@ -105,8 +114,14 @@ resource "google_container_cluster" "primary" { for_each = var.cluster_autoscaling.enabled ? [1] : [] content { - service_account = local.service_account - oauth_scopes = local.node_pools_oauth_scopes["all"] + service_account = local.service_account + oauth_scopes = local.node_pools_oauth_scopes["all"] + + management { + auto_repair = lookup(var.cluster_autoscaling, "auto_repair", true) + auto_upgrade = lookup(var.cluster_autoscaling, "auto_upgrade", true) + } + min_cpu_platform = lookup(var.node_pools[0], "min_cpu_platform", "") } } @@ -133,8 +148,9 @@ resource "google_container_cluster" "primary" { } } + enable_kubernetes_alpha = var.enable_kubernetes_alpha + enable_intranode_visibility = var.enable_intranode_visibility - enable_kubernetes_alpha = var.enable_kubernetes_alpha enable_tpu = var.enable_tpu dynamic "pod_security_policy_config" { @@ -187,7 +203,6 @@ resource "google_container_cluster" "primary" { disabled = !var.horizontal_pod_autoscaling } - network_policy_config { disabled = !var.network_policy } @@ -200,6 +215,22 @@ resource "google_container_cluster" "primary" { enabled = var.filestore_csi_driver } + dynamic "gce_persistent_disk_csi_driver_config" { + for_each = local.cluster_gce_pd_csi_config + + content { + enabled = gce_persistent_disk_csi_driver_config.value.enabled + } + } + + dynamic "gke_backup_agent_config" { + for_each = local.gke_backup_agent_config + + content { + enabled = gke_backup_agent_config.value.enabled + } + } + istio_config { disabled = !var.istio auth = var.istio_auth @@ -213,14 +244,6 @@ resource "google_container_cluster" "primary" { } } - dynamic "gce_persistent_disk_csi_driver_config" { - for_each = local.cluster_gce_pd_csi_config - - content { - enabled = gce_persistent_disk_csi_driver_config.value.enabled - } - } - kalm_config { enabled = var.kalm_config } @@ -228,14 +251,6 @@ resource "google_container_cluster" "primary" { config_connector_config { enabled = var.config_connector } - - dynamic "gke_backup_agent_config" { - for_each = local.gke_backup_agent_config - - content { - enabled = gke_backup_agent_config.value.enabled - } - } } datapath_provider = var.datapath_provider @@ -569,7 +584,8 @@ resource "google_container_node_pool" "pools" { dynamic "network_config" { for_each = length(lookup(each.value, "pod_range", "")) > 0 ? [each.value] : [] content { - pod_range = lookup(network_config.value, "pod_range", null) + pod_range = lookup(network_config.value, "pod_range", null) + enable_private_nodes = var.enable_private_nodes } } @@ -605,6 +621,10 @@ resource "google_container_node_pool" "pools" { local.node_pools_labels["all"], local.node_pools_labels[each.value["name"]], ) + resource_labels = merge( + local.node_pools_resource_labels["all"], + local.node_pools_resource_labels[each.value["name"]], + ) metadata = merge( lookup(lookup(local.node_pools_metadata, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_metadata, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, @@ -776,7 +796,8 @@ resource "google_container_node_pool" "windows_pools" { dynamic "network_config" { for_each = length(lookup(each.value, "pod_range", "")) > 0 ? [each.value] : [] content { - pod_range = lookup(network_config.value, "pod_range", null) + pod_range = lookup(network_config.value, "pod_range", null) + enable_private_nodes = var.enable_private_nodes } } @@ -812,6 +833,10 @@ resource "google_container_node_pool" "windows_pools" { local.node_pools_labels["all"], local.node_pools_labels[each.value["name"]], ) + resource_labels = merge( + local.node_pools_resource_labels["all"], + local.node_pools_resource_labels[each.value["name"]], + ) metadata = merge( lookup(lookup(local.node_pools_metadata, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_metadata, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, diff --git a/modules/beta-private-cluster-update-variant/firewall.tf b/modules/beta-private-cluster-update-variant/firewall.tf index 14c3e54cb0..96eecab803 100644 --- a/modules/beta-private-cluster-update-variant/firewall.tf +++ b/modules/beta-private-cluster-update-variant/firewall.tf @@ -34,11 +34,12 @@ resource "google_compute_firewall" "intra_egress" { direction = "EGRESS" target_tags = [local.cluster_network_tag] - destination_ranges = [ + destination_ranges = concat([ local.cluster_endpoint_for_nodes, local.cluster_subnet_cidr, - local.cluster_alias_ranges_cidr[var.ip_range_pods], - ] + ], + local.pod_all_ip_ranges + ) # Allow all possible protocols allow { protocol = "tcp" } @@ -126,7 +127,7 @@ resource "google_compute_firewall" "shadow_allow_pods" { priority = var.shadow_firewall_rules_priority direction = "INGRESS" - source_ranges = [local.cluster_alias_ranges_cidr[var.ip_range_pods]] + source_ranges = local.pod_all_ip_ranges target_tags = [local.cluster_network_tag] # Allow all possible protocols @@ -137,8 +138,11 @@ resource "google_compute_firewall" "shadow_allow_pods" { allow { protocol = "esp" } allow { protocol = "ah" } - log_config { - metadata = "INCLUDE_ALL_METADATA" + dynamic "log_config" { + for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config] + content { + metadata = log_config.value.metadata + } } } @@ -160,8 +164,11 @@ resource "google_compute_firewall" "shadow_allow_master" { ports = ["10250", "443"] } - log_config { - metadata = "INCLUDE_ALL_METADATA" + dynamic "log_config" { + for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config] + content { + metadata = log_config.value.metadata + } } } @@ -192,7 +199,63 @@ resource "google_compute_firewall" "shadow_allow_nodes" { ports = ["1-65535"] } - log_config { - metadata = "INCLUDE_ALL_METADATA" + dynamic "log_config" { + for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config] + content { + metadata = log_config.value.metadata + } + } +} + +resource "google_compute_firewall" "shadow_allow_inkubelet" { + count = var.add_shadow_firewall_rules ? 1 : 0 + + name = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-inkubelet" + description = "Managed by terraform GKE module: A shadow firewall rule to match the default rule allowing worker nodes & pods communication to kubelet." + project = local.network_project_id + network = var.network + priority = var.shadow_firewall_rules_priority - 1 # rule created by GKE robot have prio 999 + direction = "INGRESS" + + source_ranges = local.pod_all_ip_ranges + source_tags = [local.cluster_network_tag] + target_tags = [local.cluster_network_tag] + + allow { + protocol = "tcp" + ports = ["10255"] + } + + dynamic "log_config" { + for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config] + content { + metadata = log_config.value.metadata + } + } +} + +resource "google_compute_firewall" "shadow_deny_exkubelet" { + count = var.add_shadow_firewall_rules ? 1 : 0 + + name = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-exkubelet" + description = "Managed by terraform GKE module: A shadow firewall rule to match the default deny rule to kubelet." + project = local.network_project_id + network = var.network + priority = var.shadow_firewall_rules_priority # rule created by GKE robot have prio 1000 + direction = "INGRESS" + + source_ranges = ["0.0.0.0/0"] + target_tags = [local.cluster_network_tag] + + deny { + protocol = "tcp" + ports = ["10255"] + } + + dynamic "log_config" { + for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config] + content { + metadata = log_config.value.metadata + } } } diff --git a/modules/beta-private-cluster-update-variant/main.tf b/modules/beta-private-cluster-update-variant/main.tf index 7f44dcec3b..975390944e 100644 --- a/modules/beta-private-cluster-update-variant/main.tf +++ b/modules/beta-private-cluster-update-variant/main.tf @@ -50,7 +50,8 @@ locals { windows_node_pool_names = [for np in toset(var.windows_node_pools) : np.name] windows_node_pools = zipmap(local.windows_node_pool_names, tolist(toset(var.windows_node_pools))) - release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] + release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] + gateway_api_config = var.gateway_api_channel != null ? [{ channel : var.gateway_api_channel }] : [] autoscaling_resource_limits = var.cluster_autoscaling.enabled ? concat([{ resource_type = "cpu" @@ -74,6 +75,7 @@ locals { cluster_subnet_cidr = var.add_cluster_firewall_rules ? data.google_compute_subnetwork.gke_subnetwork[0].ip_cidr_range : null cluster_alias_ranges_cidr = var.add_cluster_firewall_rules ? { for range in toset(data.google_compute_subnetwork.gke_subnetwork[0].secondary_ip_range) : range.range_name => range.ip_cidr_range } : {} + pod_all_ip_ranges = var.add_cluster_firewall_rules ? compact(concat([local.cluster_alias_ranges_cidr[var.ip_range_pods]], [for k, v in merge(local.node_pools, local.windows_node_pools) : local.cluster_alias_ranges_cidr[v.pod_range] if length(lookup(v, "pod_range", "")) > 0])) : [] cluster_network_policy = var.network_policy ? [{ enabled = true @@ -82,6 +84,9 @@ locals { enabled = false provider = null }] + cluster_gce_pd_csi_config = var.gce_pd_csi_driver ? [{ enabled = true }] : [{ enabled = false }] + logmon_config_is_set = length(var.logging_enabled_components) > 0 || length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus + gke_backup_agent_config = var.gke_backup_agent_config ? [{ enabled = true }] : [{ enabled = false }] cluster_cloudrun_config_load_balancer_config = (var.cloudrun && var.cloudrun_load_balancer_type != "") ? { load_balancer_type = var.cloudrun_load_balancer_type } : {} @@ -93,10 +98,7 @@ locals { local.cluster_cloudrun_config_load_balancer_config ) ] : [] - cluster_cloudrun_enabled = var.cloudrun - cluster_gce_pd_csi_config = var.gce_pd_csi_driver ? [{ enabled = true }] : [{ enabled = false }] - gke_backup_agent_config = var.gke_backup_agent_config ? [{ enabled = true }] : [{ enabled = false }] - logmon_config_is_set = length(var.logging_enabled_components) > 0 || length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus + cluster_cloudrun_enabled = var.cloudrun cluster_authenticator_security_group = var.authenticator_security_group == null ? [] : [{ security_group = var.authenticator_security_group diff --git a/modules/beta-private-cluster-update-variant/outputs.tf b/modules/beta-private-cluster-update-variant/outputs.tf index afc6c55618..abdf16f900 100644 --- a/modules/beta-private-cluster-update-variant/outputs.tf +++ b/modules/beta-private-cluster-update-variant/outputs.tf @@ -148,6 +148,11 @@ output "release_channel" { value = var.release_channel } +output "gateway_api_channel" { + description = "The gateway api channel of this cluster." + value = var.gateway_api_channel +} + output "identity_namespace" { description = "Workload Identity pool" value = length(local.cluster_workload_identity_config) > 0 ? local.cluster_workload_identity_config[0].workload_pool : null diff --git a/modules/beta-private-cluster-update-variant/variables.tf b/modules/beta-private-cluster-update-variant/variables.tf index de952e9195..7fa57dfe6d 100644 --- a/modules/beta-private-cluster-update-variant/variables.tf +++ b/modules/beta-private-cluster-update-variant/variables.tf @@ -170,6 +170,16 @@ variable "node_pools_labels" { } } +variable "node_pools_resource_labels" { + type = map(map(string)) + description = "Map of maps containing resource labels by node-pool name" + + default = { + all = {} + default-node-pool = {} + } +} + variable "node_pools_metadata" { type = map(map(string)) description = "Map of maps containing node metadata by node-pool name" @@ -224,6 +234,8 @@ variable "cluster_autoscaling" { min_memory_gb = number max_memory_gb = number gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) + auto_repair = bool + auto_upgrade = bool }) default = { enabled = false @@ -233,6 +245,8 @@ variable "cluster_autoscaling" { max_memory_gb = 0 min_memory_gb = 0 gpu_resources = [] + auto_repair = true + auto_upgrade = true } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } @@ -426,6 +440,12 @@ variable "release_channel" { default = null } +variable "gateway_api_channel" { + type = string + description = "The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`." + default = null +} + variable "add_cluster_firewall_rules" { type = bool description = "Create additional firewall rules" @@ -460,6 +480,20 @@ variable "shadow_firewall_rules_priority" { type = number description = "The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000." default = 999 + validation { + condition = var.shadow_firewall_rules_priority < 1000 + error_message = "The shadow firewall rule priority must be lower than auto-created one(1000)." + } +} + +variable "shadow_firewall_rules_log_config" { + type = object({ + metadata = string + }) + description = "The log_config for shadow firewall rules. You can set this variable to `null` to disable logging." + default = { + metadata = "INCLUDE_ALL_METADATA" + } } variable "enable_confidential_nodes" { @@ -556,7 +590,7 @@ variable "node_metadata" { validation { condition = contains(["GKE_METADATA", "GCE_METADATA", "UNSPECIFIED", "GKE_METADATA_SERVER", "EXPOSE"], var.node_metadata) - error_message = "The node_metadata value must be one of GKE_METADATA, GCE_METADATA or UNSPECIFIED." + error_message = "The node_metadata value must be one of GKE_METADATA, GCE_METADATA, UNSPECIFIED, GKE_METADATA_SERVER or EXPOSE." } } @@ -578,6 +612,18 @@ variable "cluster_dns_domain" { default = "" } +variable "gce_pd_csi_driver" { + type = bool + description = "Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver." + default = true +} + +variable "gke_backup_agent_config" { + type = bool + description = "Whether Backup for GKE agent is enabled for this cluster." + default = false +} + variable "timeouts" { type = map(string) description = "Timeout for cluster operations." @@ -588,27 +634,27 @@ variable "timeouts" { } } -variable "enable_kubernetes_alpha" { +variable "monitoring_enable_managed_prometheus" { type = bool - description = "Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days." + description = "Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled." default = false } -variable "logging_enabled_components" { +variable "monitoring_enabled_components" { type = list(string) - description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." + description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration." default = [] } -variable "monitoring_enabled_components" { +variable "logging_enabled_components" { type = list(string) - description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration." + description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." default = [] } -variable "monitoring_enable_managed_prometheus" { +variable "enable_kubernetes_alpha" { type = bool - description = "(Beta) Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled." + description = "Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days." default = false } @@ -635,12 +681,6 @@ variable "config_connector" { default = false } -variable "gke_backup_agent_config" { - type = bool - description = "(Beta) Whether Backup for GKE agent is enabled for this cluster." - default = false -} - variable "cloudrun" { description = "(Beta) Enable CloudRun addon" default = false @@ -681,9 +721,3 @@ variable "enable_identity_service" { description = "Enable the Identity Service component, which allows customers to use external identity providers with the K8S API." default = false } - -variable "gce_pd_csi_driver" { - type = bool - description = "(Beta) Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver." - default = false -} diff --git a/modules/beta-private-cluster-update-variant/variables_defaults.tf b/modules/beta-private-cluster-update-variant/variables_defaults.tf index cc65ac9e8b..e4f3004771 100644 --- a/modules/beta-private-cluster-update-variant/variables_defaults.tf +++ b/modules/beta-private-cluster-update-variant/variables_defaults.tf @@ -34,6 +34,20 @@ locals { var.node_pools_labels ) + node_pools_resource_labels = merge( + { all = {} }, + { default-node-pool = {} }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : {}] + ), + zipmap( + [for node_pool in var.windows_node_pools : node_pool["name"]], + [for node_pool in var.windows_node_pools : {}] + ), + var.node_pools_resource_labels + ) + node_pools_metadata = merge( { all = {} }, { default-node-pool = {} }, diff --git a/modules/beta-private-cluster-update-variant/versions.tf b/modules/beta-private-cluster-update-variant/versions.tf index fe3f5580db..878a4834fe 100644 --- a/modules/beta-private-cluster-update-variant/versions.tf +++ b/modules/beta-private-cluster-update-variant/versions.tf @@ -21,7 +21,7 @@ terraform { required_providers { google-beta = { source = "hashicorp/google-beta" - version = ">= 4.42.0, < 5.0" + version = ">= 4.51.0, < 5.0" } kubernetes = { source = "hashicorp/kubernetes" @@ -29,6 +29,6 @@ terraform { } } provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:beta-private-cluster-update-variant/v24.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:beta-private-cluster-update-variant/v25.0.0" } } diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md index 06f0beb5c6..dde80c4623 100644 --- a/modules/beta-private-cluster/README.md +++ b/modules/beta-private-cluster/README.md @@ -143,7 +143,7 @@ Then perform the following commands on the root folder: | authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | | cloudrun | (Beta) Enable CloudRun addon | `bool` | `false` | no | | cloudrun\_load\_balancer\_type | (Beta) Configure the Cloud Run load balancer type. External by default. Set to `LOAD_BALANCER_TYPE_INTERNAL` to configure as an internal load balancer. | `string` | `""` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources       = list(object({ resource_type = string, minimum = number, maximum = number }))
  })
| 
{
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources       = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair         = bool
    auto_upgrade        = bool
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | | cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | | cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | | cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | @@ -179,8 +179,9 @@ Then perform the following commands on the root folder: | filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | | firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | | firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | -| gce\_pd\_csi\_driver | (Beta) Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `false` | no | -| gke\_backup\_agent\_config | (Beta) Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | +| gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | +| gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | +| gke\_backup\_agent\_config | Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | | http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no | @@ -204,7 +205,7 @@ Then perform the following commands on the root folder: | master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | | master\_global\_access\_enabled | Whether the cluster master is accessible globally (from any region) or only within the same region as the private endpoint. | `bool` | `true` | no | | master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | `string` | `"10.0.0.0/28"` | no | -| monitoring\_enable\_managed\_prometheus | (Beta) Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `false` | no | +| monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `false` | no | | monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration. | `list(string)` | `[]` | no | | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | | name | The name of the cluster (required) | `string` | n/a | yes | @@ -218,6 +219,7 @@ Then perform the following commands on the root folder: | node\_pools\_linux\_node\_configs\_sysctls | Map of maps containing linux node config sysctls by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | | node\_pools\_metadata | Map of maps containing node metadata by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | | node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | `map(list(string))` | 
{
  "all": [
    "https://www.googleapis.com/auth/cloud-platform"
  ],
  "default-node-pool": []
}
| no | +| node\_pools\_resource\_labels | Map of maps containing resource labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | | node\_pools\_tags | Map of lists containing node network tags by node-pool name | `map(list(string))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | | node\_pools\_taints | Map of lists containing node taints by node-pool name | `map(list(object({ key = string, value = string, effect = string })))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | | non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | `list(string)` | 
[
  "10.0.0.0/8",
  "172.16.0.0/12",
  "192.168.0.0/16"
]
| no | @@ -232,6 +234,7 @@ Then perform the following commands on the root folder: | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it). | `bool` | `false` | no | | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. | `string` | `""` | no | | service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | +| shadow\_firewall\_rules\_log\_config | The log\_config for shadow firewall rules. You can set this variable to `null` to disable logging. | 
object({
    metadata = string
  })
| 
{
  "metadata": "INCLUDE_ALL_METADATA"
}
| no | | shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | | skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | `bool` | `false` | no | | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | @@ -250,6 +253,7 @@ Then perform the following commands on the root folder: | cluster\_id | Cluster ID | | dns\_cache\_enabled | Whether DNS Cache enabled | | endpoint | Cluster endpoint | +| gateway\_api\_channel | The gateway api channel of this cluster. | | horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | | http\_load\_balancing\_enabled | Whether http load balancing enabled | | identity\_namespace | Workload Identity pool | @@ -320,7 +324,7 @@ The node_pools variable takes the following parameters: | min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count. Should be used when autoscaling is true | 1 | Optional | | name | The name of the node pool | | Required | | placement_policy | Placement type to set for nodes in a node pool. Can be set as [COMPACT](https://cloud.google.com/kubernetes-engine/docs/how-to/compact-placement#overview) if desired | Optional | -| pod_range | The ID of the secondary range for pod IPs. | | Optional | +| pod_range | The name of the secondary range for pod IPs. | | Optional | | node_count | The number of nodes in the nodepool when autoscaling is false. Otherwise defaults to 1. Only valid for non-autoscaling clusters | | Required | | node_locations | The list of zones in which the cluster's nodes are located. Nodes must be in the region of their regional cluster or in the same region as their cluster's zone for zonal clusters. Defaults to cluster level node locations if nothing is specified | " " | Optional | | node_metadata | Options to expose the node metadata to the workload running on the node | | Optional | @@ -331,6 +335,7 @@ The node_pools variable takes the following parameters: | tags | The list of instance tags applied to all nodes | | Required | | value | The value for the taint | | Required | | version | The Kubernetes version for the nodes in this pool. Should only be set if auto_upgrade is false | " " | Optional | +| location_policy | [Location policy](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_node_pool#location_policy) specifies the algorithm used when scaling-up the node pool. Location policy is supported only in 1.24.1+ clusters. | " " | Optional | ## windows_node_pools variable The windows_node_pools variable takes the same parameters as [node_pools](#node\_pools-variable) but is reserved for provisioning Windows based node pools only. This variable is introduced to satisfy a [specific requirement](https://cloud.google.com/kubernetes-engine/docs/how-to/creating-a-cluster-windows#create_a_cluster_and_node_pools) for the presence of at least one linux based node pool in the cluster before a windows based node pool can be created. @@ -351,8 +356,8 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog #### Kubectl - [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x #### Terraform and Plugins -- [Terraform](https://www.terraform.io/downloads.html) 0.12 -- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v3.41 +- [Terraform](https://www.terraform.io/downloads.html) 0.13+ +- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v4.51 #### gcloud Some submodules use the [terraform-google-gcloud](https://github.com/terraform-google-modules/terraform-google-gcloud) module. By default, this module assumes you already have gcloud installed in your $PATH. See the [module](https://github.com/terraform-google-modules/terraform-google-gcloud#downloading) documentation for more information. diff --git a/modules/beta-private-cluster/cluster.tf b/modules/beta-private-cluster/cluster.tf index 91210d6847..bd72dc97c9 100644 --- a/modules/beta-private-cluster/cluster.tf +++ b/modules/beta-private-cluster/cluster.tf @@ -47,6 +47,15 @@ resource "google_container_cluster" "primary" { channel = release_channel.value.channel } } + + dynamic "gateway_api_config" { + for_each = local.gateway_api_config + + content { + channel = gateway_api_config.value.channel + } + } + dynamic "cost_management_config" { for_each = var.enable_cost_allocation ? [1] : [] content { @@ -88,7 +97,7 @@ resource "google_container_cluster" "primary" { for_each = length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus ? [1] : [] content { - enable_components = length(var.monitoring_enabled_components) > 0 ? var.monitoring_enabled_components : null + enable_components = length(var.monitoring_enabled_components) > 0 ? var.monitoring_enabled_components : [] dynamic "managed_prometheus" { for_each = var.monitoring_enable_managed_prometheus ? [1] : [] @@ -105,8 +114,14 @@ resource "google_container_cluster" "primary" { for_each = var.cluster_autoscaling.enabled ? [1] : [] content { - service_account = local.service_account - oauth_scopes = local.node_pools_oauth_scopes["all"] + service_account = local.service_account + oauth_scopes = local.node_pools_oauth_scopes["all"] + + management { + auto_repair = lookup(var.cluster_autoscaling, "auto_repair", true) + auto_upgrade = lookup(var.cluster_autoscaling, "auto_upgrade", true) + } + min_cpu_platform = lookup(var.node_pools[0], "min_cpu_platform", "") } } @@ -133,8 +148,9 @@ resource "google_container_cluster" "primary" { } } + enable_kubernetes_alpha = var.enable_kubernetes_alpha + enable_intranode_visibility = var.enable_intranode_visibility - enable_kubernetes_alpha = var.enable_kubernetes_alpha enable_tpu = var.enable_tpu dynamic "pod_security_policy_config" { @@ -187,7 +203,6 @@ resource "google_container_cluster" "primary" { disabled = !var.horizontal_pod_autoscaling } - network_policy_config { disabled = !var.network_policy } @@ -200,6 +215,22 @@ resource "google_container_cluster" "primary" { enabled = var.filestore_csi_driver } + dynamic "gce_persistent_disk_csi_driver_config" { + for_each = local.cluster_gce_pd_csi_config + + content { + enabled = gce_persistent_disk_csi_driver_config.value.enabled + } + } + + dynamic "gke_backup_agent_config" { + for_each = local.gke_backup_agent_config + + content { + enabled = gke_backup_agent_config.value.enabled + } + } + istio_config { disabled = !var.istio auth = var.istio_auth @@ -213,14 +244,6 @@ resource "google_container_cluster" "primary" { } } - dynamic "gce_persistent_disk_csi_driver_config" { - for_each = local.cluster_gce_pd_csi_config - - content { - enabled = gce_persistent_disk_csi_driver_config.value.enabled - } - } - kalm_config { enabled = var.kalm_config } @@ -228,14 +251,6 @@ resource "google_container_cluster" "primary" { config_connector_config { enabled = var.config_connector } - - dynamic "gke_backup_agent_config" { - for_each = local.gke_backup_agent_config - - content { - enabled = gke_backup_agent_config.value.enabled - } - } } datapath_provider = var.datapath_provider @@ -475,7 +490,8 @@ resource "google_container_node_pool" "pools" { dynamic "network_config" { for_each = length(lookup(each.value, "pod_range", "")) > 0 ? [each.value] : [] content { - pod_range = lookup(network_config.value, "pod_range", null) + pod_range = lookup(network_config.value, "pod_range", null) + enable_private_nodes = var.enable_private_nodes } } @@ -511,6 +527,10 @@ resource "google_container_node_pool" "pools" { local.node_pools_labels["all"], local.node_pools_labels[each.value["name"]], ) + resource_labels = merge( + local.node_pools_resource_labels["all"], + local.node_pools_resource_labels[each.value["name"]], + ) metadata = merge( lookup(lookup(local.node_pools_metadata, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_metadata, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, @@ -681,7 +701,8 @@ resource "google_container_node_pool" "windows_pools" { dynamic "network_config" { for_each = length(lookup(each.value, "pod_range", "")) > 0 ? [each.value] : [] content { - pod_range = lookup(network_config.value, "pod_range", null) + pod_range = lookup(network_config.value, "pod_range", null) + enable_private_nodes = var.enable_private_nodes } } @@ -717,6 +738,10 @@ resource "google_container_node_pool" "windows_pools" { local.node_pools_labels["all"], local.node_pools_labels[each.value["name"]], ) + resource_labels = merge( + local.node_pools_resource_labels["all"], + local.node_pools_resource_labels[each.value["name"]], + ) metadata = merge( lookup(lookup(local.node_pools_metadata, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_metadata, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, diff --git a/modules/beta-private-cluster/firewall.tf b/modules/beta-private-cluster/firewall.tf index 14c3e54cb0..96eecab803 100644 --- a/modules/beta-private-cluster/firewall.tf +++ b/modules/beta-private-cluster/firewall.tf @@ -34,11 +34,12 @@ resource "google_compute_firewall" "intra_egress" { direction = "EGRESS" target_tags = [local.cluster_network_tag] - destination_ranges = [ + destination_ranges = concat([ local.cluster_endpoint_for_nodes, local.cluster_subnet_cidr, - local.cluster_alias_ranges_cidr[var.ip_range_pods], - ] + ], + local.pod_all_ip_ranges + ) # Allow all possible protocols allow { protocol = "tcp" } @@ -126,7 +127,7 @@ resource "google_compute_firewall" "shadow_allow_pods" { priority = var.shadow_firewall_rules_priority direction = "INGRESS" - source_ranges = [local.cluster_alias_ranges_cidr[var.ip_range_pods]] + source_ranges = local.pod_all_ip_ranges target_tags = [local.cluster_network_tag] # Allow all possible protocols @@ -137,8 +138,11 @@ resource "google_compute_firewall" "shadow_allow_pods" { allow { protocol = "esp" } allow { protocol = "ah" } - log_config { - metadata = "INCLUDE_ALL_METADATA" + dynamic "log_config" { + for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config] + content { + metadata = log_config.value.metadata + } } } @@ -160,8 +164,11 @@ resource "google_compute_firewall" "shadow_allow_master" { ports = ["10250", "443"] } - log_config { - metadata = "INCLUDE_ALL_METADATA" + dynamic "log_config" { + for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config] + content { + metadata = log_config.value.metadata + } } } @@ -192,7 +199,63 @@ resource "google_compute_firewall" "shadow_allow_nodes" { ports = ["1-65535"] } - log_config { - metadata = "INCLUDE_ALL_METADATA" + dynamic "log_config" { + for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config] + content { + metadata = log_config.value.metadata + } + } +} + +resource "google_compute_firewall" "shadow_allow_inkubelet" { + count = var.add_shadow_firewall_rules ? 1 : 0 + + name = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-inkubelet" + description = "Managed by terraform GKE module: A shadow firewall rule to match the default rule allowing worker nodes & pods communication to kubelet." + project = local.network_project_id + network = var.network + priority = var.shadow_firewall_rules_priority - 1 # rule created by GKE robot have prio 999 + direction = "INGRESS" + + source_ranges = local.pod_all_ip_ranges + source_tags = [local.cluster_network_tag] + target_tags = [local.cluster_network_tag] + + allow { + protocol = "tcp" + ports = ["10255"] + } + + dynamic "log_config" { + for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config] + content { + metadata = log_config.value.metadata + } + } +} + +resource "google_compute_firewall" "shadow_deny_exkubelet" { + count = var.add_shadow_firewall_rules ? 1 : 0 + + name = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-exkubelet" + description = "Managed by terraform GKE module: A shadow firewall rule to match the default deny rule to kubelet." + project = local.network_project_id + network = var.network + priority = var.shadow_firewall_rules_priority # rule created by GKE robot have prio 1000 + direction = "INGRESS" + + source_ranges = ["0.0.0.0/0"] + target_tags = [local.cluster_network_tag] + + deny { + protocol = "tcp" + ports = ["10255"] + } + + dynamic "log_config" { + for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config] + content { + metadata = log_config.value.metadata + } } } diff --git a/modules/beta-private-cluster/main.tf b/modules/beta-private-cluster/main.tf index 7f44dcec3b..975390944e 100644 --- a/modules/beta-private-cluster/main.tf +++ b/modules/beta-private-cluster/main.tf @@ -50,7 +50,8 @@ locals { windows_node_pool_names = [for np in toset(var.windows_node_pools) : np.name] windows_node_pools = zipmap(local.windows_node_pool_names, tolist(toset(var.windows_node_pools))) - release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] + release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] + gateway_api_config = var.gateway_api_channel != null ? [{ channel : var.gateway_api_channel }] : [] autoscaling_resource_limits = var.cluster_autoscaling.enabled ? concat([{ resource_type = "cpu" @@ -74,6 +75,7 @@ locals { cluster_subnet_cidr = var.add_cluster_firewall_rules ? data.google_compute_subnetwork.gke_subnetwork[0].ip_cidr_range : null cluster_alias_ranges_cidr = var.add_cluster_firewall_rules ? { for range in toset(data.google_compute_subnetwork.gke_subnetwork[0].secondary_ip_range) : range.range_name => range.ip_cidr_range } : {} + pod_all_ip_ranges = var.add_cluster_firewall_rules ? compact(concat([local.cluster_alias_ranges_cidr[var.ip_range_pods]], [for k, v in merge(local.node_pools, local.windows_node_pools) : local.cluster_alias_ranges_cidr[v.pod_range] if length(lookup(v, "pod_range", "")) > 0])) : [] cluster_network_policy = var.network_policy ? [{ enabled = true @@ -82,6 +84,9 @@ locals { enabled = false provider = null }] + cluster_gce_pd_csi_config = var.gce_pd_csi_driver ? [{ enabled = true }] : [{ enabled = false }] + logmon_config_is_set = length(var.logging_enabled_components) > 0 || length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus + gke_backup_agent_config = var.gke_backup_agent_config ? [{ enabled = true }] : [{ enabled = false }] cluster_cloudrun_config_load_balancer_config = (var.cloudrun && var.cloudrun_load_balancer_type != "") ? { load_balancer_type = var.cloudrun_load_balancer_type } : {} @@ -93,10 +98,7 @@ locals { local.cluster_cloudrun_config_load_balancer_config ) ] : [] - cluster_cloudrun_enabled = var.cloudrun - cluster_gce_pd_csi_config = var.gce_pd_csi_driver ? [{ enabled = true }] : [{ enabled = false }] - gke_backup_agent_config = var.gke_backup_agent_config ? [{ enabled = true }] : [{ enabled = false }] - logmon_config_is_set = length(var.logging_enabled_components) > 0 || length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus + cluster_cloudrun_enabled = var.cloudrun cluster_authenticator_security_group = var.authenticator_security_group == null ? [] : [{ security_group = var.authenticator_security_group diff --git a/modules/beta-private-cluster/outputs.tf b/modules/beta-private-cluster/outputs.tf index afc6c55618..abdf16f900 100644 --- a/modules/beta-private-cluster/outputs.tf +++ b/modules/beta-private-cluster/outputs.tf @@ -148,6 +148,11 @@ output "release_channel" { value = var.release_channel } +output "gateway_api_channel" { + description = "The gateway api channel of this cluster." + value = var.gateway_api_channel +} + output "identity_namespace" { description = "Workload Identity pool" value = length(local.cluster_workload_identity_config) > 0 ? local.cluster_workload_identity_config[0].workload_pool : null diff --git a/modules/beta-private-cluster/variables.tf b/modules/beta-private-cluster/variables.tf index de952e9195..7fa57dfe6d 100644 --- a/modules/beta-private-cluster/variables.tf +++ b/modules/beta-private-cluster/variables.tf @@ -170,6 +170,16 @@ variable "node_pools_labels" { } } +variable "node_pools_resource_labels" { + type = map(map(string)) + description = "Map of maps containing resource labels by node-pool name" + + default = { + all = {} + default-node-pool = {} + } +} + variable "node_pools_metadata" { type = map(map(string)) description = "Map of maps containing node metadata by node-pool name" @@ -224,6 +234,8 @@ variable "cluster_autoscaling" { min_memory_gb = number max_memory_gb = number gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) + auto_repair = bool + auto_upgrade = bool }) default = { enabled = false @@ -233,6 +245,8 @@ variable "cluster_autoscaling" { max_memory_gb = 0 min_memory_gb = 0 gpu_resources = [] + auto_repair = true + auto_upgrade = true } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } @@ -426,6 +440,12 @@ variable "release_channel" { default = null } +variable "gateway_api_channel" { + type = string + description = "The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`." + default = null +} + variable "add_cluster_firewall_rules" { type = bool description = "Create additional firewall rules" @@ -460,6 +480,20 @@ variable "shadow_firewall_rules_priority" { type = number description = "The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000." default = 999 + validation { + condition = var.shadow_firewall_rules_priority < 1000 + error_message = "The shadow firewall rule priority must be lower than auto-created one(1000)." + } +} + +variable "shadow_firewall_rules_log_config" { + type = object({ + metadata = string + }) + description = "The log_config for shadow firewall rules. You can set this variable to `null` to disable logging." + default = { + metadata = "INCLUDE_ALL_METADATA" + } } variable "enable_confidential_nodes" { @@ -556,7 +590,7 @@ variable "node_metadata" { validation { condition = contains(["GKE_METADATA", "GCE_METADATA", "UNSPECIFIED", "GKE_METADATA_SERVER", "EXPOSE"], var.node_metadata) - error_message = "The node_metadata value must be one of GKE_METADATA, GCE_METADATA or UNSPECIFIED." + error_message = "The node_metadata value must be one of GKE_METADATA, GCE_METADATA, UNSPECIFIED, GKE_METADATA_SERVER or EXPOSE." } } @@ -578,6 +612,18 @@ variable "cluster_dns_domain" { default = "" } +variable "gce_pd_csi_driver" { + type = bool + description = "Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver." + default = true +} + +variable "gke_backup_agent_config" { + type = bool + description = "Whether Backup for GKE agent is enabled for this cluster." + default = false +} + variable "timeouts" { type = map(string) description = "Timeout for cluster operations." @@ -588,27 +634,27 @@ variable "timeouts" { } } -variable "enable_kubernetes_alpha" { +variable "monitoring_enable_managed_prometheus" { type = bool - description = "Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days." + description = "Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled." default = false } -variable "logging_enabled_components" { +variable "monitoring_enabled_components" { type = list(string) - description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." + description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration." default = [] } -variable "monitoring_enabled_components" { +variable "logging_enabled_components" { type = list(string) - description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration." + description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." default = [] } -variable "monitoring_enable_managed_prometheus" { +variable "enable_kubernetes_alpha" { type = bool - description = "(Beta) Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled." + description = "Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days." default = false } @@ -635,12 +681,6 @@ variable "config_connector" { default = false } -variable "gke_backup_agent_config" { - type = bool - description = "(Beta) Whether Backup for GKE agent is enabled for this cluster." - default = false -} - variable "cloudrun" { description = "(Beta) Enable CloudRun addon" default = false @@ -681,9 +721,3 @@ variable "enable_identity_service" { description = "Enable the Identity Service component, which allows customers to use external identity providers with the K8S API." default = false } - -variable "gce_pd_csi_driver" { - type = bool - description = "(Beta) Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver." - default = false -} diff --git a/modules/beta-private-cluster/variables_defaults.tf b/modules/beta-private-cluster/variables_defaults.tf index cc65ac9e8b..e4f3004771 100644 --- a/modules/beta-private-cluster/variables_defaults.tf +++ b/modules/beta-private-cluster/variables_defaults.tf @@ -34,6 +34,20 @@ locals { var.node_pools_labels ) + node_pools_resource_labels = merge( + { all = {} }, + { default-node-pool = {} }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : {}] + ), + zipmap( + [for node_pool in var.windows_node_pools : node_pool["name"]], + [for node_pool in var.windows_node_pools : {}] + ), + var.node_pools_resource_labels + ) + node_pools_metadata = merge( { all = {} }, { default-node-pool = {} }, diff --git a/modules/beta-private-cluster/versions.tf b/modules/beta-private-cluster/versions.tf index 7a504e28ce..da5f3f6323 100644 --- a/modules/beta-private-cluster/versions.tf +++ b/modules/beta-private-cluster/versions.tf @@ -21,7 +21,7 @@ terraform { required_providers { google-beta = { source = "hashicorp/google-beta" - version = ">= 4.42.0, < 5.0" + version = ">= 4.51.0, < 5.0" } kubernetes = { source = "hashicorp/kubernetes" @@ -29,6 +29,6 @@ terraform { } } provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:beta-private-cluster/v24.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:beta-private-cluster/v25.0.0" } } diff --git a/modules/beta-public-cluster-update-variant/README.md b/modules/beta-public-cluster-update-variant/README.md index edd667ad70..b366957d8f 100644 --- a/modules/beta-public-cluster-update-variant/README.md +++ b/modules/beta-public-cluster-update-variant/README.md @@ -159,7 +159,7 @@ Then perform the following commands on the root folder: | authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | | cloudrun | (Beta) Enable CloudRun addon | `bool` | `false` | no | | cloudrun\_load\_balancer\_type | (Beta) Configure the Cloud Run load balancer type. External by default. Set to `LOAD_BALANCER_TYPE_INTERNAL` to configure as an internal load balancer. | `string` | `""` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources       = list(object({ resource_type = string, minimum = number, maximum = number }))
  })
| 
{
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources       = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair         = bool
    auto_upgrade        = bool
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | | cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | | cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | | cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | @@ -192,8 +192,9 @@ Then perform the following commands on the root folder: | filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | | firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | | firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | -| gce\_pd\_csi\_driver | (Beta) Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `false` | no | -| gke\_backup\_agent\_config | (Beta) Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | +| gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | +| gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | +| gke\_backup\_agent\_config | Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | | http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no | @@ -215,7 +216,7 @@ Then perform the following commands on the root folder: | maintenance\_recurrence | Frequency of the recurring maintenance window in RFC5545 format. | `string` | `""` | no | | maintenance\_start\_time | Time window specified for daily or recurring maintenance operations in RFC3339 format | `string` | `"05:00"` | no | | master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | -| monitoring\_enable\_managed\_prometheus | (Beta) Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `false` | no | +| monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `false` | no | | monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration. | `list(string)` | `[]` | no | | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | | name | The name of the cluster (required) | `string` | n/a | yes | @@ -229,6 +230,7 @@ Then perform the following commands on the root folder: | node\_pools\_linux\_node\_configs\_sysctls | Map of maps containing linux node config sysctls by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | | node\_pools\_metadata | Map of maps containing node metadata by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | | node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | `map(list(string))` | 
{
  "all": [
    "https://www.googleapis.com/auth/cloud-platform"
  ],
  "default-node-pool": []
}
| no | +| node\_pools\_resource\_labels | Map of maps containing resource labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | | node\_pools\_tags | Map of lists containing node network tags by node-pool name | `map(list(string))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | | node\_pools\_taints | Map of lists containing node taints by node-pool name | `map(list(object({ key = string, value = string, effect = string })))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | | non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | `list(string)` | 
[
  "10.0.0.0/8",
  "172.16.0.0/12",
  "192.168.0.0/16"
]
| no | @@ -243,6 +245,7 @@ Then perform the following commands on the root folder: | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it). | `bool` | `false` | no | | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. | `string` | `""` | no | | service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | +| shadow\_firewall\_rules\_log\_config | The log\_config for shadow firewall rules. You can set this variable to `null` to disable logging. | 
object({
    metadata = string
  })
| 
{
  "metadata": "INCLUDE_ALL_METADATA"
}
| no | | shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | | skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | `bool` | `false` | no | | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | @@ -261,6 +264,7 @@ Then perform the following commands on the root folder: | cluster\_id | Cluster ID | | dns\_cache\_enabled | Whether DNS Cache enabled | | endpoint | Cluster endpoint | +| gateway\_api\_channel | The gateway api channel of this cluster. | | horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | | http\_load\_balancing\_enabled | Whether http load balancing enabled | | identity\_namespace | Workload Identity pool | @@ -329,7 +333,7 @@ The node_pools variable takes the following parameters: | min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count. Should be used when autoscaling is true | 1 | Optional | | name | The name of the node pool | | Required | | placement_policy | Placement type to set for nodes in a node pool. Can be set as [COMPACT](https://cloud.google.com/kubernetes-engine/docs/how-to/compact-placement#overview) if desired | Optional | -| pod_range | The ID of the secondary range for pod IPs. | | Optional | +| pod_range | The name of the secondary range for pod IPs. | | Optional | | node_count | The number of nodes in the nodepool when autoscaling is false. Otherwise defaults to 1. Only valid for non-autoscaling clusters | | Required | | node_locations | The list of zones in which the cluster's nodes are located. Nodes must be in the region of their regional cluster or in the same region as their cluster's zone for zonal clusters. Defaults to cluster level node locations if nothing is specified | " " | Optional | | node_metadata | Options to expose the node metadata to the workload running on the node | | Optional | @@ -340,6 +344,7 @@ The node_pools variable takes the following parameters: | tags | The list of instance tags applied to all nodes | | Required | | value | The value for the taint | | Required | | version | The Kubernetes version for the nodes in this pool. Should only be set if auto_upgrade is false | " " | Optional | +| location_policy | [Location policy](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_node_pool#location_policy) specifies the algorithm used when scaling-up the node pool. Location policy is supported only in 1.24.1+ clusters. | " " | Optional | ## windows_node_pools variable The windows_node_pools variable takes the same parameters as [node_pools](#node\_pools-variable) but is reserved for provisioning Windows based node pools only. This variable is introduced to satisfy a [specific requirement](https://cloud.google.com/kubernetes-engine/docs/how-to/creating-a-cluster-windows#create_a_cluster_and_node_pools) for the presence of at least one linux based node pool in the cluster before a windows based node pool can be created. @@ -360,8 +365,8 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog #### Kubectl - [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x #### Terraform and Plugins -- [Terraform](https://www.terraform.io/downloads.html) 0.12 -- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v3.41 +- [Terraform](https://www.terraform.io/downloads.html) 0.13+ +- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v4.51 #### gcloud Some submodules use the [terraform-google-gcloud](https://github.com/terraform-google-modules/terraform-google-gcloud) module. By default, this module assumes you already have gcloud installed in your $PATH. See the [module](https://github.com/terraform-google-modules/terraform-google-gcloud#downloading) documentation for more information. diff --git a/modules/beta-public-cluster-update-variant/cluster.tf b/modules/beta-public-cluster-update-variant/cluster.tf index 86ab23a8bf..6b348b6111 100644 --- a/modules/beta-public-cluster-update-variant/cluster.tf +++ b/modules/beta-public-cluster-update-variant/cluster.tf @@ -47,6 +47,15 @@ resource "google_container_cluster" "primary" { channel = release_channel.value.channel } } + + dynamic "gateway_api_config" { + for_each = local.gateway_api_config + + content { + channel = gateway_api_config.value.channel + } + } + dynamic "cost_management_config" { for_each = var.enable_cost_allocation ? [1] : [] content { @@ -88,7 +97,7 @@ resource "google_container_cluster" "primary" { for_each = length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus ? [1] : [] content { - enable_components = length(var.monitoring_enabled_components) > 0 ? var.monitoring_enabled_components : null + enable_components = length(var.monitoring_enabled_components) > 0 ? var.monitoring_enabled_components : [] dynamic "managed_prometheus" { for_each = var.monitoring_enable_managed_prometheus ? [1] : [] @@ -105,8 +114,14 @@ resource "google_container_cluster" "primary" { for_each = var.cluster_autoscaling.enabled ? [1] : [] content { - service_account = local.service_account - oauth_scopes = local.node_pools_oauth_scopes["all"] + service_account = local.service_account + oauth_scopes = local.node_pools_oauth_scopes["all"] + + management { + auto_repair = lookup(var.cluster_autoscaling, "auto_repair", true) + auto_upgrade = lookup(var.cluster_autoscaling, "auto_upgrade", true) + } + min_cpu_platform = lookup(var.node_pools[0], "min_cpu_platform", "") } } @@ -133,8 +148,9 @@ resource "google_container_cluster" "primary" { } } + enable_kubernetes_alpha = var.enable_kubernetes_alpha + enable_intranode_visibility = var.enable_intranode_visibility - enable_kubernetes_alpha = var.enable_kubernetes_alpha enable_tpu = var.enable_tpu dynamic "pod_security_policy_config" { @@ -187,7 +203,6 @@ resource "google_container_cluster" "primary" { disabled = !var.horizontal_pod_autoscaling } - network_policy_config { disabled = !var.network_policy } @@ -200,6 +215,22 @@ resource "google_container_cluster" "primary" { enabled = var.filestore_csi_driver } + dynamic "gce_persistent_disk_csi_driver_config" { + for_each = local.cluster_gce_pd_csi_config + + content { + enabled = gce_persistent_disk_csi_driver_config.value.enabled + } + } + + dynamic "gke_backup_agent_config" { + for_each = local.gke_backup_agent_config + + content { + enabled = gke_backup_agent_config.value.enabled + } + } + istio_config { disabled = !var.istio auth = var.istio_auth @@ -213,14 +244,6 @@ resource "google_container_cluster" "primary" { } } - dynamic "gce_persistent_disk_csi_driver_config" { - for_each = local.cluster_gce_pd_csi_config - - content { - enabled = gce_persistent_disk_csi_driver_config.value.enabled - } - } - kalm_config { enabled = var.kalm_config } @@ -228,14 +251,6 @@ resource "google_container_cluster" "primary" { config_connector_config { enabled = var.config_connector } - - dynamic "gke_backup_agent_config" { - for_each = local.gke_backup_agent_config - - content { - enabled = gke_backup_agent_config.value.enabled - } - } } datapath_provider = var.datapath_provider @@ -586,6 +601,10 @@ resource "google_container_node_pool" "pools" { local.node_pools_labels["all"], local.node_pools_labels[each.value["name"]], ) + resource_labels = merge( + local.node_pools_resource_labels["all"], + local.node_pools_resource_labels[each.value["name"]], + ) metadata = merge( lookup(lookup(local.node_pools_metadata, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_metadata, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, @@ -793,6 +812,10 @@ resource "google_container_node_pool" "windows_pools" { local.node_pools_labels["all"], local.node_pools_labels[each.value["name"]], ) + resource_labels = merge( + local.node_pools_resource_labels["all"], + local.node_pools_resource_labels[each.value["name"]], + ) metadata = merge( lookup(lookup(local.node_pools_metadata, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_metadata, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, diff --git a/modules/beta-public-cluster-update-variant/firewall.tf b/modules/beta-public-cluster-update-variant/firewall.tf index 6b71404b79..df15a02367 100644 --- a/modules/beta-public-cluster-update-variant/firewall.tf +++ b/modules/beta-public-cluster-update-variant/firewall.tf @@ -34,11 +34,12 @@ resource "google_compute_firewall" "intra_egress" { direction = "EGRESS" target_tags = [local.cluster_network_tag] - destination_ranges = [ + destination_ranges = concat([ local.cluster_endpoint_for_nodes, local.cluster_subnet_cidr, - local.cluster_alias_ranges_cidr[var.ip_range_pods], - ] + ], + local.pod_all_ip_ranges + ) # Allow all possible protocols allow { protocol = "tcp" } @@ -135,7 +136,7 @@ resource "google_compute_firewall" "shadow_allow_pods" { priority = var.shadow_firewall_rules_priority direction = "INGRESS" - source_ranges = [local.cluster_alias_ranges_cidr[var.ip_range_pods]] + source_ranges = local.pod_all_ip_ranges target_tags = [local.cluster_network_tag] # Allow all possible protocols @@ -146,8 +147,11 @@ resource "google_compute_firewall" "shadow_allow_pods" { allow { protocol = "esp" } allow { protocol = "ah" } - log_config { - metadata = "INCLUDE_ALL_METADATA" + dynamic "log_config" { + for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config] + content { + metadata = log_config.value.metadata + } } } @@ -169,8 +173,11 @@ resource "google_compute_firewall" "shadow_allow_master" { ports = ["10250", "443"] } - log_config { - metadata = "INCLUDE_ALL_METADATA" + dynamic "log_config" { + for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config] + content { + metadata = log_config.value.metadata + } } } @@ -201,7 +208,63 @@ resource "google_compute_firewall" "shadow_allow_nodes" { ports = ["1-65535"] } - log_config { - metadata = "INCLUDE_ALL_METADATA" + dynamic "log_config" { + for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config] + content { + metadata = log_config.value.metadata + } + } +} + +resource "google_compute_firewall" "shadow_allow_inkubelet" { + count = var.add_shadow_firewall_rules ? 1 : 0 + + name = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-inkubelet" + description = "Managed by terraform GKE module: A shadow firewall rule to match the default rule allowing worker nodes & pods communication to kubelet." + project = local.network_project_id + network = var.network + priority = var.shadow_firewall_rules_priority - 1 # rule created by GKE robot have prio 999 + direction = "INGRESS" + + source_ranges = local.pod_all_ip_ranges + source_tags = [local.cluster_network_tag] + target_tags = [local.cluster_network_tag] + + allow { + protocol = "tcp" + ports = ["10255"] + } + + dynamic "log_config" { + for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config] + content { + metadata = log_config.value.metadata + } + } +} + +resource "google_compute_firewall" "shadow_deny_exkubelet" { + count = var.add_shadow_firewall_rules ? 1 : 0 + + name = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-exkubelet" + description = "Managed by terraform GKE module: A shadow firewall rule to match the default deny rule to kubelet." + project = local.network_project_id + network = var.network + priority = var.shadow_firewall_rules_priority # rule created by GKE robot have prio 1000 + direction = "INGRESS" + + source_ranges = ["0.0.0.0/0"] + target_tags = [local.cluster_network_tag] + + deny { + protocol = "tcp" + ports = ["10255"] + } + + dynamic "log_config" { + for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config] + content { + metadata = log_config.value.metadata + } } } diff --git a/modules/beta-public-cluster-update-variant/main.tf b/modules/beta-public-cluster-update-variant/main.tf index 94d950d0e2..491200ba7e 100644 --- a/modules/beta-public-cluster-update-variant/main.tf +++ b/modules/beta-public-cluster-update-variant/main.tf @@ -50,7 +50,8 @@ locals { windows_node_pool_names = [for np in toset(var.windows_node_pools) : np.name] windows_node_pools = zipmap(local.windows_node_pool_names, tolist(toset(var.windows_node_pools))) - release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] + release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] + gateway_api_config = var.gateway_api_channel != null ? [{ channel : var.gateway_api_channel }] : [] autoscaling_resource_limits = var.cluster_autoscaling.enabled ? concat([{ resource_type = "cpu" @@ -74,6 +75,7 @@ locals { cluster_subnet_cidr = var.add_cluster_firewall_rules ? data.google_compute_subnetwork.gke_subnetwork[0].ip_cidr_range : null cluster_alias_ranges_cidr = var.add_cluster_firewall_rules ? { for range in toset(data.google_compute_subnetwork.gke_subnetwork[0].secondary_ip_range) : range.range_name => range.ip_cidr_range } : {} + pod_all_ip_ranges = var.add_cluster_firewall_rules ? compact(concat([local.cluster_alias_ranges_cidr[var.ip_range_pods]], [for k, v in merge(local.node_pools, local.windows_node_pools) : local.cluster_alias_ranges_cidr[v.pod_range] if length(lookup(v, "pod_range", "")) > 0])) : [] cluster_network_policy = var.network_policy ? [{ enabled = true @@ -82,6 +84,9 @@ locals { enabled = false provider = null }] + cluster_gce_pd_csi_config = var.gce_pd_csi_driver ? [{ enabled = true }] : [{ enabled = false }] + logmon_config_is_set = length(var.logging_enabled_components) > 0 || length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus + gke_backup_agent_config = var.gke_backup_agent_config ? [{ enabled = true }] : [{ enabled = false }] cluster_cloudrun_config_load_balancer_config = (var.cloudrun && var.cloudrun_load_balancer_type != "") ? { load_balancer_type = var.cloudrun_load_balancer_type } : {} @@ -93,10 +98,7 @@ locals { local.cluster_cloudrun_config_load_balancer_config ) ] : [] - cluster_cloudrun_enabled = var.cloudrun - cluster_gce_pd_csi_config = var.gce_pd_csi_driver ? [{ enabled = true }] : [{ enabled = false }] - gke_backup_agent_config = var.gke_backup_agent_config ? [{ enabled = true }] : [{ enabled = false }] - logmon_config_is_set = length(var.logging_enabled_components) > 0 || length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus + cluster_cloudrun_enabled = var.cloudrun cluster_authenticator_security_group = var.authenticator_security_group == null ? [] : [{ security_group = var.authenticator_security_group diff --git a/modules/beta-public-cluster-update-variant/outputs.tf b/modules/beta-public-cluster-update-variant/outputs.tf index 31556abecd..ed73acae2e 100644 --- a/modules/beta-public-cluster-update-variant/outputs.tf +++ b/modules/beta-public-cluster-update-variant/outputs.tf @@ -148,6 +148,11 @@ output "release_channel" { value = var.release_channel } +output "gateway_api_channel" { + description = "The gateway api channel of this cluster." + value = var.gateway_api_channel +} + output "identity_namespace" { description = "Workload Identity pool" value = length(local.cluster_workload_identity_config) > 0 ? local.cluster_workload_identity_config[0].workload_pool : null diff --git a/modules/beta-public-cluster-update-variant/variables.tf b/modules/beta-public-cluster-update-variant/variables.tf index 65ced56983..970d25f40d 100644 --- a/modules/beta-public-cluster-update-variant/variables.tf +++ b/modules/beta-public-cluster-update-variant/variables.tf @@ -170,6 +170,16 @@ variable "node_pools_labels" { } } +variable "node_pools_resource_labels" { + type = map(map(string)) + description = "Map of maps containing resource labels by node-pool name" + + default = { + all = {} + default-node-pool = {} + } +} + variable "node_pools_metadata" { type = map(map(string)) description = "Map of maps containing node metadata by node-pool name" @@ -224,6 +234,8 @@ variable "cluster_autoscaling" { min_memory_gb = number max_memory_gb = number gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) + auto_repair = bool + auto_upgrade = bool }) default = { enabled = false @@ -233,6 +245,8 @@ variable "cluster_autoscaling" { max_memory_gb = 0 min_memory_gb = 0 gpu_resources = [] + auto_repair = true + auto_upgrade = true } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } @@ -396,6 +410,12 @@ variable "release_channel" { default = null } +variable "gateway_api_channel" { + type = string + description = "The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`." + default = null +} + variable "add_cluster_firewall_rules" { type = bool description = "Create additional firewall rules" @@ -430,6 +450,20 @@ variable "shadow_firewall_rules_priority" { type = number description = "The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000." default = 999 + validation { + condition = var.shadow_firewall_rules_priority < 1000 + error_message = "The shadow firewall rule priority must be lower than auto-created one(1000)." + } +} + +variable "shadow_firewall_rules_log_config" { + type = object({ + metadata = string + }) + description = "The log_config for shadow firewall rules. You can set this variable to `null` to disable logging." + default = { + metadata = "INCLUDE_ALL_METADATA" + } } variable "enable_confidential_nodes" { @@ -526,7 +560,7 @@ variable "node_metadata" { validation { condition = contains(["GKE_METADATA", "GCE_METADATA", "UNSPECIFIED", "GKE_METADATA_SERVER", "EXPOSE"], var.node_metadata) - error_message = "The node_metadata value must be one of GKE_METADATA, GCE_METADATA or UNSPECIFIED." + error_message = "The node_metadata value must be one of GKE_METADATA, GCE_METADATA, UNSPECIFIED, GKE_METADATA_SERVER or EXPOSE." } } @@ -548,6 +582,18 @@ variable "cluster_dns_domain" { default = "" } +variable "gce_pd_csi_driver" { + type = bool + description = "Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver." + default = true +} + +variable "gke_backup_agent_config" { + type = bool + description = "Whether Backup for GKE agent is enabled for this cluster." + default = false +} + variable "timeouts" { type = map(string) description = "Timeout for cluster operations." @@ -558,27 +604,27 @@ variable "timeouts" { } } -variable "enable_kubernetes_alpha" { +variable "monitoring_enable_managed_prometheus" { type = bool - description = "Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days." + description = "Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled." default = false } -variable "logging_enabled_components" { +variable "monitoring_enabled_components" { type = list(string) - description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." + description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration." default = [] } -variable "monitoring_enabled_components" { +variable "logging_enabled_components" { type = list(string) - description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration." + description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." default = [] } -variable "monitoring_enable_managed_prometheus" { +variable "enable_kubernetes_alpha" { type = bool - description = "(Beta) Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled." + description = "Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days." default = false } @@ -605,12 +651,6 @@ variable "config_connector" { default = false } -variable "gke_backup_agent_config" { - type = bool - description = "(Beta) Whether Backup for GKE agent is enabled for this cluster." - default = false -} - variable "cloudrun" { description = "(Beta) Enable CloudRun addon" default = false @@ -651,9 +691,3 @@ variable "enable_identity_service" { description = "Enable the Identity Service component, which allows customers to use external identity providers with the K8S API." default = false } - -variable "gce_pd_csi_driver" { - type = bool - description = "(Beta) Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver." - default = false -} diff --git a/modules/beta-public-cluster-update-variant/variables_defaults.tf b/modules/beta-public-cluster-update-variant/variables_defaults.tf index cc65ac9e8b..e4f3004771 100644 --- a/modules/beta-public-cluster-update-variant/variables_defaults.tf +++ b/modules/beta-public-cluster-update-variant/variables_defaults.tf @@ -34,6 +34,20 @@ locals { var.node_pools_labels ) + node_pools_resource_labels = merge( + { all = {} }, + { default-node-pool = {} }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : {}] + ), + zipmap( + [for node_pool in var.windows_node_pools : node_pool["name"]], + [for node_pool in var.windows_node_pools : {}] + ), + var.node_pools_resource_labels + ) + node_pools_metadata = merge( { all = {} }, { default-node-pool = {} }, diff --git a/modules/beta-public-cluster-update-variant/versions.tf b/modules/beta-public-cluster-update-variant/versions.tf index 0e0bee1e76..622145d914 100644 --- a/modules/beta-public-cluster-update-variant/versions.tf +++ b/modules/beta-public-cluster-update-variant/versions.tf @@ -21,7 +21,7 @@ terraform { required_providers { google-beta = { source = "hashicorp/google-beta" - version = ">= 4.42.0, < 5.0" + version = ">= 4.51.0, < 5.0" } kubernetes = { source = "hashicorp/kubernetes" @@ -29,6 +29,6 @@ terraform { } } provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:beta-public-cluster-update-variant/v24.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:beta-public-cluster-update-variant/v25.0.0" } } diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md index 66fd8e6fc4..c018d01fd6 100644 --- a/modules/beta-public-cluster/README.md +++ b/modules/beta-public-cluster/README.md @@ -137,7 +137,7 @@ Then perform the following commands on the root folder: | authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | | cloudrun | (Beta) Enable CloudRun addon | `bool` | `false` | no | | cloudrun\_load\_balancer\_type | (Beta) Configure the Cloud Run load balancer type. External by default. Set to `LOAD_BALANCER_TYPE_INTERNAL` to configure as an internal load balancer. | `string` | `""` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources       = list(object({ resource_type = string, minimum = number, maximum = number }))
  })
| 
{
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources       = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair         = bool
    auto_upgrade        = bool
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | | cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | | cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | | cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | @@ -170,8 +170,9 @@ Then perform the following commands on the root folder: | filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | | firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | | firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | -| gce\_pd\_csi\_driver | (Beta) Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `false` | no | -| gke\_backup\_agent\_config | (Beta) Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | +| gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | +| gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | +| gke\_backup\_agent\_config | Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | | http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no | @@ -193,7 +194,7 @@ Then perform the following commands on the root folder: | maintenance\_recurrence | Frequency of the recurring maintenance window in RFC5545 format. | `string` | `""` | no | | maintenance\_start\_time | Time window specified for daily or recurring maintenance operations in RFC3339 format | `string` | `"05:00"` | no | | master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | -| monitoring\_enable\_managed\_prometheus | (Beta) Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `false` | no | +| monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `false` | no | | monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration. | `list(string)` | `[]` | no | | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | | name | The name of the cluster (required) | `string` | n/a | yes | @@ -207,6 +208,7 @@ Then perform the following commands on the root folder: | node\_pools\_linux\_node\_configs\_sysctls | Map of maps containing linux node config sysctls by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | | node\_pools\_metadata | Map of maps containing node metadata by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | | node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | `map(list(string))` | 
{
  "all": [
    "https://www.googleapis.com/auth/cloud-platform"
  ],
  "default-node-pool": []
}
| no | +| node\_pools\_resource\_labels | Map of maps containing resource labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | | node\_pools\_tags | Map of lists containing node network tags by node-pool name | `map(list(string))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | | node\_pools\_taints | Map of lists containing node taints by node-pool name | `map(list(object({ key = string, value = string, effect = string })))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | | non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | `list(string)` | 
[
  "10.0.0.0/8",
  "172.16.0.0/12",
  "192.168.0.0/16"
]
| no | @@ -221,6 +223,7 @@ Then perform the following commands on the root folder: | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it). | `bool` | `false` | no | | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. | `string` | `""` | no | | service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | +| shadow\_firewall\_rules\_log\_config | The log\_config for shadow firewall rules. You can set this variable to `null` to disable logging. | 
object({
    metadata = string
  })
| 
{
  "metadata": "INCLUDE_ALL_METADATA"
}
| no | | shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | | skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | `bool` | `false` | no | | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | @@ -239,6 +242,7 @@ Then perform the following commands on the root folder: | cluster\_id | Cluster ID | | dns\_cache\_enabled | Whether DNS Cache enabled | | endpoint | Cluster endpoint | +| gateway\_api\_channel | The gateway api channel of this cluster. | | horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | | http\_load\_balancing\_enabled | Whether http load balancing enabled | | identity\_namespace | Workload Identity pool | @@ -307,7 +311,7 @@ The node_pools variable takes the following parameters: | min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count. Should be used when autoscaling is true | 1 | Optional | | name | The name of the node pool | | Required | | placement_policy | Placement type to set for nodes in a node pool. Can be set as [COMPACT](https://cloud.google.com/kubernetes-engine/docs/how-to/compact-placement#overview) if desired | Optional | -| pod_range | The ID of the secondary range for pod IPs. | | Optional | +| pod_range | The name of the secondary range for pod IPs. | | Optional | | node_count | The number of nodes in the nodepool when autoscaling is false. Otherwise defaults to 1. Only valid for non-autoscaling clusters | | Required | | node_locations | The list of zones in which the cluster's nodes are located. Nodes must be in the region of their regional cluster or in the same region as their cluster's zone for zonal clusters. Defaults to cluster level node locations if nothing is specified | " " | Optional | | node_metadata | Options to expose the node metadata to the workload running on the node | | Optional | @@ -318,6 +322,7 @@ The node_pools variable takes the following parameters: | tags | The list of instance tags applied to all nodes | | Required | | value | The value for the taint | | Required | | version | The Kubernetes version for the nodes in this pool. Should only be set if auto_upgrade is false | " " | Optional | +| location_policy | [Location policy](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_node_pool#location_policy) specifies the algorithm used when scaling-up the node pool. Location policy is supported only in 1.24.1+ clusters. | " " | Optional | ## windows_node_pools variable The windows_node_pools variable takes the same parameters as [node_pools](#node\_pools-variable) but is reserved for provisioning Windows based node pools only. This variable is introduced to satisfy a [specific requirement](https://cloud.google.com/kubernetes-engine/docs/how-to/creating-a-cluster-windows#create_a_cluster_and_node_pools) for the presence of at least one linux based node pool in the cluster before a windows based node pool can be created. @@ -338,8 +343,8 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog #### Kubectl - [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x #### Terraform and Plugins -- [Terraform](https://www.terraform.io/downloads.html) 0.12 -- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v3.41 +- [Terraform](https://www.terraform.io/downloads.html) 0.13+ +- [Terraform Provider for GCP Beta][terraform-provider-google-beta] v4.51 #### gcloud Some submodules use the [terraform-google-gcloud](https://github.com/terraform-google-modules/terraform-google-gcloud) module. By default, this module assumes you already have gcloud installed in your $PATH. See the [module](https://github.com/terraform-google-modules/terraform-google-gcloud#downloading) documentation for more information. diff --git a/modules/beta-public-cluster/cluster.tf b/modules/beta-public-cluster/cluster.tf index 1d17b229ed..d21864f65b 100644 --- a/modules/beta-public-cluster/cluster.tf +++ b/modules/beta-public-cluster/cluster.tf @@ -47,6 +47,15 @@ resource "google_container_cluster" "primary" { channel = release_channel.value.channel } } + + dynamic "gateway_api_config" { + for_each = local.gateway_api_config + + content { + channel = gateway_api_config.value.channel + } + } + dynamic "cost_management_config" { for_each = var.enable_cost_allocation ? [1] : [] content { @@ -88,7 +97,7 @@ resource "google_container_cluster" "primary" { for_each = length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus ? [1] : [] content { - enable_components = length(var.monitoring_enabled_components) > 0 ? var.monitoring_enabled_components : null + enable_components = length(var.monitoring_enabled_components) > 0 ? var.monitoring_enabled_components : [] dynamic "managed_prometheus" { for_each = var.monitoring_enable_managed_prometheus ? [1] : [] @@ -105,8 +114,14 @@ resource "google_container_cluster" "primary" { for_each = var.cluster_autoscaling.enabled ? [1] : [] content { - service_account = local.service_account - oauth_scopes = local.node_pools_oauth_scopes["all"] + service_account = local.service_account + oauth_scopes = local.node_pools_oauth_scopes["all"] + + management { + auto_repair = lookup(var.cluster_autoscaling, "auto_repair", true) + auto_upgrade = lookup(var.cluster_autoscaling, "auto_upgrade", true) + } + min_cpu_platform = lookup(var.node_pools[0], "min_cpu_platform", "") } } @@ -133,8 +148,9 @@ resource "google_container_cluster" "primary" { } } + enable_kubernetes_alpha = var.enable_kubernetes_alpha + enable_intranode_visibility = var.enable_intranode_visibility - enable_kubernetes_alpha = var.enable_kubernetes_alpha enable_tpu = var.enable_tpu dynamic "pod_security_policy_config" { @@ -187,7 +203,6 @@ resource "google_container_cluster" "primary" { disabled = !var.horizontal_pod_autoscaling } - network_policy_config { disabled = !var.network_policy } @@ -200,6 +215,22 @@ resource "google_container_cluster" "primary" { enabled = var.filestore_csi_driver } + dynamic "gce_persistent_disk_csi_driver_config" { + for_each = local.cluster_gce_pd_csi_config + + content { + enabled = gce_persistent_disk_csi_driver_config.value.enabled + } + } + + dynamic "gke_backup_agent_config" { + for_each = local.gke_backup_agent_config + + content { + enabled = gke_backup_agent_config.value.enabled + } + } + istio_config { disabled = !var.istio auth = var.istio_auth @@ -213,14 +244,6 @@ resource "google_container_cluster" "primary" { } } - dynamic "gce_persistent_disk_csi_driver_config" { - for_each = local.cluster_gce_pd_csi_config - - content { - enabled = gce_persistent_disk_csi_driver_config.value.enabled - } - } - kalm_config { enabled = var.kalm_config } @@ -228,14 +251,6 @@ resource "google_container_cluster" "primary" { config_connector_config { enabled = var.config_connector } - - dynamic "gke_backup_agent_config" { - for_each = local.gke_backup_agent_config - - content { - enabled = gke_backup_agent_config.value.enabled - } - } } datapath_provider = var.datapath_provider @@ -492,6 +507,10 @@ resource "google_container_node_pool" "pools" { local.node_pools_labels["all"], local.node_pools_labels[each.value["name"]], ) + resource_labels = merge( + local.node_pools_resource_labels["all"], + local.node_pools_resource_labels[each.value["name"]], + ) metadata = merge( lookup(lookup(local.node_pools_metadata, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_metadata, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, @@ -698,6 +717,10 @@ resource "google_container_node_pool" "windows_pools" { local.node_pools_labels["all"], local.node_pools_labels[each.value["name"]], ) + resource_labels = merge( + local.node_pools_resource_labels["all"], + local.node_pools_resource_labels[each.value["name"]], + ) metadata = merge( lookup(lookup(local.node_pools_metadata, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_metadata, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, diff --git a/modules/beta-public-cluster/firewall.tf b/modules/beta-public-cluster/firewall.tf index 6b71404b79..df15a02367 100644 --- a/modules/beta-public-cluster/firewall.tf +++ b/modules/beta-public-cluster/firewall.tf @@ -34,11 +34,12 @@ resource "google_compute_firewall" "intra_egress" { direction = "EGRESS" target_tags = [local.cluster_network_tag] - destination_ranges = [ + destination_ranges = concat([ local.cluster_endpoint_for_nodes, local.cluster_subnet_cidr, - local.cluster_alias_ranges_cidr[var.ip_range_pods], - ] + ], + local.pod_all_ip_ranges + ) # Allow all possible protocols allow { protocol = "tcp" } @@ -135,7 +136,7 @@ resource "google_compute_firewall" "shadow_allow_pods" { priority = var.shadow_firewall_rules_priority direction = "INGRESS" - source_ranges = [local.cluster_alias_ranges_cidr[var.ip_range_pods]] + source_ranges = local.pod_all_ip_ranges target_tags = [local.cluster_network_tag] # Allow all possible protocols @@ -146,8 +147,11 @@ resource "google_compute_firewall" "shadow_allow_pods" { allow { protocol = "esp" } allow { protocol = "ah" } - log_config { - metadata = "INCLUDE_ALL_METADATA" + dynamic "log_config" { + for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config] + content { + metadata = log_config.value.metadata + } } } @@ -169,8 +173,11 @@ resource "google_compute_firewall" "shadow_allow_master" { ports = ["10250", "443"] } - log_config { - metadata = "INCLUDE_ALL_METADATA" + dynamic "log_config" { + for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config] + content { + metadata = log_config.value.metadata + } } } @@ -201,7 +208,63 @@ resource "google_compute_firewall" "shadow_allow_nodes" { ports = ["1-65535"] } - log_config { - metadata = "INCLUDE_ALL_METADATA" + dynamic "log_config" { + for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config] + content { + metadata = log_config.value.metadata + } + } +} + +resource "google_compute_firewall" "shadow_allow_inkubelet" { + count = var.add_shadow_firewall_rules ? 1 : 0 + + name = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-inkubelet" + description = "Managed by terraform GKE module: A shadow firewall rule to match the default rule allowing worker nodes & pods communication to kubelet." + project = local.network_project_id + network = var.network + priority = var.shadow_firewall_rules_priority - 1 # rule created by GKE robot have prio 999 + direction = "INGRESS" + + source_ranges = local.pod_all_ip_ranges + source_tags = [local.cluster_network_tag] + target_tags = [local.cluster_network_tag] + + allow { + protocol = "tcp" + ports = ["10255"] + } + + dynamic "log_config" { + for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config] + content { + metadata = log_config.value.metadata + } + } +} + +resource "google_compute_firewall" "shadow_deny_exkubelet" { + count = var.add_shadow_firewall_rules ? 1 : 0 + + name = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-exkubelet" + description = "Managed by terraform GKE module: A shadow firewall rule to match the default deny rule to kubelet." + project = local.network_project_id + network = var.network + priority = var.shadow_firewall_rules_priority # rule created by GKE robot have prio 1000 + direction = "INGRESS" + + source_ranges = ["0.0.0.0/0"] + target_tags = [local.cluster_network_tag] + + deny { + protocol = "tcp" + ports = ["10255"] + } + + dynamic "log_config" { + for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config] + content { + metadata = log_config.value.metadata + } } } diff --git a/modules/beta-public-cluster/main.tf b/modules/beta-public-cluster/main.tf index 94d950d0e2..491200ba7e 100644 --- a/modules/beta-public-cluster/main.tf +++ b/modules/beta-public-cluster/main.tf @@ -50,7 +50,8 @@ locals { windows_node_pool_names = [for np in toset(var.windows_node_pools) : np.name] windows_node_pools = zipmap(local.windows_node_pool_names, tolist(toset(var.windows_node_pools))) - release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] + release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] + gateway_api_config = var.gateway_api_channel != null ? [{ channel : var.gateway_api_channel }] : [] autoscaling_resource_limits = var.cluster_autoscaling.enabled ? concat([{ resource_type = "cpu" @@ -74,6 +75,7 @@ locals { cluster_subnet_cidr = var.add_cluster_firewall_rules ? data.google_compute_subnetwork.gke_subnetwork[0].ip_cidr_range : null cluster_alias_ranges_cidr = var.add_cluster_firewall_rules ? { for range in toset(data.google_compute_subnetwork.gke_subnetwork[0].secondary_ip_range) : range.range_name => range.ip_cidr_range } : {} + pod_all_ip_ranges = var.add_cluster_firewall_rules ? compact(concat([local.cluster_alias_ranges_cidr[var.ip_range_pods]], [for k, v in merge(local.node_pools, local.windows_node_pools) : local.cluster_alias_ranges_cidr[v.pod_range] if length(lookup(v, "pod_range", "")) > 0])) : [] cluster_network_policy = var.network_policy ? [{ enabled = true @@ -82,6 +84,9 @@ locals { enabled = false provider = null }] + cluster_gce_pd_csi_config = var.gce_pd_csi_driver ? [{ enabled = true }] : [{ enabled = false }] + logmon_config_is_set = length(var.logging_enabled_components) > 0 || length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus + gke_backup_agent_config = var.gke_backup_agent_config ? [{ enabled = true }] : [{ enabled = false }] cluster_cloudrun_config_load_balancer_config = (var.cloudrun && var.cloudrun_load_balancer_type != "") ? { load_balancer_type = var.cloudrun_load_balancer_type } : {} @@ -93,10 +98,7 @@ locals { local.cluster_cloudrun_config_load_balancer_config ) ] : [] - cluster_cloudrun_enabled = var.cloudrun - cluster_gce_pd_csi_config = var.gce_pd_csi_driver ? [{ enabled = true }] : [{ enabled = false }] - gke_backup_agent_config = var.gke_backup_agent_config ? [{ enabled = true }] : [{ enabled = false }] - logmon_config_is_set = length(var.logging_enabled_components) > 0 || length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus + cluster_cloudrun_enabled = var.cloudrun cluster_authenticator_security_group = var.authenticator_security_group == null ? [] : [{ security_group = var.authenticator_security_group diff --git a/modules/beta-public-cluster/outputs.tf b/modules/beta-public-cluster/outputs.tf index 31556abecd..ed73acae2e 100644 --- a/modules/beta-public-cluster/outputs.tf +++ b/modules/beta-public-cluster/outputs.tf @@ -148,6 +148,11 @@ output "release_channel" { value = var.release_channel } +output "gateway_api_channel" { + description = "The gateway api channel of this cluster." + value = var.gateway_api_channel +} + output "identity_namespace" { description = "Workload Identity pool" value = length(local.cluster_workload_identity_config) > 0 ? local.cluster_workload_identity_config[0].workload_pool : null diff --git a/modules/beta-public-cluster/variables.tf b/modules/beta-public-cluster/variables.tf index 65ced56983..970d25f40d 100644 --- a/modules/beta-public-cluster/variables.tf +++ b/modules/beta-public-cluster/variables.tf @@ -170,6 +170,16 @@ variable "node_pools_labels" { } } +variable "node_pools_resource_labels" { + type = map(map(string)) + description = "Map of maps containing resource labels by node-pool name" + + default = { + all = {} + default-node-pool = {} + } +} + variable "node_pools_metadata" { type = map(map(string)) description = "Map of maps containing node metadata by node-pool name" @@ -224,6 +234,8 @@ variable "cluster_autoscaling" { min_memory_gb = number max_memory_gb = number gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) + auto_repair = bool + auto_upgrade = bool }) default = { enabled = false @@ -233,6 +245,8 @@ variable "cluster_autoscaling" { max_memory_gb = 0 min_memory_gb = 0 gpu_resources = [] + auto_repair = true + auto_upgrade = true } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } @@ -396,6 +410,12 @@ variable "release_channel" { default = null } +variable "gateway_api_channel" { + type = string + description = "The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`." + default = null +} + variable "add_cluster_firewall_rules" { type = bool description = "Create additional firewall rules" @@ -430,6 +450,20 @@ variable "shadow_firewall_rules_priority" { type = number description = "The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000." default = 999 + validation { + condition = var.shadow_firewall_rules_priority < 1000 + error_message = "The shadow firewall rule priority must be lower than auto-created one(1000)." + } +} + +variable "shadow_firewall_rules_log_config" { + type = object({ + metadata = string + }) + description = "The log_config for shadow firewall rules. You can set this variable to `null` to disable logging." + default = { + metadata = "INCLUDE_ALL_METADATA" + } } variable "enable_confidential_nodes" { @@ -526,7 +560,7 @@ variable "node_metadata" { validation { condition = contains(["GKE_METADATA", "GCE_METADATA", "UNSPECIFIED", "GKE_METADATA_SERVER", "EXPOSE"], var.node_metadata) - error_message = "The node_metadata value must be one of GKE_METADATA, GCE_METADATA or UNSPECIFIED." + error_message = "The node_metadata value must be one of GKE_METADATA, GCE_METADATA, UNSPECIFIED, GKE_METADATA_SERVER or EXPOSE." } } @@ -548,6 +582,18 @@ variable "cluster_dns_domain" { default = "" } +variable "gce_pd_csi_driver" { + type = bool + description = "Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver." + default = true +} + +variable "gke_backup_agent_config" { + type = bool + description = "Whether Backup for GKE agent is enabled for this cluster." + default = false +} + variable "timeouts" { type = map(string) description = "Timeout for cluster operations." @@ -558,27 +604,27 @@ variable "timeouts" { } } -variable "enable_kubernetes_alpha" { +variable "monitoring_enable_managed_prometheus" { type = bool - description = "Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days." + description = "Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled." default = false } -variable "logging_enabled_components" { +variable "monitoring_enabled_components" { type = list(string) - description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." + description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration." default = [] } -variable "monitoring_enabled_components" { +variable "logging_enabled_components" { type = list(string) - description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration." + description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." default = [] } -variable "monitoring_enable_managed_prometheus" { +variable "enable_kubernetes_alpha" { type = bool - description = "(Beta) Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled." + description = "Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days." default = false } @@ -605,12 +651,6 @@ variable "config_connector" { default = false } -variable "gke_backup_agent_config" { - type = bool - description = "(Beta) Whether Backup for GKE agent is enabled for this cluster." - default = false -} - variable "cloudrun" { description = "(Beta) Enable CloudRun addon" default = false @@ -651,9 +691,3 @@ variable "enable_identity_service" { description = "Enable the Identity Service component, which allows customers to use external identity providers with the K8S API." default = false } - -variable "gce_pd_csi_driver" { - type = bool - description = "(Beta) Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver." - default = false -} diff --git a/modules/beta-public-cluster/variables_defaults.tf b/modules/beta-public-cluster/variables_defaults.tf index cc65ac9e8b..e4f3004771 100644 --- a/modules/beta-public-cluster/variables_defaults.tf +++ b/modules/beta-public-cluster/variables_defaults.tf @@ -34,6 +34,20 @@ locals { var.node_pools_labels ) + node_pools_resource_labels = merge( + { all = {} }, + { default-node-pool = {} }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : {}] + ), + zipmap( + [for node_pool in var.windows_node_pools : node_pool["name"]], + [for node_pool in var.windows_node_pools : {}] + ), + var.node_pools_resource_labels + ) + node_pools_metadata = merge( { all = {} }, { default-node-pool = {} }, diff --git a/modules/beta-public-cluster/versions.tf b/modules/beta-public-cluster/versions.tf index 75cc6b48d9..ccf46416fa 100644 --- a/modules/beta-public-cluster/versions.tf +++ b/modules/beta-public-cluster/versions.tf @@ -21,7 +21,7 @@ terraform { required_providers { google-beta = { source = "hashicorp/google-beta" - version = ">= 4.42.0, < 5.0" + version = ">= 4.51.0, < 5.0" } kubernetes = { source = "hashicorp/kubernetes" @@ -29,6 +29,6 @@ terraform { } } provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:beta-public-cluster/v24.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:beta-public-cluster/v25.0.0" } } diff --git a/modules/binary-authorization/versions.tf b/modules/binary-authorization/versions.tf index a5369ec41f..6c7b10adc8 100644 --- a/modules/binary-authorization/versions.tf +++ b/modules/binary-authorization/versions.tf @@ -19,6 +19,6 @@ terraform { required_version = ">= 0.13.0" provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:binary-authorization/v24.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:binary-authorization/v25.0.0" } } diff --git a/modules/fleet-membership/versions.tf b/modules/fleet-membership/versions.tf index 0dbfb534bd..da9bee0dee 100644 --- a/modules/fleet-membership/versions.tf +++ b/modules/fleet-membership/versions.tf @@ -18,7 +18,15 @@ terraform { required_version = ">= 0.13.0" + required_providers { + google = { + source = "hashicorp/google" + # Avoid v25.0.0 for https://github.com/hashicorp/terraform-provider-google/issues/13507 + version = ">= 4.47.0, != 4.49.0, != 4.50.0, < 5.0" + } + } + provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:hub/v24.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:hub/v25.0.0" } } diff --git a/modules/hub-legacy/versions.tf b/modules/hub-legacy/versions.tf index 0dbfb534bd..8d2f021aed 100644 --- a/modules/hub-legacy/versions.tf +++ b/modules/hub-legacy/versions.tf @@ -19,6 +19,6 @@ terraform { required_version = ">= 0.13.0" provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:hub/v24.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:hub/v25.0.0" } } diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md index 9e282c0021..8d952ab9c1 100644 --- a/modules/private-cluster-update-variant/README.md +++ b/modules/private-cluster-update-variant/README.md @@ -159,7 +159,7 @@ Then perform the following commands on the root folder: | add\_master\_webhook\_firewall\_rules | Create master\_webhook firewall rules for ports defined in `firewall_inbound_ports` | `bool` | `false` | no | | add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | | authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled       = bool
    min_cpu_cores = number
    max_cpu_cores = number
    min_memory_gb = number
    max_memory_gb = number
    gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number }))
  })
| 
{
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled       = bool
    min_cpu_cores = number
    max_cpu_cores = number
    min_memory_gb = number
    max_memory_gb = number
    gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair   = bool
    auto_upgrade  = bool
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | | cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | | cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | | cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | @@ -177,6 +177,7 @@ Then perform the following commands on the root folder: | dns\_cache | The status of the NodeLocal DNSCache addon. | `bool` | `false` | no | | enable\_binary\_authorization | Enable BinAuthZ Admission controller | `bool` | `false` | no | | enable\_cost\_allocation | Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery | `bool` | `false` | no | +| enable\_kubernetes\_alpha | Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days. | `bool` | `false` | no | | enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `bool` | `false` | no | | enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | `bool` | `false` | no | | enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | `bool` | `false` | no | @@ -186,6 +187,9 @@ Then perform the following commands on the root folder: | filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | | firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | | firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | +| gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | +| gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | +| gke\_backup\_agent\_config | Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | | http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no | @@ -197,6 +201,7 @@ Then perform the following commands on the root folder: | ip\_range\_services | The _name_ of the secondary subnet range to use for services | `string` | n/a | yes | | issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | | kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | `string` | `"latest"` | no | +| logging\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | | logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | `string` | `"logging.googleapis.com/kubernetes"` | no | | maintenance\_end\_time | Time window specified for recurring maintenance operations in RFC3339 format | `string` | `""` | no | | maintenance\_exclusions | List of maintenance exclusions. A cluster can have up to three | `list(object({ name = string, start_time = string, end_time = string, exclusion_scope = string }))` | `[]` | no | @@ -205,6 +210,8 @@ Then perform the following commands on the root folder: | master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | | master\_global\_access\_enabled | Whether the cluster master is accessible globally (from any region) or only within the same region as the private endpoint. | `bool` | `true` | no | | master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | `string` | `"10.0.0.0/28"` | no | +| monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `false` | no | +| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration. | `list(string)` | `[]` | no | | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | | name | The name of the cluster (required) | `string` | n/a | yes | | network | The VPC network to host the cluster in (required) | `string` | n/a | yes | @@ -214,8 +221,10 @@ Then perform the following commands on the root folder: | node\_metadata | Specifies how node metadata is exposed to the workload running on the node | `string` | `"GKE_METADATA"` | no | | node\_pools | List of maps containing node pools | `list(map(any))` | 
[
  {
    "name": "default-node-pool"
  }
]
| no | | node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_linux\_node\_configs\_sysctls | Map of maps containing linux node config sysctls by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | | node\_pools\_metadata | Map of maps containing node metadata by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | | node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | `map(list(string))` | 
{
  "all": [
    "https://www.googleapis.com/auth/cloud-platform"
  ],
  "default-node-pool": []
}
| no | +| node\_pools\_resource\_labels | Map of maps containing resource labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | | node\_pools\_tags | Map of lists containing node network tags by node-pool name | `map(list(string))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | | node\_pools\_taints | Map of lists containing node taints by node-pool name | `map(list(object({ key = string, value = string, effect = string })))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | | non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | `list(string)` | 
[
  "10.0.0.0/8",
  "172.16.0.0/12",
  "192.168.0.0/16"
]
| no | @@ -229,6 +238,7 @@ Then perform the following commands on the root folder: | resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. | `string` | `""` | no | | service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | +| shadow\_firewall\_rules\_log\_config | The log\_config for shadow firewall rules. You can set this variable to `null` to disable logging. | 
object({
    metadata = string
  })
| 
{
  "metadata": "INCLUDE_ALL_METADATA"
}
| no | | shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | | skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | `bool` | `false` | no | | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | @@ -245,6 +255,7 @@ Then perform the following commands on the root folder: | ca\_certificate | Cluster ca certificate (base64 encoded) | | cluster\_id | Cluster ID | | endpoint | Cluster endpoint | +| gateway\_api\_channel | The gateway api channel of this cluster. | | horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | | http\_load\_balancing\_enabled | Whether http load balancing enabled | | identity\_namespace | Workload Identity pool | @@ -313,6 +324,7 @@ The node_pools variable takes the following parameters: | tags | The list of instance tags applied to all nodes | | Required | | value | The value for the taint | | Required | | version | The Kubernetes version for the nodes in this pool. Should only be set if auto_upgrade is false | " " | Optional | +| location_policy | [Location policy](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_node_pool#location_policy) specifies the algorithm used when scaling-up the node pool. Location policy is supported only in 1.24.1+ clusters. | " " | Optional | ## windows_node_pools variable The windows_node_pools variable takes the same parameters as [node_pools](#node\_pools-variable) but is reserved for provisioning Windows based node pools only. This variable is introduced to satisfy a [specific requirement](https://cloud.google.com/kubernetes-engine/docs/how-to/creating-a-cluster-windows#create_a_cluster_and_node_pools) for the presence of at least one linux based node pool in the cluster before a windows based node pool can be created. @@ -333,8 +345,8 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog #### Kubectl - [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x #### Terraform and Plugins -- [Terraform](https://www.terraform.io/downloads.html) 0.12 -- [Terraform Provider for GCP][terraform-provider-google] v3.41 +- [Terraform](https://www.terraform.io/downloads.html) 0.13+ +- [Terraform Provider for GCP][terraform-provider-google] v4.51 #### gcloud Some submodules use the [terraform-google-gcloud](https://github.com/terraform-google-modules/terraform-google-gcloud) module. By default, this module assumes you already have gcloud installed in your $PATH. See the [module](https://github.com/terraform-google-modules/terraform-google-gcloud#downloading) documentation for more information. diff --git a/modules/private-cluster-update-variant/cluster.tf b/modules/private-cluster-update-variant/cluster.tf index 0d177c90ef..eafefad05f 100644 --- a/modules/private-cluster-update-variant/cluster.tf +++ b/modules/private-cluster-update-variant/cluster.tf @@ -47,6 +47,15 @@ resource "google_container_cluster" "primary" { channel = release_channel.value.channel } } + + dynamic "gateway_api_config" { + for_each = local.gateway_api_config + + content { + channel = gateway_api_config.value.channel + } + } + dynamic "cost_management_config" { for_each = var.enable_cost_allocation ? [1] : [] content { @@ -62,8 +71,31 @@ resource "google_container_cluster" "primary" { min_master_version = var.release_channel == null || var.release_channel == "UNSPECIFIED" ? local.master_version : null - logging_service = var.logging_service - monitoring_service = var.monitoring_service + # only one of logging/monitoring_service or logging/monitoring_config can be specified + logging_service = local.logmon_config_is_set ? null : var.logging_service + dynamic "logging_config" { + for_each = length(var.logging_enabled_components) > 0 ? [1] : [] + + content { + enable_components = var.logging_enabled_components + } + } + monitoring_service = local.logmon_config_is_set ? null : var.monitoring_service + dynamic "monitoring_config" { + for_each = length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus ? [1] : [] + + content { + enable_components = length(var.monitoring_enabled_components) > 0 ? var.monitoring_enabled_components : [] + + dynamic "managed_prometheus" { + for_each = var.monitoring_enable_managed_prometheus ? [1] : [] + + content { + enabled = var.monitoring_enable_managed_prometheus + } + } + } + } cluster_autoscaling { enabled = var.cluster_autoscaling.enabled dynamic "auto_provisioning_defaults" { @@ -72,6 +104,12 @@ resource "google_container_cluster" "primary" { content { service_account = local.service_account oauth_scopes = local.node_pools_oauth_scopes["all"] + + management { + auto_repair = lookup(var.cluster_autoscaling, "auto_repair", true) + auto_upgrade = lookup(var.cluster_autoscaling, "auto_upgrade", true) + } + } } dynamic "resource_limits" { @@ -96,6 +134,8 @@ resource "google_container_cluster" "primary" { } } + enable_kubernetes_alpha = var.enable_kubernetes_alpha + dynamic "master_authorized_networks_config" { for_each = local.master_authorized_networks_config content { @@ -131,7 +171,6 @@ resource "google_container_cluster" "primary" { disabled = !var.horizontal_pod_autoscaling } - network_policy_config { disabled = !var.network_policy } @@ -143,6 +182,22 @@ resource "google_container_cluster" "primary" { gcp_filestore_csi_driver_config { enabled = var.filestore_csi_driver } + + dynamic "gce_persistent_disk_csi_driver_config" { + for_each = local.cluster_gce_pd_csi_config + + content { + enabled = gce_persistent_disk_csi_driver_config.value.enabled + } + } + + dynamic "gke_backup_agent_config" { + for_each = local.gke_backup_agent_config + + content { + enabled = gke_backup_agent_config.value.enabled + } + } } datapath_provider = var.datapath_provider @@ -490,6 +545,10 @@ resource "google_container_node_pool" "pools" { local.node_pools_labels["all"], local.node_pools_labels[each.value["name"]], ) + resource_labels = merge( + local.node_pools_resource_labels["all"], + local.node_pools_resource_labels[each.value["name"]], + ) metadata = merge( lookup(lookup(local.node_pools_metadata, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_metadata, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, @@ -553,6 +612,20 @@ resource "google_container_node_pool" "pools" { } + dynamic "linux_node_config" { + for_each = length(merge( + local.node_pools_linux_node_configs_sysctls["all"], + local.node_pools_linux_node_configs_sysctls[each.value["name"]] + )) != 0 ? [1] : [] + + content { + sysctls = merge( + local.node_pools_linux_node_configs_sysctls["all"], + local.node_pools_linux_node_configs_sysctls[each.value["name"]] + ) + } + } + boot_disk_kms_key = lookup(each.value, "boot_disk_kms_key", "") shielded_instance_config { @@ -645,6 +718,10 @@ resource "google_container_node_pool" "windows_pools" { local.node_pools_labels["all"], local.node_pools_labels[each.value["name"]], ) + resource_labels = merge( + local.node_pools_resource_labels["all"], + local.node_pools_resource_labels[each.value["name"]], + ) metadata = merge( lookup(lookup(local.node_pools_metadata, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_metadata, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, @@ -708,6 +785,7 @@ resource "google_container_node_pool" "windows_pools" { } + boot_disk_kms_key = lookup(each.value, "boot_disk_kms_key", "") shielded_instance_config { diff --git a/modules/private-cluster-update-variant/firewall.tf b/modules/private-cluster-update-variant/firewall.tf index c9b4d29972..d9507532c3 100644 --- a/modules/private-cluster-update-variant/firewall.tf +++ b/modules/private-cluster-update-variant/firewall.tf @@ -34,11 +34,12 @@ resource "google_compute_firewall" "intra_egress" { direction = "EGRESS" target_tags = [local.cluster_network_tag] - destination_ranges = [ + destination_ranges = concat([ local.cluster_endpoint_for_nodes, local.cluster_subnet_cidr, - local.cluster_alias_ranges_cidr[var.ip_range_pods], - ] + ], + local.pod_all_ip_ranges + ) # Allow all possible protocols allow { protocol = "tcp" } @@ -93,7 +94,7 @@ resource "google_compute_firewall" "shadow_allow_pods" { priority = var.shadow_firewall_rules_priority direction = "INGRESS" - source_ranges = [local.cluster_alias_ranges_cidr[var.ip_range_pods]] + source_ranges = local.pod_all_ip_ranges target_tags = [local.cluster_network_tag] # Allow all possible protocols @@ -104,8 +105,11 @@ resource "google_compute_firewall" "shadow_allow_pods" { allow { protocol = "esp" } allow { protocol = "ah" } - log_config { - metadata = "INCLUDE_ALL_METADATA" + dynamic "log_config" { + for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config] + content { + metadata = log_config.value.metadata + } } } @@ -127,8 +131,11 @@ resource "google_compute_firewall" "shadow_allow_master" { ports = ["10250", "443"] } - log_config { - metadata = "INCLUDE_ALL_METADATA" + dynamic "log_config" { + for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config] + content { + metadata = log_config.value.metadata + } } } @@ -159,7 +166,63 @@ resource "google_compute_firewall" "shadow_allow_nodes" { ports = ["1-65535"] } - log_config { - metadata = "INCLUDE_ALL_METADATA" + dynamic "log_config" { + for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config] + content { + metadata = log_config.value.metadata + } + } +} + +resource "google_compute_firewall" "shadow_allow_inkubelet" { + count = var.add_shadow_firewall_rules ? 1 : 0 + + name = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-inkubelet" + description = "Managed by terraform GKE module: A shadow firewall rule to match the default rule allowing worker nodes & pods communication to kubelet." + project = local.network_project_id + network = var.network + priority = var.shadow_firewall_rules_priority - 1 # rule created by GKE robot have prio 999 + direction = "INGRESS" + + source_ranges = local.pod_all_ip_ranges + source_tags = [local.cluster_network_tag] + target_tags = [local.cluster_network_tag] + + allow { + protocol = "tcp" + ports = ["10255"] + } + + dynamic "log_config" { + for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config] + content { + metadata = log_config.value.metadata + } + } +} + +resource "google_compute_firewall" "shadow_deny_exkubelet" { + count = var.add_shadow_firewall_rules ? 1 : 0 + + name = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-exkubelet" + description = "Managed by terraform GKE module: A shadow firewall rule to match the default deny rule to kubelet." + project = local.network_project_id + network = var.network + priority = var.shadow_firewall_rules_priority # rule created by GKE robot have prio 1000 + direction = "INGRESS" + + source_ranges = ["0.0.0.0/0"] + target_tags = [local.cluster_network_tag] + + deny { + protocol = "tcp" + ports = ["10255"] + } + + dynamic "log_config" { + for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config] + content { + metadata = log_config.value.metadata + } } } diff --git a/modules/private-cluster-update-variant/main.tf b/modules/private-cluster-update-variant/main.tf index b5a546d874..1082e1abca 100644 --- a/modules/private-cluster-update-variant/main.tf +++ b/modules/private-cluster-update-variant/main.tf @@ -50,7 +50,8 @@ locals { windows_node_pool_names = [for np in toset(var.windows_node_pools) : np.name] windows_node_pools = zipmap(local.windows_node_pool_names, tolist(toset(var.windows_node_pools))) - release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] + release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] + gateway_api_config = var.gateway_api_channel != null ? [{ channel : var.gateway_api_channel }] : [] autoscaling_resource_limits = var.cluster_autoscaling.enabled ? concat([{ resource_type = "cpu" @@ -73,6 +74,7 @@ locals { cluster_subnet_cidr = var.add_cluster_firewall_rules ? data.google_compute_subnetwork.gke_subnetwork[0].ip_cidr_range : null cluster_alias_ranges_cidr = var.add_cluster_firewall_rules ? { for range in toset(data.google_compute_subnetwork.gke_subnetwork[0].secondary_ip_range) : range.range_name => range.ip_cidr_range } : {} + pod_all_ip_ranges = var.add_cluster_firewall_rules ? compact(concat([local.cluster_alias_ranges_cidr[var.ip_range_pods]], [for k, v in merge(local.node_pools, local.windows_node_pools) : local.cluster_alias_ranges_cidr[v.pod_range] if length(lookup(v, "pod_range", "")) > 0])) : [] cluster_network_policy = var.network_policy ? [{ enabled = true @@ -81,6 +83,9 @@ locals { enabled = false provider = null }] + cluster_gce_pd_csi_config = var.gce_pd_csi_driver ? [{ enabled = true }] : [{ enabled = false }] + logmon_config_is_set = length(var.logging_enabled_components) > 0 || length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus + gke_backup_agent_config = var.gke_backup_agent_config ? [{ enabled = true }] : [{ enabled = false }] cluster_authenticator_security_group = var.authenticator_security_group == null ? [] : [{ security_group = var.authenticator_security_group diff --git a/modules/private-cluster-update-variant/outputs.tf b/modules/private-cluster-update-variant/outputs.tf index c3436263f2..28350e722b 100644 --- a/modules/private-cluster-update-variant/outputs.tf +++ b/modules/private-cluster-update-variant/outputs.tf @@ -148,6 +148,11 @@ output "release_channel" { value = var.release_channel } +output "gateway_api_channel" { + description = "The gateway api channel of this cluster." + value = var.gateway_api_channel +} + output "identity_namespace" { description = "Workload Identity pool" value = length(local.cluster_workload_identity_config) > 0 ? local.cluster_workload_identity_config[0].workload_pool : null diff --git a/modules/private-cluster-update-variant/variables.tf b/modules/private-cluster-update-variant/variables.tf index c3d3bb511d..ae93690587 100644 --- a/modules/private-cluster-update-variant/variables.tf +++ b/modules/private-cluster-update-variant/variables.tf @@ -170,6 +170,16 @@ variable "node_pools_labels" { } } +variable "node_pools_resource_labels" { + type = map(map(string)) + description = "Map of maps containing resource labels by node-pool name" + + default = { + all = {} + default-node-pool = {} + } +} + variable "node_pools_metadata" { type = map(map(string)) description = "Map of maps containing node metadata by node-pool name" @@ -181,6 +191,17 @@ variable "node_pools_metadata" { } } +variable "node_pools_linux_node_configs_sysctls" { + type = map(map(string)) + description = "Map of maps containing linux node config sysctls by node-pool name" + + # Default is being set in variables_defaults.tf + default = { + all = {} + default-node-pool = {} + } +} + variable "enable_cost_allocation" { type = bool description = "Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery" @@ -212,6 +233,8 @@ variable "cluster_autoscaling" { min_memory_gb = number max_memory_gb = number gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) + auto_repair = bool + auto_upgrade = bool }) default = { enabled = false @@ -220,6 +243,8 @@ variable "cluster_autoscaling" { max_memory_gb = 0 min_memory_gb = 0 gpu_resources = [] + auto_repair = true + auto_upgrade = true } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } @@ -407,6 +432,12 @@ variable "release_channel" { default = null } +variable "gateway_api_channel" { + type = string + description = "The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`." + default = null +} + variable "add_cluster_firewall_rules" { type = bool description = "Create additional firewall rules" @@ -441,6 +472,20 @@ variable "shadow_firewall_rules_priority" { type = number description = "The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000." default = 999 + validation { + condition = var.shadow_firewall_rules_priority < 1000 + error_message = "The shadow firewall rule priority must be lower than auto-created one(1000)." + } +} + +variable "shadow_firewall_rules_log_config" { + type = object({ + metadata = string + }) + description = "The log_config for shadow firewall rules. You can set this variable to `null` to disable logging." + default = { + metadata = "INCLUDE_ALL_METADATA" + } } @@ -527,7 +572,7 @@ variable "node_metadata" { validation { condition = contains(["GKE_METADATA", "GCE_METADATA", "UNSPECIFIED", "GKE_METADATA_SERVER", "EXPOSE"], var.node_metadata) - error_message = "The node_metadata value must be one of GKE_METADATA, GCE_METADATA or UNSPECIFIED." + error_message = "The node_metadata value must be one of GKE_METADATA, GCE_METADATA, UNSPECIFIED, GKE_METADATA_SERVER or EXPOSE." } } @@ -549,6 +594,18 @@ variable "cluster_dns_domain" { default = "" } +variable "gce_pd_csi_driver" { + type = bool + description = "Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver." + default = true +} + +variable "gke_backup_agent_config" { + type = bool + description = "Whether Backup for GKE agent is enabled for this cluster." + default = false +} + variable "timeouts" { type = map(string) description = "Timeout for cluster operations." @@ -558,3 +615,27 @@ variable "timeouts" { error_message = "Only create, update, delete timeouts can be specified." } } + +variable "monitoring_enable_managed_prometheus" { + type = bool + description = "Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled." + default = false +} + +variable "monitoring_enabled_components" { + type = list(string) + description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration." + default = [] +} + +variable "logging_enabled_components" { + type = list(string) + description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." + default = [] +} + +variable "enable_kubernetes_alpha" { + type = bool + description = "Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days." + default = false +} diff --git a/modules/private-cluster-update-variant/variables_defaults.tf b/modules/private-cluster-update-variant/variables_defaults.tf index b570f5f850..e4f3004771 100644 --- a/modules/private-cluster-update-variant/variables_defaults.tf +++ b/modules/private-cluster-update-variant/variables_defaults.tf @@ -34,6 +34,20 @@ locals { var.node_pools_labels ) + node_pools_resource_labels = merge( + { all = {} }, + { default-node-pool = {} }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : {}] + ), + zipmap( + [for node_pool in var.windows_node_pools : node_pool["name"]], + [for node_pool in var.windows_node_pools : {}] + ), + var.node_pools_resource_labels + ) + node_pools_metadata = merge( { all = {} }, { default-node-pool = {} }, @@ -89,4 +103,14 @@ locals { ), var.node_pools_oauth_scopes ) + + node_pools_linux_node_configs_sysctls = merge( + { all = {} }, + { default-node-pool = {} }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : {}] + ), + var.node_pools_linux_node_configs_sysctls + ) } diff --git a/modules/private-cluster-update-variant/versions.tf b/modules/private-cluster-update-variant/versions.tf index d59900d203..55c4424ee8 100644 --- a/modules/private-cluster-update-variant/versions.tf +++ b/modules/private-cluster-update-variant/versions.tf @@ -21,7 +21,7 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 4.36.0, < 5.0" + version = ">= 4.51.0, < 5.0" } kubernetes = { source = "hashicorp/kubernetes" @@ -29,6 +29,6 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:private-cluster-update-variant/v24.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:private-cluster-update-variant/v25.0.0" } } diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md index a336389570..e82ae76c2b 100644 --- a/modules/private-cluster/README.md +++ b/modules/private-cluster/README.md @@ -137,7 +137,7 @@ Then perform the following commands on the root folder: | add\_master\_webhook\_firewall\_rules | Create master\_webhook firewall rules for ports defined in `firewall_inbound_ports` | `bool` | `false` | no | | add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | | authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled       = bool
    min_cpu_cores = number
    max_cpu_cores = number
    min_memory_gb = number
    max_memory_gb = number
    gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number }))
  })
| 
{
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled       = bool
    min_cpu_cores = number
    max_cpu_cores = number
    min_memory_gb = number
    max_memory_gb = number
    gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair   = bool
    auto_upgrade  = bool
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | | cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | | cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | | cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | @@ -155,6 +155,7 @@ Then perform the following commands on the root folder: | dns\_cache | The status of the NodeLocal DNSCache addon. | `bool` | `false` | no | | enable\_binary\_authorization | Enable BinAuthZ Admission controller | `bool` | `false` | no | | enable\_cost\_allocation | Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery | `bool` | `false` | no | +| enable\_kubernetes\_alpha | Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days. | `bool` | `false` | no | | enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `bool` | `false` | no | | enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | `bool` | `false` | no | | enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | `bool` | `false` | no | @@ -164,6 +165,9 @@ Then perform the following commands on the root folder: | filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | | firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | | firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | +| gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | +| gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | +| gke\_backup\_agent\_config | Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | | http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no | @@ -175,6 +179,7 @@ Then perform the following commands on the root folder: | ip\_range\_services | The _name_ of the secondary subnet range to use for services | `string` | n/a | yes | | issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | | kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | `string` | `"latest"` | no | +| logging\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | | logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | `string` | `"logging.googleapis.com/kubernetes"` | no | | maintenance\_end\_time | Time window specified for recurring maintenance operations in RFC3339 format | `string` | `""` | no | | maintenance\_exclusions | List of maintenance exclusions. A cluster can have up to three | `list(object({ name = string, start_time = string, end_time = string, exclusion_scope = string }))` | `[]` | no | @@ -183,6 +188,8 @@ Then perform the following commands on the root folder: | master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | | master\_global\_access\_enabled | Whether the cluster master is accessible globally (from any region) or only within the same region as the private endpoint. | `bool` | `true` | no | | master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | `string` | `"10.0.0.0/28"` | no | +| monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `false` | no | +| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration. | `list(string)` | `[]` | no | | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | | name | The name of the cluster (required) | `string` | n/a | yes | | network | The VPC network to host the cluster in (required) | `string` | n/a | yes | @@ -192,8 +199,10 @@ Then perform the following commands on the root folder: | node\_metadata | Specifies how node metadata is exposed to the workload running on the node | `string` | `"GKE_METADATA"` | no | | node\_pools | List of maps containing node pools | `list(map(any))` | 
[
  {
    "name": "default-node-pool"
  }
]
| no | | node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_linux\_node\_configs\_sysctls | Map of maps containing linux node config sysctls by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | | node\_pools\_metadata | Map of maps containing node metadata by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | | node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | `map(list(string))` | 
{
  "all": [
    "https://www.googleapis.com/auth/cloud-platform"
  ],
  "default-node-pool": []
}
| no | +| node\_pools\_resource\_labels | Map of maps containing resource labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | | node\_pools\_tags | Map of lists containing node network tags by node-pool name | `map(list(string))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | | node\_pools\_taints | Map of lists containing node taints by node-pool name | `map(list(object({ key = string, value = string, effect = string })))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | | non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | `list(string)` | 
[
  "10.0.0.0/8",
  "172.16.0.0/12",
  "192.168.0.0/16"
]
| no | @@ -207,6 +216,7 @@ Then perform the following commands on the root folder: | resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. | `string` | `""` | no | | service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | +| shadow\_firewall\_rules\_log\_config | The log\_config for shadow firewall rules. You can set this variable to `null` to disable logging. | 
object({
    metadata = string
  })
| 
{
  "metadata": "INCLUDE_ALL_METADATA"
}
| no | | shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | | skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | `bool` | `false` | no | | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | @@ -223,6 +233,7 @@ Then perform the following commands on the root folder: | ca\_certificate | Cluster ca certificate (base64 encoded) | | cluster\_id | Cluster ID | | endpoint | Cluster endpoint | +| gateway\_api\_channel | The gateway api channel of this cluster. | | horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | | http\_load\_balancing\_enabled | Whether http load balancing enabled | | identity\_namespace | Workload Identity pool | @@ -291,6 +302,7 @@ The node_pools variable takes the following parameters: | tags | The list of instance tags applied to all nodes | | Required | | value | The value for the taint | | Required | | version | The Kubernetes version for the nodes in this pool. Should only be set if auto_upgrade is false | " " | Optional | +| location_policy | [Location policy](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_node_pool#location_policy) specifies the algorithm used when scaling-up the node pool. Location policy is supported only in 1.24.1+ clusters. | " " | Optional | ## windows_node_pools variable The windows_node_pools variable takes the same parameters as [node_pools](#node\_pools-variable) but is reserved for provisioning Windows based node pools only. This variable is introduced to satisfy a [specific requirement](https://cloud.google.com/kubernetes-engine/docs/how-to/creating-a-cluster-windows#create_a_cluster_and_node_pools) for the presence of at least one linux based node pool in the cluster before a windows based node pool can be created. @@ -311,8 +323,8 @@ The [project factory](https://github.com/terraform-google-modules/terraform-goog #### Kubectl - [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x #### Terraform and Plugins -- [Terraform](https://www.terraform.io/downloads.html) 0.12 -- [Terraform Provider for GCP][terraform-provider-google] v3.41 +- [Terraform](https://www.terraform.io/downloads.html) 0.13+ +- [Terraform Provider for GCP][terraform-provider-google] v4.51 #### gcloud Some submodules use the [terraform-google-gcloud](https://github.com/terraform-google-modules/terraform-google-gcloud) module. By default, this module assumes you already have gcloud installed in your $PATH. See the [module](https://github.com/terraform-google-modules/terraform-google-gcloud#downloading) documentation for more information. diff --git a/modules/private-cluster/cluster.tf b/modules/private-cluster/cluster.tf index f8dd0b8cee..9711d27d98 100644 --- a/modules/private-cluster/cluster.tf +++ b/modules/private-cluster/cluster.tf @@ -47,6 +47,15 @@ resource "google_container_cluster" "primary" { channel = release_channel.value.channel } } + + dynamic "gateway_api_config" { + for_each = local.gateway_api_config + + content { + channel = gateway_api_config.value.channel + } + } + dynamic "cost_management_config" { for_each = var.enable_cost_allocation ? [1] : [] content { @@ -62,8 +71,31 @@ resource "google_container_cluster" "primary" { min_master_version = var.release_channel == null || var.release_channel == "UNSPECIFIED" ? local.master_version : null - logging_service = var.logging_service - monitoring_service = var.monitoring_service + # only one of logging/monitoring_service or logging/monitoring_config can be specified + logging_service = local.logmon_config_is_set ? null : var.logging_service + dynamic "logging_config" { + for_each = length(var.logging_enabled_components) > 0 ? [1] : [] + + content { + enable_components = var.logging_enabled_components + } + } + monitoring_service = local.logmon_config_is_set ? null : var.monitoring_service + dynamic "monitoring_config" { + for_each = length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus ? [1] : [] + + content { + enable_components = length(var.monitoring_enabled_components) > 0 ? var.monitoring_enabled_components : [] + + dynamic "managed_prometheus" { + for_each = var.monitoring_enable_managed_prometheus ? [1] : [] + + content { + enabled = var.monitoring_enable_managed_prometheus + } + } + } + } cluster_autoscaling { enabled = var.cluster_autoscaling.enabled dynamic "auto_provisioning_defaults" { @@ -72,6 +104,12 @@ resource "google_container_cluster" "primary" { content { service_account = local.service_account oauth_scopes = local.node_pools_oauth_scopes["all"] + + management { + auto_repair = lookup(var.cluster_autoscaling, "auto_repair", true) + auto_upgrade = lookup(var.cluster_autoscaling, "auto_upgrade", true) + } + } } dynamic "resource_limits" { @@ -96,6 +134,8 @@ resource "google_container_cluster" "primary" { } } + enable_kubernetes_alpha = var.enable_kubernetes_alpha + dynamic "master_authorized_networks_config" { for_each = local.master_authorized_networks_config content { @@ -131,7 +171,6 @@ resource "google_container_cluster" "primary" { disabled = !var.horizontal_pod_autoscaling } - network_policy_config { disabled = !var.network_policy } @@ -143,6 +182,22 @@ resource "google_container_cluster" "primary" { gcp_filestore_csi_driver_config { enabled = var.filestore_csi_driver } + + dynamic "gce_persistent_disk_csi_driver_config" { + for_each = local.cluster_gce_pd_csi_config + + content { + enabled = gce_persistent_disk_csi_driver_config.value.enabled + } + } + + dynamic "gke_backup_agent_config" { + for_each = local.gke_backup_agent_config + + content { + enabled = gke_backup_agent_config.value.enabled + } + } } datapath_provider = var.datapath_provider @@ -396,6 +451,10 @@ resource "google_container_node_pool" "pools" { local.node_pools_labels["all"], local.node_pools_labels[each.value["name"]], ) + resource_labels = merge( + local.node_pools_resource_labels["all"], + local.node_pools_resource_labels[each.value["name"]], + ) metadata = merge( lookup(lookup(local.node_pools_metadata, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_metadata, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, @@ -459,6 +518,20 @@ resource "google_container_node_pool" "pools" { } + dynamic "linux_node_config" { + for_each = length(merge( + local.node_pools_linux_node_configs_sysctls["all"], + local.node_pools_linux_node_configs_sysctls[each.value["name"]] + )) != 0 ? [1] : [] + + content { + sysctls = merge( + local.node_pools_linux_node_configs_sysctls["all"], + local.node_pools_linux_node_configs_sysctls[each.value["name"]] + ) + } + } + boot_disk_kms_key = lookup(each.value, "boot_disk_kms_key", "") shielded_instance_config { @@ -550,6 +623,10 @@ resource "google_container_node_pool" "windows_pools" { local.node_pools_labels["all"], local.node_pools_labels[each.value["name"]], ) + resource_labels = merge( + local.node_pools_resource_labels["all"], + local.node_pools_resource_labels[each.value["name"]], + ) metadata = merge( lookup(lookup(local.node_pools_metadata, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_metadata, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, @@ -613,6 +690,7 @@ resource "google_container_node_pool" "windows_pools" { } + boot_disk_kms_key = lookup(each.value, "boot_disk_kms_key", "") shielded_instance_config { diff --git a/modules/private-cluster/firewall.tf b/modules/private-cluster/firewall.tf index c9b4d29972..d9507532c3 100644 --- a/modules/private-cluster/firewall.tf +++ b/modules/private-cluster/firewall.tf @@ -34,11 +34,12 @@ resource "google_compute_firewall" "intra_egress" { direction = "EGRESS" target_tags = [local.cluster_network_tag] - destination_ranges = [ + destination_ranges = concat([ local.cluster_endpoint_for_nodes, local.cluster_subnet_cidr, - local.cluster_alias_ranges_cidr[var.ip_range_pods], - ] + ], + local.pod_all_ip_ranges + ) # Allow all possible protocols allow { protocol = "tcp" } @@ -93,7 +94,7 @@ resource "google_compute_firewall" "shadow_allow_pods" { priority = var.shadow_firewall_rules_priority direction = "INGRESS" - source_ranges = [local.cluster_alias_ranges_cidr[var.ip_range_pods]] + source_ranges = local.pod_all_ip_ranges target_tags = [local.cluster_network_tag] # Allow all possible protocols @@ -104,8 +105,11 @@ resource "google_compute_firewall" "shadow_allow_pods" { allow { protocol = "esp" } allow { protocol = "ah" } - log_config { - metadata = "INCLUDE_ALL_METADATA" + dynamic "log_config" { + for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config] + content { + metadata = log_config.value.metadata + } } } @@ -127,8 +131,11 @@ resource "google_compute_firewall" "shadow_allow_master" { ports = ["10250", "443"] } - log_config { - metadata = "INCLUDE_ALL_METADATA" + dynamic "log_config" { + for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config] + content { + metadata = log_config.value.metadata + } } } @@ -159,7 +166,63 @@ resource "google_compute_firewall" "shadow_allow_nodes" { ports = ["1-65535"] } - log_config { - metadata = "INCLUDE_ALL_METADATA" + dynamic "log_config" { + for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config] + content { + metadata = log_config.value.metadata + } + } +} + +resource "google_compute_firewall" "shadow_allow_inkubelet" { + count = var.add_shadow_firewall_rules ? 1 : 0 + + name = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-inkubelet" + description = "Managed by terraform GKE module: A shadow firewall rule to match the default rule allowing worker nodes & pods communication to kubelet." + project = local.network_project_id + network = var.network + priority = var.shadow_firewall_rules_priority - 1 # rule created by GKE robot have prio 999 + direction = "INGRESS" + + source_ranges = local.pod_all_ip_ranges + source_tags = [local.cluster_network_tag] + target_tags = [local.cluster_network_tag] + + allow { + protocol = "tcp" + ports = ["10255"] + } + + dynamic "log_config" { + for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config] + content { + metadata = log_config.value.metadata + } + } +} + +resource "google_compute_firewall" "shadow_deny_exkubelet" { + count = var.add_shadow_firewall_rules ? 1 : 0 + + name = "gke-shadow-${substr(var.name, 0, min(25, length(var.name)))}-exkubelet" + description = "Managed by terraform GKE module: A shadow firewall rule to match the default deny rule to kubelet." + project = local.network_project_id + network = var.network + priority = var.shadow_firewall_rules_priority # rule created by GKE robot have prio 1000 + direction = "INGRESS" + + source_ranges = ["0.0.0.0/0"] + target_tags = [local.cluster_network_tag] + + deny { + protocol = "tcp" + ports = ["10255"] + } + + dynamic "log_config" { + for_each = var.shadow_firewall_rules_log_config == null ? [] : [var.shadow_firewall_rules_log_config] + content { + metadata = log_config.value.metadata + } } } diff --git a/modules/private-cluster/main.tf b/modules/private-cluster/main.tf index b5a546d874..1082e1abca 100644 --- a/modules/private-cluster/main.tf +++ b/modules/private-cluster/main.tf @@ -50,7 +50,8 @@ locals { windows_node_pool_names = [for np in toset(var.windows_node_pools) : np.name] windows_node_pools = zipmap(local.windows_node_pool_names, tolist(toset(var.windows_node_pools))) - release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] + release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] + gateway_api_config = var.gateway_api_channel != null ? [{ channel : var.gateway_api_channel }] : [] autoscaling_resource_limits = var.cluster_autoscaling.enabled ? concat([{ resource_type = "cpu" @@ -73,6 +74,7 @@ locals { cluster_subnet_cidr = var.add_cluster_firewall_rules ? data.google_compute_subnetwork.gke_subnetwork[0].ip_cidr_range : null cluster_alias_ranges_cidr = var.add_cluster_firewall_rules ? { for range in toset(data.google_compute_subnetwork.gke_subnetwork[0].secondary_ip_range) : range.range_name => range.ip_cidr_range } : {} + pod_all_ip_ranges = var.add_cluster_firewall_rules ? compact(concat([local.cluster_alias_ranges_cidr[var.ip_range_pods]], [for k, v in merge(local.node_pools, local.windows_node_pools) : local.cluster_alias_ranges_cidr[v.pod_range] if length(lookup(v, "pod_range", "")) > 0])) : [] cluster_network_policy = var.network_policy ? [{ enabled = true @@ -81,6 +83,9 @@ locals { enabled = false provider = null }] + cluster_gce_pd_csi_config = var.gce_pd_csi_driver ? [{ enabled = true }] : [{ enabled = false }] + logmon_config_is_set = length(var.logging_enabled_components) > 0 || length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus + gke_backup_agent_config = var.gke_backup_agent_config ? [{ enabled = true }] : [{ enabled = false }] cluster_authenticator_security_group = var.authenticator_security_group == null ? [] : [{ security_group = var.authenticator_security_group diff --git a/modules/private-cluster/outputs.tf b/modules/private-cluster/outputs.tf index c3436263f2..28350e722b 100644 --- a/modules/private-cluster/outputs.tf +++ b/modules/private-cluster/outputs.tf @@ -148,6 +148,11 @@ output "release_channel" { value = var.release_channel } +output "gateway_api_channel" { + description = "The gateway api channel of this cluster." + value = var.gateway_api_channel +} + output "identity_namespace" { description = "Workload Identity pool" value = length(local.cluster_workload_identity_config) > 0 ? local.cluster_workload_identity_config[0].workload_pool : null diff --git a/modules/private-cluster/variables.tf b/modules/private-cluster/variables.tf index c3d3bb511d..ae93690587 100644 --- a/modules/private-cluster/variables.tf +++ b/modules/private-cluster/variables.tf @@ -170,6 +170,16 @@ variable "node_pools_labels" { } } +variable "node_pools_resource_labels" { + type = map(map(string)) + description = "Map of maps containing resource labels by node-pool name" + + default = { + all = {} + default-node-pool = {} + } +} + variable "node_pools_metadata" { type = map(map(string)) description = "Map of maps containing node metadata by node-pool name" @@ -181,6 +191,17 @@ variable "node_pools_metadata" { } } +variable "node_pools_linux_node_configs_sysctls" { + type = map(map(string)) + description = "Map of maps containing linux node config sysctls by node-pool name" + + # Default is being set in variables_defaults.tf + default = { + all = {} + default-node-pool = {} + } +} + variable "enable_cost_allocation" { type = bool description = "Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery" @@ -212,6 +233,8 @@ variable "cluster_autoscaling" { min_memory_gb = number max_memory_gb = number gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) + auto_repair = bool + auto_upgrade = bool }) default = { enabled = false @@ -220,6 +243,8 @@ variable "cluster_autoscaling" { max_memory_gb = 0 min_memory_gb = 0 gpu_resources = [] + auto_repair = true + auto_upgrade = true } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } @@ -407,6 +432,12 @@ variable "release_channel" { default = null } +variable "gateway_api_channel" { + type = string + description = "The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`." + default = null +} + variable "add_cluster_firewall_rules" { type = bool description = "Create additional firewall rules" @@ -441,6 +472,20 @@ variable "shadow_firewall_rules_priority" { type = number description = "The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000." default = 999 + validation { + condition = var.shadow_firewall_rules_priority < 1000 + error_message = "The shadow firewall rule priority must be lower than auto-created one(1000)." + } +} + +variable "shadow_firewall_rules_log_config" { + type = object({ + metadata = string + }) + description = "The log_config for shadow firewall rules. You can set this variable to `null` to disable logging." + default = { + metadata = "INCLUDE_ALL_METADATA" + } } @@ -527,7 +572,7 @@ variable "node_metadata" { validation { condition = contains(["GKE_METADATA", "GCE_METADATA", "UNSPECIFIED", "GKE_METADATA_SERVER", "EXPOSE"], var.node_metadata) - error_message = "The node_metadata value must be one of GKE_METADATA, GCE_METADATA or UNSPECIFIED." + error_message = "The node_metadata value must be one of GKE_METADATA, GCE_METADATA, UNSPECIFIED, GKE_METADATA_SERVER or EXPOSE." } } @@ -549,6 +594,18 @@ variable "cluster_dns_domain" { default = "" } +variable "gce_pd_csi_driver" { + type = bool + description = "Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver." + default = true +} + +variable "gke_backup_agent_config" { + type = bool + description = "Whether Backup for GKE agent is enabled for this cluster." + default = false +} + variable "timeouts" { type = map(string) description = "Timeout for cluster operations." @@ -558,3 +615,27 @@ variable "timeouts" { error_message = "Only create, update, delete timeouts can be specified." } } + +variable "monitoring_enable_managed_prometheus" { + type = bool + description = "Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled." + default = false +} + +variable "monitoring_enabled_components" { + type = list(string) + description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration." + default = [] +} + +variable "logging_enabled_components" { + type = list(string) + description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." + default = [] +} + +variable "enable_kubernetes_alpha" { + type = bool + description = "Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days." + default = false +} diff --git a/modules/private-cluster/variables_defaults.tf b/modules/private-cluster/variables_defaults.tf index b570f5f850..e4f3004771 100644 --- a/modules/private-cluster/variables_defaults.tf +++ b/modules/private-cluster/variables_defaults.tf @@ -34,6 +34,20 @@ locals { var.node_pools_labels ) + node_pools_resource_labels = merge( + { all = {} }, + { default-node-pool = {} }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : {}] + ), + zipmap( + [for node_pool in var.windows_node_pools : node_pool["name"]], + [for node_pool in var.windows_node_pools : {}] + ), + var.node_pools_resource_labels + ) + node_pools_metadata = merge( { all = {} }, { default-node-pool = {} }, @@ -89,4 +103,14 @@ locals { ), var.node_pools_oauth_scopes ) + + node_pools_linux_node_configs_sysctls = merge( + { all = {} }, + { default-node-pool = {} }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : {}] + ), + var.node_pools_linux_node_configs_sysctls + ) } diff --git a/modules/private-cluster/versions.tf b/modules/private-cluster/versions.tf index f914717b7d..d8bbe63be4 100644 --- a/modules/private-cluster/versions.tf +++ b/modules/private-cluster/versions.tf @@ -21,7 +21,7 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 4.36.0, < 5.0" + version = ">= 4.51.0, < 5.0" } kubernetes = { source = "hashicorp/kubernetes" @@ -29,6 +29,6 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:private-cluster/v24.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:private-cluster/v25.0.0" } } diff --git a/modules/safer-cluster-update-variant/README.md b/modules/safer-cluster-update-variant/README.md index 3a67aa66b0..e47612dcbb 100644 --- a/modules/safer-cluster-update-variant/README.md +++ b/modules/safer-cluster-update-variant/README.md @@ -204,7 +204,7 @@ For simplicity, we suggest using `roles/container.admin` and | add\_cluster\_firewall\_rules | Create additional firewall rules | `bool` | `false` | no | | authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | | cloudrun | (Beta) Enable CloudRun addon | `bool` | `false` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources       = list(object({ resource_type = string, minimum = number, maximum = number }))
  })
| 
{
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources       = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair         = bool
    auto_upgrade        = bool
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | | cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | | cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | | cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | @@ -228,6 +228,7 @@ For simplicity, we suggest using `roles/container.admin` and | filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | | firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | | firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | +| gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | | gce\_pd\_csi\_driver | (Beta) Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | | gke\_backup\_agent\_config | (Beta) Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | `bool` | `true` | no | @@ -255,6 +256,7 @@ For simplicity, we suggest using `roles/container.admin` and | node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | | node\_pools\_metadata | Map of maps containing node metadata by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | | node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | `map(list(string))` | 
{
  "all": [
    "https://www.googleapis.com/auth/cloud-platform"
  ],
  "default-node-pool": []
}
| no | +| node\_pools\_resource\_labels | Map of maps containing resource labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | | node\_pools\_tags | Map of lists containing node network tags by node-pool name | `map(list(string))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | | node\_pools\_taints | Map of lists containing node taints by node-pool name | `map(list(object({ key = string, value = string, effect = string })))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | | notification\_config\_topic | The desired Pub/Sub topic to which notifications will be sent by GKE. Format is projects/{project}/topics/{topic}. | `string` | `""` | no | diff --git a/modules/safer-cluster-update-variant/main.tf b/modules/safer-cluster-update-variant/main.tf index 73a413d335..9366427a43 100644 --- a/modules/safer-cluster-update-variant/main.tf +++ b/modules/safer-cluster-update-variant/main.tf @@ -37,7 +37,8 @@ module "gke" { // the master upgrades. // // https://cloud.google.com/kubernetes-engine/versioning-and-upgrades - release_channel = var.release_channel + release_channel = var.release_channel + gateway_api_channel = var.gateway_api_channel master_authorized_networks = var.master_authorized_networks @@ -82,12 +83,13 @@ module "gke" { // If removing the default node pool, initial_node_count should be at least 1. initial_node_count = (var.initial_node_count == 0) ? 1 : var.initial_node_count - node_pools = var.node_pools - windows_node_pools = var.windows_node_pools - node_pools_labels = var.node_pools_labels - node_pools_metadata = var.node_pools_metadata - node_pools_taints = var.node_pools_taints - node_pools_tags = var.node_pools_tags + node_pools = var.node_pools + windows_node_pools = var.windows_node_pools + node_pools_labels = var.node_pools_labels + node_pools_resource_labels = var.node_pools_resource_labels + node_pools_metadata = var.node_pools_metadata + node_pools_taints = var.node_pools_taints + node_pools_tags = var.node_pools_tags node_pools_oauth_scopes = var.node_pools_oauth_scopes diff --git a/modules/safer-cluster-update-variant/variables.tf b/modules/safer-cluster-update-variant/variables.tf index cc53aaed63..ceb2611cb0 100644 --- a/modules/safer-cluster-update-variant/variables.tf +++ b/modules/safer-cluster-update-variant/variables.tf @@ -77,6 +77,12 @@ variable "release_channel" { default = "REGULAR" } +variable "gateway_api_channel" { + type = string + description = "The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`." + default = null +} + variable "master_authorized_networks" { type = list(object({ cidr_block = string, display_name = string })) description = "List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists)." @@ -168,6 +174,16 @@ variable "node_pools_labels" { } } +variable "node_pools_resource_labels" { + type = map(map(string)) + description = "Map of maps containing resource labels by node-pool name" + + default = { + all = {} + default-node-pool = {} + } +} + variable "node_pools_metadata" { type = map(map(string)) description = "Map of maps containing node metadata by node-pool name" @@ -217,6 +233,8 @@ variable "cluster_autoscaling" { min_memory_gb = number max_memory_gb = number gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) + auto_repair = bool + auto_upgrade = bool }) default = { enabled = false @@ -226,6 +244,8 @@ variable "cluster_autoscaling" { max_memory_gb = 0 min_memory_gb = 0 gpu_resources = [] + auto_repair = true + auto_upgrade = true } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } diff --git a/modules/safer-cluster-update-variant/versions.tf b/modules/safer-cluster-update-variant/versions.tf index 775c647b21..cb8edcfd16 100644 --- a/modules/safer-cluster-update-variant/versions.tf +++ b/modules/safer-cluster-update-variant/versions.tf @@ -21,6 +21,6 @@ terraform { required_version = ">=0.13" provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:safer-cluster-update-variant/v24.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:safer-cluster-update-variant/v25.0.0" } } diff --git a/modules/safer-cluster/README.md b/modules/safer-cluster/README.md index 3a67aa66b0..e47612dcbb 100644 --- a/modules/safer-cluster/README.md +++ b/modules/safer-cluster/README.md @@ -204,7 +204,7 @@ For simplicity, we suggest using `roles/container.admin` and | add\_cluster\_firewall\_rules | Create additional firewall rules | `bool` | `false` | no | | authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | | cloudrun | (Beta) Enable CloudRun addon | `bool` | `false` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources       = list(object({ resource_type = string, minimum = number, maximum = number }))
  })
| 
{
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources       = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair         = bool
    auto_upgrade        = bool
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | | cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | | cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | | cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | @@ -228,6 +228,7 @@ For simplicity, we suggest using `roles/container.admin` and | filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | | firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | | firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | +| gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | | gce\_pd\_csi\_driver | (Beta) Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | | gke\_backup\_agent\_config | (Beta) Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | `bool` | `true` | no | @@ -255,6 +256,7 @@ For simplicity, we suggest using `roles/container.admin` and | node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | | node\_pools\_metadata | Map of maps containing node metadata by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | | node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | `map(list(string))` | 
{
  "all": [
    "https://www.googleapis.com/auth/cloud-platform"
  ],
  "default-node-pool": []
}
| no | +| node\_pools\_resource\_labels | Map of maps containing resource labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | | node\_pools\_tags | Map of lists containing node network tags by node-pool name | `map(list(string))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | | node\_pools\_taints | Map of lists containing node taints by node-pool name | `map(list(object({ key = string, value = string, effect = string })))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | | notification\_config\_topic | The desired Pub/Sub topic to which notifications will be sent by GKE. Format is projects/{project}/topics/{topic}. | `string` | `""` | no | diff --git a/modules/safer-cluster/main.tf b/modules/safer-cluster/main.tf index e54e6e68fe..e79a73d8aa 100644 --- a/modules/safer-cluster/main.tf +++ b/modules/safer-cluster/main.tf @@ -37,7 +37,8 @@ module "gke" { // the master upgrades. // // https://cloud.google.com/kubernetes-engine/versioning-and-upgrades - release_channel = var.release_channel + release_channel = var.release_channel + gateway_api_channel = var.gateway_api_channel master_authorized_networks = var.master_authorized_networks @@ -82,12 +83,13 @@ module "gke" { // If removing the default node pool, initial_node_count should be at least 1. initial_node_count = (var.initial_node_count == 0) ? 1 : var.initial_node_count - node_pools = var.node_pools - windows_node_pools = var.windows_node_pools - node_pools_labels = var.node_pools_labels - node_pools_metadata = var.node_pools_metadata - node_pools_taints = var.node_pools_taints - node_pools_tags = var.node_pools_tags + node_pools = var.node_pools + windows_node_pools = var.windows_node_pools + node_pools_labels = var.node_pools_labels + node_pools_resource_labels = var.node_pools_resource_labels + node_pools_metadata = var.node_pools_metadata + node_pools_taints = var.node_pools_taints + node_pools_tags = var.node_pools_tags node_pools_oauth_scopes = var.node_pools_oauth_scopes diff --git a/modules/safer-cluster/variables.tf b/modules/safer-cluster/variables.tf index cc53aaed63..ceb2611cb0 100644 --- a/modules/safer-cluster/variables.tf +++ b/modules/safer-cluster/variables.tf @@ -77,6 +77,12 @@ variable "release_channel" { default = "REGULAR" } +variable "gateway_api_channel" { + type = string + description = "The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`." + default = null +} + variable "master_authorized_networks" { type = list(object({ cidr_block = string, display_name = string })) description = "List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists)." @@ -168,6 +174,16 @@ variable "node_pools_labels" { } } +variable "node_pools_resource_labels" { + type = map(map(string)) + description = "Map of maps containing resource labels by node-pool name" + + default = { + all = {} + default-node-pool = {} + } +} + variable "node_pools_metadata" { type = map(map(string)) description = "Map of maps containing node metadata by node-pool name" @@ -217,6 +233,8 @@ variable "cluster_autoscaling" { min_memory_gb = number max_memory_gb = number gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) + auto_repair = bool + auto_upgrade = bool }) default = { enabled = false @@ -226,6 +244,8 @@ variable "cluster_autoscaling" { max_memory_gb = 0 min_memory_gb = 0 gpu_resources = [] + auto_repair = true + auto_upgrade = true } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } diff --git a/modules/safer-cluster/versions.tf b/modules/safer-cluster/versions.tf index 2f3f0a99c7..ccc58e11db 100644 --- a/modules/safer-cluster/versions.tf +++ b/modules/safer-cluster/versions.tf @@ -21,6 +21,6 @@ terraform { required_version = ">=0.13" provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:safer-cluster/v24.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:safer-cluster/v25.0.0" } } diff --git a/modules/services/versions.tf b/modules/services/versions.tf index 2c152c3ddd..cae9ec8a0a 100644 --- a/modules/services/versions.tf +++ b/modules/services/versions.tf @@ -19,6 +19,6 @@ terraform { required_version = ">= 0.13.0" provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:services/v24.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:services/v25.0.0" } } diff --git a/modules/workload-identity/versions.tf b/modules/workload-identity/versions.tf index e2e2edfae9..9a9e57bf9e 100644 --- a/modules/workload-identity/versions.tf +++ b/modules/workload-identity/versions.tf @@ -30,6 +30,6 @@ terraform { } provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:workload-identity/v24.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:workload-identity/v25.0.0" } } diff --git a/outputs.tf b/outputs.tf index 7e3fb254c6..ca55e48b8e 100644 --- a/outputs.tf +++ b/outputs.tf @@ -148,6 +148,11 @@ output "release_channel" { value = var.release_channel } +output "gateway_api_channel" { + description = "The gateway api channel of this cluster." + value = var.gateway_api_channel +} + output "identity_namespace" { description = "Workload Identity pool" value = length(local.cluster_workload_identity_config) > 0 ? local.cluster_workload_identity_config[0].workload_pool : null diff --git a/test/fixtures/node_pool/example.tf b/test/fixtures/node_pool/example.tf index c29124b2b7..6e5002869d 100644 --- a/test/fixtures/node_pool/example.tf +++ b/test/fixtures/node_pool/example.tf @@ -35,6 +35,8 @@ module "example" { max_memory_gb = 30 min_memory_gb = 10 gpu_resources = [] + auto_repair = true + auto_upgrade = true } } diff --git a/test/fixtures/safer_cluster_iap_bastion/version.tf b/test/fixtures/safer_cluster_iap_bastion/version.tf new file mode 100644 index 0000000000..46c8c86f1d --- /dev/null +++ b/test/fixtures/safer_cluster_iap_bastion/version.tf @@ -0,0 +1,25 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.13.0" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.51.0" + } + } +} diff --git a/test/fixtures/simple_regional_with_gateway_api/example.tf b/test/fixtures/simple_regional_with_gateway_api/example.tf new file mode 100644 index 0000000000..189a65ba1e --- /dev/null +++ b/test/fixtures/simple_regional_with_gateway_api/example.tf @@ -0,0 +1,35 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + cluster_index = 1 +} + +module "example" { + source = "../../../examples/simple_regional_with_gateway_api" + + project_id = var.project_ids[local.cluster_index] + cluster_name_suffix = "-${random_string.suffix.result}" + region = var.region + network = google_compute_network.main.name + subnetwork = google_compute_subnetwork.main.name + ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name + ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name + compute_engine_service_account = var.compute_engine_service_accounts[local.cluster_index] + skip_provisioners = true + enable_binary_authorization = true + gateway_api_channel = "CHANNEL_STANDARD" +} diff --git a/test/fixtures/simple_regional_with_gateway_api/network.tf b/test/fixtures/simple_regional_with_gateway_api/network.tf new file mode 100644 index 0000000000..a7ca31698a --- /dev/null +++ b/test/fixtures/simple_regional_with_gateway_api/network.tf @@ -0,0 +1,48 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "random_string" "suffix" { + length = 4 + special = false + upper = false +} + +provider "google" { + project = var.project_ids[local.cluster_index] +} + +resource "google_compute_network" "main" { + name = "cft-gke-test-${random_string.suffix.result}" + auto_create_subnetworks = false +} + +resource "google_compute_subnetwork" "main" { + name = "cft-gke-test-${random_string.suffix.result}" + ip_cidr_range = "10.0.0.0/17" + region = var.region + network = google_compute_network.main.self_link + + secondary_ip_range { + range_name = "cft-gke-test-pods-${random_string.suffix.result}" + ip_cidr_range = "192.168.0.0/18" + } + + secondary_ip_range { + range_name = "cft-gke-test-services-${random_string.suffix.result}" + ip_cidr_range = "192.168.64.0/18" + } +} + diff --git a/test/fixtures/simple_regional_with_gateway_api/outputs.tf b/test/fixtures/simple_regional_with_gateway_api/outputs.tf new file mode 120000 index 0000000000..726bdc722f --- /dev/null +++ b/test/fixtures/simple_regional_with_gateway_api/outputs.tf @@ -0,0 +1 @@ +../shared/outputs.tf \ No newline at end of file diff --git a/test/fixtures/simple_regional_with_gateway_api/variables.tf b/test/fixtures/simple_regional_with_gateway_api/variables.tf new file mode 120000 index 0000000000..c113c00a3d --- /dev/null +++ b/test/fixtures/simple_regional_with_gateway_api/variables.tf @@ -0,0 +1 @@ +../shared/variables.tf \ No newline at end of file diff --git a/test/integration/disable_client_cert/testdata/TestDisableClientCert.json b/test/integration/disable_client_cert/testdata/TestDisableClientCert.json index 3128aac00a..9d58f326f8 100755 --- a/test/integration/disable_client_cert/testdata/TestDisableClientCert.json +++ b/test/integration/disable_client_cert/testdata/TestDisableClientCert.json @@ -5,6 +5,7 @@ "enabled": true }, "gcpFilestoreCsiDriverConfig": {}, + "gkeBackupAgentConfig": {}, "horizontalPodAutoscaling": {}, "httpLoadBalancing": {}, "kubernetesDashboard": { diff --git a/test/integration/go.mod b/test/integration/go.mod old mode 100755 new mode 100644 index c567082daf..c0b1c1d903 --- a/test/integration/go.mod +++ b/test/integration/go.mod @@ -1,11 +1,76 @@ -module github.com/terraform-google-modules/workspace/test/integration +module github.com/terraform-google-modules/terraform-google-kubernetes-engine/test/integration -go 1.16 +go 1.18 require ( - github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test v0.0.0-20220317050137-1c8897fbd42c - github.com/gruntwork-io/terratest v0.35.6 // indirect - github.com/stretchr/testify v1.8.0 - k8s.io/client-go v0.25.4 // indirect + github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test v0.4.1 + github.com/go-openapi/jsonreference v0.19.5 // indirect + github.com/go-openapi/swag v0.19.14 // indirect + github.com/google/go-cmp v0.5.8 // indirect + github.com/stretchr/testify v1.8.1 + golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd // indirect + golang.org/x/net v0.0.0-20220722155237-a158d28d115b // indirect + golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f // indirect + google.golang.org/protobuf v1.28.0 // indirect +) + +require ( + cloud.google.com/go v0.83.0 // indirect + cloud.google.com/go/storage v1.10.0 // indirect + github.com/PuerkitoBio/purell v1.1.1 // indirect + github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect + github.com/agext/levenshtein v1.2.3 // indirect + github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect + github.com/aws/aws-sdk-go v1.40.56 // indirect + github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/go-errors/errors v1.0.2-0.20180813162953-d98b870cc4e0 // indirect + github.com/go-openapi/jsonpointer v0.19.5 // indirect + github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/snappy v0.0.3 // indirect + github.com/google/gnostic v0.5.7-v3refs // indirect + github.com/googleapis/gax-go/v2 v2.0.5 // indirect + github.com/gruntwork-io/terratest v0.40.7 // indirect + github.com/hashicorp/errwrap v1.0.0 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-getter v1.6.1 // indirect + github.com/hashicorp/go-multierror v1.1.0 // indirect + github.com/hashicorp/go-safetemp v1.0.0 // indirect + github.com/hashicorp/go-version v1.3.0 // indirect + github.com/hashicorp/hcl/v2 v2.9.1 // indirect + github.com/hashicorp/terraform-json v0.13.0 // indirect + github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/jstemmer/go-junit-report v0.9.1 // indirect + github.com/klauspost/compress v1.13.0 // indirect + github.com/mailru/easyjson v0.7.6 // indirect + github.com/mattn/go-zglob v0.0.2-0.20190814121620-e3c945676326 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/mitchellh/go-testing-interface v1.14.2-0.20210217184823-a52172cd2f64 // indirect + github.com/mitchellh/go-wordwrap v1.0.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/tidwall/gjson v1.12.1 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.0 // indirect + github.com/tidwall/sjson v1.2.4 // indirect + github.com/tmccombs/hcl2json v0.3.3 // indirect + github.com/ulikunitz/xz v0.5.8 // indirect + github.com/zclconf/go-cty v1.9.1 // indirect + go.opencensus.io v0.23.0 // indirect + golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect + golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect + golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c // indirect + golang.org/x/text v0.3.7 // indirect + golang.org/x/tools v0.1.10 // indirect + golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect + google.golang.org/api v0.47.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c // indirect + google.golang.org/grpc v1.38.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 // indirect + sigs.k8s.io/kustomize/kyaml v0.13.6 // indirect ) diff --git a/test/integration/go.sum b/test/integration/go.sum old mode 100755 new mode 100644 index b1f5f10cf8..eaf355ab24 --- a/test/integration/go.sum +++ b/test/integration/go.sum @@ -1,3 +1,4 @@ +bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= @@ -6,7 +7,6 @@ cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxK cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.51.0/go.mod h1:hWtGJ6gnXH+KgDv+V0zFGDvpi07n3z8ZNj3T1RW0Gcw= cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= @@ -19,13 +19,8 @@ cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmW cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= +cloud.google.com/go v0.83.0 h1:bAMqZidYkmIsUqe6PtkEPT7Q+vfizScn+jfNA6jwK9c= cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= -cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= -cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= -cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= -cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= -cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= -cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -34,6 +29,7 @@ cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4g cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -42,157 +38,274 @@ cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiy cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0 h1:STgFzyU5/8miMl0//zKh2aQeTyeaUH3WN9bSUiJ09bA= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Azure/azure-sdk-for-go v35.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-sdk-for-go v38.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-sdk-for-go v46.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go v50.2.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= +github.com/Azure/go-autorest v10.8.1+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= -github.com/Azure/go-autorest/autorest v0.9.3/go.mod h1:GsRuLYvwzLjjjRoWEIyMUaYq8GNUx2nRB378IPt/1p0= -github.com/Azure/go-autorest/autorest v0.9.6/go.mod h1:/FALq9T/kS7b5J5qsQ+RSTUdAmGFqi0vUdVNNx8q630= -github.com/Azure/go-autorest/autorest v0.11.0/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw= -github.com/Azure/go-autorest/autorest v0.11.5/go.mod h1:foo3aIXRQ90zFve3r0QiDsrjGDUwWhKl0ZOQy1CT14k= -github.com/Azure/go-autorest/autorest v0.11.27/go.mod h1:7l8ybrIdUmGqZMTD0sRtAr8NvbHjfofbf8RSP2q7w7U= -github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= -github.com/Azure/go-autorest/autorest/adal v0.8.0/go.mod h1:Z6vX6WXXuyieHAXwMj0S6HY6e6wcHn37qQMBQlvY3lc= -github.com/Azure/go-autorest/autorest/adal v0.8.1/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= -github.com/Azure/go-autorest/autorest/adal v0.8.2/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= +github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw= +github.com/Azure/go-autorest/autorest v0.11.17/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= +github.com/Azure/go-autorest/autorest v0.11.20/go.mod h1:o3tqFY+QR40VOlk+pV4d77mORO64jOXSgEnPQgLK6JY= github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg= -github.com/Azure/go-autorest/autorest/adal v0.9.2/go.mod h1:/3SMAM86bP6wC9Ev35peQDUeqFZBMH07vvUOmg4z/fE= -github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= -github.com/Azure/go-autorest/autorest/adal v0.9.20/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= -github.com/Azure/go-autorest/autorest/azure/auth v0.5.1/go.mod h1:ea90/jvmnAwDrSooLH4sRIehEPtG/EPUXavDh31MnA4= -github.com/Azure/go-autorest/autorest/azure/cli v0.4.0/go.mod h1:JljT387FplPzBA31vUcvsetLKF3pec5bdAxjVU4kI2s= -github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= -github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g= +github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= +github.com/Azure/go-autorest/autorest/adal v0.9.11/go.mod h1:nBKAnTomx8gDtl+3ZCJv2v0KACFHWTB2drffI1B68Pk= +github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= +github.com/Azure/go-autorest/autorest/azure/auth v0.5.8/go.mod h1:kxyKZTSfKh8OVFWPAgOgQ/frrJgeYQJPyR5fLFmXko4= +github.com/Azure/go-autorest/autorest/azure/cli v0.4.2/go.mod h1:7qkJkT+j6b+hIpzMOwPChJhTqS8VbsqqgULzMNRugoM= github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= -github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= -github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= -github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM= github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= -github.com/Azure/go-autorest/autorest/mocks v0.4.2/go.mod h1:Vy7OitM9Kei0i1Oj+LvyAWMXJHeKH1MVlzFugfVrmyU= -github.com/Azure/go-autorest/autorest/to v0.2.0/go.mod h1:GunWKJp1AEqgMaGLV+iocmRAJWqST1wQYhyyjXJ3SJc= -github.com/Azure/go-autorest/autorest/to v0.3.0/go.mod h1:MgwOyqaIuKdG4TL/2ywSsIWKAfJfgHDo8ObuUk3t5sA= -github.com/Azure/go-autorest/autorest/validation v0.1.0/go.mod h1:Ha3z/SqBeaalWQvokg3NZAlQTalVMtOIAs1aGK7G6u8= -github.com/Azure/go-autorest/autorest/validation v0.3.0/go.mod h1:yhLgjC0Wda5DYXl6JAsWyUe4KVNffhoDhG0zVzUMo3E= -github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= +github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE= +github.com/Azure/go-autorest/autorest/validation v0.3.1/go.mod h1:yhLgjC0Wda5DYXl6JAsWyUe4KVNffhoDhG0zVzUMo3E= github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= -github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test v0.0.0-20220317050137-1c8897fbd42c h1:c3vW8V5iGiaP3367VXbuF7AMGAHC5eSzwW5o188shx0= -github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test v0.0.0-20220317050137-1c8897fbd42c/go.mod h1:cdqqfBObgqdv4oyKGBS8Lwan5XVisYV9RvZ7JuXTx44= -github.com/GoogleCloudPlatform/k8s-cloud-provider v0.0.0-20190822182118-27a4ced34534/go.mod h1:iroGtC8B3tQiqtds1l+mgk/BBOrxbqjH+eUfFQYRc14= -github.com/GoogleContainerTools/kpt-functions-sdk/go v0.0.0-20220301220754-6964a09d6cd2/go.mod h1:lJYiqfBOl6AOiefK9kmkhinbffIysu+nnclOBwKEPlQ= +github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test v0.4.1 h1:9ytRGPGZLxjPJGMa6T6y3aF5Wmk0IhN9Hk0AtG8qfx4= +github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test v0.4.1/go.mod h1:JUepalU0A81Ry+dRm/MtSd/YP7UEu5HfrRL2+mJbz1Q= +github.com/GoogleContainerTools/kpt-functions-sdk/go/api v0.0.0-20221109010843-1f7d0c07a381/go.mod h1:prNhhUAODrB2VqHVead9tB8nLU9ffY4e4jjBwLMNO1M= +github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= +github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw= +github.com/Microsoft/go-winio v0.4.16-0.20201130162521-d1ffc52c7331/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= +github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= +github.com/Microsoft/go-winio v0.4.17-0.20210211115548-6eac466e5fa3/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= +github.com/Microsoft/go-winio v0.4.17-0.20210324224401-5516f17a5958/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= +github.com/Microsoft/go-winio v0.4.17/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= +github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= +github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= +github.com/Microsoft/hcsshim v0.8.7-0.20190325164909-8abdbb8205e4/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= +github.com/Microsoft/hcsshim v0.8.7/go.mod h1:OHd7sQqRFrYd3RmSgbgji+ctCwkbq2wbEYNSzOYtcBQ= +github.com/Microsoft/hcsshim v0.8.9/go.mod h1:5692vkUqntj1idxauYlpoINNKeqCiG6Sg38RRsjT5y8= +github.com/Microsoft/hcsshim v0.8.14/go.mod h1:NtVKoYxQuTLx6gEq0L96c9Ju4JbRJ4nY2ow3VK6a9Lg= +github.com/Microsoft/hcsshim v0.8.15/go.mod h1:x38A4YbHbdxJtc0sF6oIz+RG0npwSCAvn69iY6URG00= +github.com/Microsoft/hcsshim v0.8.16/go.mod h1:o5/SZqmR7x9JNKsW3pu+nqHm0MF8vbA+VxGOoXdC600= +github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU= +github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/agext/levenshtein v1.2.1 h1:QmvMAjj2aEICytGiWzmxoE0x2KZvE0fvmqMOfy2tjT8= +github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= github.com/agext/levenshtein v1.2.1/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= +github.com/agext/levenshtein v1.2.3 h1:YB2fHEn0UJagG8T1rrWknE3ZQzWM06O8AMAatNn7lmo= +github.com/agext/levenshtein v1.2.3/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apparentlymart/go-dump v0.0.0-20180507223929-23540a00eaa3/go.mod h1:oL81AME2rN47vu18xqj1S1jPIPuN7afo62yKTNn3XMM= -github.com/apparentlymart/go-textseg v1.0.0 h1:rRmlIsPEEhUTIKQb7T++Nz/A5Q6C9IuX2wFoYVvnCs0= github.com/apparentlymart/go-textseg v1.0.0/go.mod h1:z96Txxhf3xSFMPmb5X/1W05FF/Nj9VFpLOpjS5yuumk= -github.com/apparentlymart/go-textseg/v12 v12.0.0 h1:bNEQyAGak9tojivJNkoqWErVCQbjdL7GzRt3F8NvfJ0= -github.com/apparentlymart/go-textseg/v12 v12.0.0/go.mod h1:S/4uRK2UtaQttw1GenVJEynmyUenKwP++x/+DdGV/Ec= +github.com/apparentlymart/go-textseg/v13 v13.0.0 h1:Y+KvPE1NYz0xl601PVImeQfFyEy6iT90AvPUL1NNfNw= +github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= -github.com/aws/aws-sdk-go v1.16.26/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go v1.27.1/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go v1.38.28/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= +github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= +github.com/aws/aws-sdk-go v1.15.78/go.mod h1:E3/ieXAlvM0XWO57iftYVDLLvQ824smPP3ATZkfNZeM= +github.com/aws/aws-sdk-go v1.40.56 h1:FM2yjR0UUYFzDTMx+mH9Vyw1k1EUUxsAFzk+BjkzANA= +github.com/aws/aws-sdk-go v1.40.56/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= +github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d h1:xDfNPAt8lFiC1UJrqV3uuy861HCTo708pDMbjHHdCas= +github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d/go.mod h1:6QX/PXZ00z/TKoufEY6K/a0k6AhaJrQKdFe6OfVXsa4= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= +github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= +github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= +github.com/bshuster-repo/logrus-logstash-hook v0.4.1/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk= +github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= +github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= +github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50= +github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw= +github.com/cheggaaa/pb v1.0.27/go.mod h1:pQciLPpbU0oxA0h+VJYYLxO+XeDQb5pZijXscXHm81s= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/cilium/ebpf v0.0.0-20200110133405-4032b1d8aae3/go.mod h1:MA5e5Lr8slmEg9bt0VpxxWqJlO4iwu3FBdHUzV7wQVg= +github.com/cilium/ebpf v0.0.0-20200702112145-1c8d4c9ef775/go.mod h1:7cR51M8ViRLIdUjrmSXlK9pkrsDlLHbO8jiB8X8JnOc= +github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs= +github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE= +github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU= +github.com/containerd/aufs v0.0.0-20210316121734-20793ff83c97/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU= +github.com/containerd/aufs v1.0.0/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU= +github.com/containerd/btrfs v0.0.0-20201111183144-404b9149801e/go.mod h1:jg2QkJcsabfHugurUvvPhS3E08Oxiuh5W/g1ybB4e0E= +github.com/containerd/btrfs v0.0.0-20210316141732-918d888fb676/go.mod h1:zMcX3qkXTAi9GI50+0HOeuV8LU2ryCE/V2vG/ZBiTss= +github.com/containerd/btrfs v1.0.0/go.mod h1:zMcX3qkXTAi9GI50+0HOeuV8LU2ryCE/V2vG/ZBiTss= +github.com/containerd/cgroups v0.0.0-20190717030353-c4b9ac5c7601/go.mod h1:X9rLEHIqSf/wfK8NsPqxJmeZgW4pcfzdXITDrUSJ6uI= +github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko= +github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59/go.mod h1:pA0z1pT8KYB3TCXK/ocprsh7MAkoW8bZVzPdih9snmM= +github.com/containerd/cgroups v0.0.0-20200710171044-318312a37340/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo= +github.com/containerd/cgroups v0.0.0-20200824123100-0b889c03f102/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo= +github.com/containerd/cgroups v0.0.0-20210114181951-8a68de567b68/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= +github.com/containerd/cgroups v1.0.1/go.mod h1:0SJrPIenamHDcZhEcJMNBB85rHcUsw4f25ZfBiPYRkU= +github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= +github.com/containerd/console v0.0.0-20181022165439-0650fd9eeb50/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= +github.com/containerd/console v0.0.0-20191206165004-02ecf6a7291e/go.mod h1:8Pf4gM6VEbTNRIT26AyyU7hxdQU3MvAvxVI0sc00XBE= +github.com/containerd/console v1.0.1/go.mod h1:XUsP6YE/mKtz6bxc+I8UiKKTP04qjQL4qcS3XoQ5xkw= +github.com/containerd/console v1.0.2/go.mod h1:ytZPjGgY2oeTkAONYafi2kSj0aYggsf8acV1PGKCbzQ= +github.com/containerd/containerd v1.2.10/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/containerd v1.3.0/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.3.1-0.20191213020239-082f7e3aed57/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.4.0-beta.2.0.20200729163537-40b22ef07410/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.4.1/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.4.3/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.5.0-beta.1/go.mod h1:5HfvG1V2FsKesEGQ17k5/T7V960Tmcumvqn8Mc+pCYQ= +github.com/containerd/containerd v1.5.0-beta.3/go.mod h1:/wr9AVtEM7x9c+n0+stptlo/uBBoBORwEx6ardVcmKU= +github.com/containerd/containerd v1.5.0-beta.4/go.mod h1:GmdgZd2zA2GYIBZ0w09ZvgqEq8EfBp/m3lcVZIvPHhI= +github.com/containerd/containerd v1.5.0-rc.0/go.mod h1:V/IXoMqNGgBlabz3tHD2TWDoTJseu1FGOKuoA4nNb2s= +github.com/containerd/containerd v1.5.2/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTVn7dJnIOwtYR4g= +github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= +github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= +github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= +github.com/containerd/continuity v0.0.0-20200710164510-efbc4488d8fe/go.mod h1:cECdGN1O8G9bgKTlLhuPJimka6Xb/Gg7vYzCTNVxhvo= +github.com/containerd/continuity v0.0.0-20201208142359-180525291bb7/go.mod h1:kR3BEg7bDFaEddKm54WSmrol1fKWDU1nKYkgrcgZT7Y= +github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e/go.mod h1:EXlVlkqNba9rJe3j7w3Xa924itAMLgZH4UD/Q4PExuQ= +github.com/containerd/continuity v0.1.0/go.mod h1:ICJu0PwR54nI0yPEnJ6jcS+J7CZAUXrLh8lPo2knzsM= +github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= +github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= +github.com/containerd/fifo v0.0.0-20200410184934-f15a3290365b/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0= +github.com/containerd/fifo v0.0.0-20201026212402-0724c46b320c/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0= +github.com/containerd/fifo v0.0.0-20210316144830-115abcc95a1d/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4= +github.com/containerd/fifo v1.0.0/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4= +github.com/containerd/go-cni v1.0.1/go.mod h1:+vUpYxKvAF72G9i1WoDOiPGRtQpqsNW/ZHtSlv++smU= +github.com/containerd/go-cni v1.0.2/go.mod h1:nrNABBHzu0ZwCug9Ije8hL2xBCYh/pjfMb1aZGrrohk= +github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= +github.com/containerd/go-runc v0.0.0-20190911050354-e029b79d8cda/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= +github.com/containerd/go-runc v0.0.0-20200220073739-7016d3ce2328/go.mod h1:PpyHrqVs8FTi9vpyHwPwiNEGaACDxT/N/pLcvMSRA9g= +github.com/containerd/go-runc v0.0.0-20201020171139-16b287bc67d0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok= +github.com/containerd/go-runc v1.0.0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok= +github.com/containerd/imgcrypt v1.0.1/go.mod h1:mdd8cEPW7TPgNG4FpuP3sGBiQ7Yi/zak9TYCG3juvb0= +github.com/containerd/imgcrypt v1.0.4-0.20210301171431-0ae5c75f59ba/go.mod h1:6TNsg0ctmizkrOgXRNQjAPFWpMYRWuiB6dSF4Pfa5SA= +github.com/containerd/imgcrypt v1.1.1-0.20210312161619-7ed62a527887/go.mod h1:5AZJNI6sLHJljKuI9IHnw1pWqo/F0nGDOuR9zgTs7ow= +github.com/containerd/imgcrypt v1.1.1/go.mod h1:xpLnwiQmEUJPvQoAapeb2SNCxz7Xr6PJrXQb0Dpc4ms= +github.com/containerd/nri v0.0.0-20201007170849-eb1350a75164/go.mod h1:+2wGSDGFYfE5+So4M5syatU0N0f0LbWpuqyMi4/BE8c= +github.com/containerd/nri v0.0.0-20210316161719-dbaa18c31c14/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= +github.com/containerd/nri v0.1.0/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= +github.com/containerd/stargz-snapshotter/estargz v0.7.0/go.mod h1:83VWDqHnurTKliEB0YvWMiCfLDwv4Cjj1X9Vk98GJZw= +github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= +github.com/containerd/ttrpc v0.0.0-20190828172938-92c8520ef9f8/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= +github.com/containerd/ttrpc v0.0.0-20191028202541-4f1b8fe65a5c/go.mod h1:LPm1u0xBw8r8NOKoOdNMeVHSawSsltak+Ihv+etqsE8= +github.com/containerd/ttrpc v1.0.1/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y= +github.com/containerd/ttrpc v1.0.2/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y= +github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc= +github.com/containerd/typeurl v0.0.0-20190911142611-5eb25027c9fd/go.mod h1:GeKYzf2pQcqv7tJ0AoCuuhtnqhva5LNU3U+OyKxxJpk= +github.com/containerd/typeurl v1.0.1/go.mod h1:TB1hUtrpaiO88KEK56ijojHS1+NeF0izUACaJW2mdXg= +github.com/containerd/typeurl v1.0.2/go.mod h1:9trJWW2sRlGub4wZJRTW83VtbOLS6hwcDZXTn6oPz9s= +github.com/containerd/zfs v0.0.0-20200918131355-0a33824f23a2/go.mod h1:8IgZOBdv8fAgXddBT4dBXJPtxyRsejFIpXoklgxgEjw= +github.com/containerd/zfs v0.0.0-20210301145711-11e8f1707f62/go.mod h1:A9zfAbMlQwE+/is6hi0Xw8ktpL+6glmqZYtevJgaB8Y= +github.com/containerd/zfs v0.0.0-20210315114300-dde8f0fda960/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY= +github.com/containerd/zfs v0.0.0-20210324211415-d5c4544f0433/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY= +github.com/containerd/zfs v1.0.0/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY= +github.com/containernetworking/cni v0.7.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= +github.com/containernetworking/cni v0.8.0/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= +github.com/containernetworking/cni v0.8.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= +github.com/containernetworking/plugins v0.8.6/go.mod h1:qnw5mN19D8fIwkqW7oHHYDHVlzhJpcY6TQxn/fUyDDM= +github.com/containernetworking/plugins v0.9.1/go.mod h1:xP/idU2ldlzN6m4p5LmGiwRDjeJr6FLK6vuiUwoH7P8= +github.com/containers/ocicrypt v1.0.1/go.mod h1:MeJDzk1RJHv89LjsH0Sp5KTY3ZYkjXO/C+bKAeWFIrc= +github.com/containers/ocicrypt v1.1.0/go.mod h1:b8AOe0YR67uU8OqfVNcznfFpAzu3rdgUV4GP9qXPfu4= +github.com/containers/ocicrypt v1.1.1/go.mod h1:Dm55fwWm1YZAjYRaJ94z2mfZikIyIN4B0oB3dj3jFxY= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= +github.com/coreos/go-iptables v0.5.0/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20161114122254-48702e0da86b/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= +github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4= +github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1SMSibvLzxjeJLnrYEVLULFNiHY9YfQ= +github.com/d2g/dhcp4client v1.0.0/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW34z5W5s= +github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5/go.mod h1:Eo87+Kg/IX2hfWJfwxMzLyuSZyxSoAug2nGa1G2QAi8= +github.com/d2g/hardwareaddr v0.0.0-20190221164911-e7d9fbe030e4/go.mod h1:bMl4RjIciD2oAxI7DmWRx6gbeqrkoLqv3MV0vzNad+I= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0= +github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= +github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= -github.com/docker/cli v0.0.0-20191017083524-a8ff7f821017/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= -github.com/docker/cli v0.0.0-20200109221225-a4f60165b7a3/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/cli v20.10.7+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY= +github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v20.10.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= +github.com/docker/go-events v0.0.0-20170721190031-9461782956ad/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= +github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= +github.com/docker/go-metrics v0.0.0-20180209012529-399ea8c73916/go.mod h1:/u0gXw0Gay3ceNrsHubL3BtdOL2fHf93USgMTe0W5dI= +github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/elazarl/goproxy v0.0.0-20190911111923-ecfe977594f1/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao2r4iyvLdACqsl/Ljk= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful/v3 v3.8.0 h1:eCZ8ulSerjdAiaNpF7GxXIE7ZCMo1moN1qX+S609eVw= -github.com/emicklei/go-restful/v3 v3.8.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA= +github.com/garyburd/redigo v0.0.0-20150301180006-535138d7bcd7/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -202,52 +315,51 @@ github.com/go-errors/errors v1.0.2-0.20180813162953-d98b870cc4e0/go.mod h1:f4zRH github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= -github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= -github.com/go-openapi/jsonpointer v0.19.3 h1:gihV7YNZK1iK6Tgwwsxo2rJbD1GTbdm72325Bq8FI3w= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= -github.com/go-openapi/jsonreference v0.19.3 h1:5cxNfTy0UVC3X8JL5ymxzyoUZmo8iZb+jeTWn7tUa8o= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= github.com/go-openapi/jsonreference v0.19.5 h1:1WJP/wi4OjB4iV8KVbH73rQaoialJrqv8gitZLxGLtM= github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= -github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= -github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng= github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-test/deep v1.0.3/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= +github.com/go-test/deep v1.0.7 h1:/VSMRlnY/JSyqxQUzQLKVMAskpY/NZKFA5j2P+0pP2M= +github.com/go-test/deep v1.0.7/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8= +github.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= +github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= +github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= +github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gogo/googleapis v1.2.0/go.mod h1:Njal3psf3qN6dwBtQfUmBZh2ybovJ0tlu3o/AC7HYjU= +github.com/gogo/googleapis v1.4.0/go.mod h1:5YRNX2z1oM5gXdAkurHa942MDgEJyk02w4OecKY87+c= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= -github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= @@ -256,8 +368,6 @@ github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= -github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= -github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -277,10 +387,10 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54= github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -295,14 +405,17 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-containerregistry v0.0.0-20200110202235-f4fb41bf00a3/go.mod h1:2wIuQute9+hhWqvL3vEI7YB0EKluF4WcPzI1eAliazk= -github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= +github.com/google/go-containerregistry v0.6.0/go.mod h1:euCCtNbZ6tKqi1E72vwDj2xZcN5ttKpZLfa/wSo5iLw= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.2.1 h1:d8MncMlErDFTwQGBK1xhv026j9kqhvw1Qv9IbWT1VLQ= github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -315,22 +428,20 @@ github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= -github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= -github.com/googleapis/gnostic v0.2.2/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= -github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/handlers v0.0.0-20150720190736-60c7bfde3e33/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= +github.com/gorilla/mux v1.7.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= @@ -344,220 +455,355 @@ github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/gruntwork-io/go-commons v0.8.0/go.mod h1:gtp0yTtIBExIZp7vyIV9I0XQkVwiQZze678hvDXof78= -github.com/gruntwork-io/terratest v0.35.6 h1:Q7pUd3JI4i5mmR/KgYkZJJ4q9ZbV8ru9KydwjA/ohaA= -github.com/gruntwork-io/terratest v0.35.6/go.mod h1:GIVJGBV1WIv1vxIG31Ycy0CuHYfXuvvkilNQuC9Wi+o= +github.com/gruntwork-io/terratest v0.40.7 h1:kp6Ymc3hPMdsCoV2Ij2C5QqooCCwELuIopKXhWho/jE= +github.com/gruntwork-io/terratest v0.40.7/go.mod h1:CjHsEgP1Pe987X5N8K5qEqCuLtu1bqERGIAF8bTj1s0= +github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-getter v1.5.9/go.mod h1:BrrV/1clo8cCYu6mxvboYg+KutTiFnXjMEgDD8+i7ZI= +github.com/hashicorp/go-getter v1.6.1 h1:NASsgP4q6tL94WH6nJxKWj8As2H/2kop/bB1d8JMyRY= +github.com/hashicorp/go-getter v1.6.1/go.mod h1:IZCrswsZPeWv9IkVnLElzRU/gz/QPi6pZHn4tv6vbwA= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-multierror v1.1.0 h1:B9UzwGQJehnUY1yNrnwREHc3fGbC2xefo8g4TbElacI= github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= +github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-safetemp v1.0.0 h1:2HR189eFNrjHQyENnQMMpCiBAsRxzbTMIgBhEyExpmo= +github.com/hashicorp/go-safetemp v1.0.0/go.mod h1:oaerMy3BhqiTbVye6QuFhFtIceqFoDHxNAB65b+Rj1I= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.1.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/go-version v1.3.0 h1:McDWVJIU/y+u1BRV06dPaLfLCaT7fUTJLp5r04x7iNw= +github.com/hashicorp/go-version v1.3.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/hcl/v2 v2.8.2 h1:wmFle3D1vu0okesm8BTLVDyJ6/OL9DCLUwn0b2OptiY= -github.com/hashicorp/hcl/v2 v2.8.2/go.mod h1:bQTN5mpo+jewjJgh8jr0JUguIi7qPHUF6yIfAEN3jqY= -github.com/hashicorp/terraform-json v0.9.0 h1:WE7+Wt93W93feOiCligElSyS0tlDzwZUtJuDGIBr8zg= -github.com/hashicorp/terraform-json v0.9.0/go.mod h1:3defM4kkMfttwiE7VakJDwCd4R+umhSQnvJwORXbprE= +github.com/hashicorp/hcl/v2 v2.9.1 h1:eOy4gREY0/ZQHNItlfuEZqtcQbXIxzojlP301hDpnac= +github.com/hashicorp/hcl/v2 v2.9.1/go.mod h1:FwWsfWEjyV/CMj8s/gqAuiviY72rJ1/oayI9WftqcKg= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/hashicorp/terraform-json v0.13.0 h1:Li9L+lKD1FO5RVFRM1mMMIBDoUHslOniyEi5CM+FWGY= +github.com/hashicorp/terraform-json v0.13.0/go.mod h1:y5OdLBCT+rxbwnpxZs9kGL7R9ExU76+cpdY8zHwoazk= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.7/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA= github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a h1:zPPuIq2jAWWPTrGt70eK/BSch+gFAGrNzecsoENgu2o= github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a/go.mod h1:yL958EeXv8Ylng6IfnvG4oflryUi3vgA3xPs9hmII1s= -github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/joefitzgerald/rainbow-reporter v0.1.0/go.mod h1:481CNgqmVHQZzdIbN52CupLJyoVwB10FQ/IQlF1pdL8= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.11.2/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= +github.com/klauspost/compress v1.13.0 h1:2T7tUoQrQT+fQWdaY5rjWztFGAFwbGD04iPJg90ZiOs= +github.com/klauspost/compress v1.13.0/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= github.com/mattn/go-zglob v0.0.1/go.mod h1:9fxibJccNxU2cnpIKLRRFA7zX7qhkJIQWBb449FYHOo= +github.com/mattn/go-zglob v0.0.2-0.20190814121620-e3c945676326 h1:ofNAzWCcyTALn2Zv40+8XitdzCgXY6e9qvXwN9W0YXg= github.com/mattn/go-zglob v0.0.2-0.20190814121620-e3c945676326/go.mod h1:9fxibJccNxU2cnpIKLRRFA7zX7qhkJIQWBb449FYHOo= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88Jz2VyhSmden33/aXg4oVIY= +github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.31/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= +github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= +github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/go-testing-interface v1.14.2-0.20210217184823-a52172cd2f64 h1:+9bM6qWXndPx7+czi9+Jj6zHPioFpfdhwVGOYOgujMY= github.com/mitchellh/go-testing-interface v1.14.2-0.20210217184823-a52172cd2f64/go.mod h1:gfgS7OtZj6MA4U1UrDRp04twqAjfvlZyCfX3sDjEym8= github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= -github.com/mitchellh/go-wordwrap v1.0.0 h1:6GlHJ/LTGMrIJbwgdqdl2eEH8o+Exx/0m8ir9Gns0u4= -github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= +github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= +github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= +github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= +github.com/moby/sys/mountinfo v0.4.0/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= +github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= +github.com/moby/sys/symlink v0.1.0/go.mod h1:GGDODQmbFOjFsXvfLVn3+ZRxkch54RkSiGqsZeMYowQ= +github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= +github.com/onsi/ginkgo v0.0.0-20151202141238-7f8ab55aaf3b/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= -github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= -github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= -github.com/onsi/ginkgo/v2 v2.1.6/go.mod h1:MEH45j8TBi6u9BMogfbp0stKC5cdGjumZj5Y7AG4VIk= +github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= -github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= -github.com/onsi/gomega v1.20.1/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo= +github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= +github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/opencontainers/go-digest v1.0.0-rc1.0.20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.0.0/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/opencontainers/runc v1.0.0-rc8.0.20190926000215-3e425f80a8c9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/opencontainers/runc v1.0.0-rc9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/opencontainers/runc v1.0.0-rc93/go.mod h1:3NOsor4w32B2tC0Zbl8Knk4Wg84SM2ImC1fxBuqJ/H0= +github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.0.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.0.2-0.20190207185410-29686dbc5559/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.0.3-0.20200929063507-e6143ca7d51d/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs= +github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE= +github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo= github.com/oracle/oci-go-sdk v7.1.0+incompatible/go.mod h1:VQb79nF8Z2cwLkLS35ukwStZIg5F66tcBccjip/j888= github.com/otiai10/copy v1.6.0/go.mod h1:XWfuS3CrI0R6IE0FbgHsEazaXO8G0LpMp9o8tos0x4E= github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs= github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo= github.com/otiai10/mint v1.3.2/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= +github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= github.com/pquerna/otp v1.2.0/go.mod h1:dkJfzwRKNiegxyNb54X/3fLwhCynbMspSyWKnvi1AEg= +github.com/prometheus/client_golang v0.0.0-20180209125602-c332b6f63c06/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20180110214958-89604d197083/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.0-20190522114515-bc1a522cf7b1/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= +github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= +github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rubiojr/go-vhd v0.0.0-20160810183302-0bfd3b39853c/go.mod h1:DM5xW0nvfNNm2uytzsvhI3OnX8uzaRAg8UX/CnDqbto= -github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= -github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/sebdah/goldie v1.0.0/go.mod h1:jXP4hmWywNEwZzhMuv2ccnqTSFpuq8iyQhtQdkkZBH4= +github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= +github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= +github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= +github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= +github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980/go.mod h1:AO3tvPzVZ/ayst6UlUKUv6rcPQInYe3IknH3jYhAKu8= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= +github.com/stretchr/objx v0.0.0-20180129172003-8a3f7159479f/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v0.0.0-20180303142811-b89eecf5ca5d/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/tidwall/gjson v1.10.2 h1:APbLGOM0rrEkd8WBw9C24nllro4ajFuJu0Sc9hRz8Bo= -github.com/tidwall/gjson v1.10.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= +github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= +github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= +github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I= +github.com/tidwall/gjson v1.12.1 h1:ikuZsLdhr8Ws0IdROXUS1Gi4v9Z4pGqpX/CvJkxvfpo= +github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.4 h1:cuiLzLnaMeBhRmEv00Lpk3tkYrcxpmbU81tAY4Dw0tc= +github.com/tidwall/sjson v1.2.4/go.mod h1:098SZ494YoMWPmMO6ct4dcFnqxwj9r/gF0Etp19pSNM= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tmccombs/hcl2json v0.3.3 h1:+DLNYqpWE0CsOQiEZu+OZm5ZBImake3wtITYxQ8uLFQ= +github.com/tmccombs/hcl2json v0.3.3/go.mod h1:Y2chtz2x9bAeRTvSibVRVgbLJhLJXKlUeIvjeVdnm4w= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= -github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/ulikunitz/xz v0.5.8 h1:ERv8V6GKqVi23rgu5cj9pVfVzJbOqAY2Ntl88O6c2nQ= +github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/vdemeester/k8s-pkg-credentialprovider v0.0.0-20200107171650-7c61ffa44238/go.mod h1:JwQJCMWpUDqjZrB5jpw0f5VbN7U95zxFy1ZDpoEarGo= +github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= +github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= +github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= +github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI= +github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= +github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= github.com/vmihailenco/msgpack v3.3.3+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= -github.com/vmware/govmomi v0.20.3/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU= +github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4= +github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= +github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= +github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= @@ -566,49 +812,60 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs= +github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA= +github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg= github.com/zclconf/go-cty v1.2.0/go.mod h1:hOPWgoHbaTUnI5k4D2ld+GRpFJSCe6bCM7m1q/N4PQ8= -github.com/zclconf/go-cty v1.2.1 h1:vGMsygfmeCl4Xb6OA5U5XVAaQZ69FvoG7X2jUtQujb8= -github.com/zclconf/go-cty v1.2.1/go.mod h1:hOPWgoHbaTUnI5k4D2ld+GRpFJSCe6bCM7m1q/N4PQ8= +github.com/zclconf/go-cty v1.8.0/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk= +github.com/zclconf/go-cty v1.8.1/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk= +github.com/zclconf/go-cty v1.9.1 h1:viqrgQwFl5UpSxc046qblj78wZXVDFnSOufaOTER+cc= +github.com/zclconf/go-cty v1.9.1/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk= +github.com/zclconf/go-cty-debug v0.0.0-20191215020915-b22d67c1ba0b/go.mod h1:ZRKQfBXbGkpdV6QMzT3rU1kSTAnfu1dO8dPKjYprgj8= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= +go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= +go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= +go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= +go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= +go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= -go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5/go.mod h1:nmDLcffg48OtT/PSW0Hg7FvpRQsQh5OSqIylirxKC7o= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= +golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g= -golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd h1:XcWmESyNjXJMLahc3mqVQJcgSTDxFxhETVlfk9uGc38= golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= @@ -630,6 +887,7 @@ golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRu golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= @@ -642,14 +900,16 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 h1:kQgndtyPBW/JIYERgdxfwMYh3AVStj88WQTlNDi2a+o= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180811021610-c39426892332/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -660,6 +920,7 @@ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -677,26 +938,24 @@ golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= -golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420 h1:a8jGStKg0XqKDlKqjLrXn0ioF5MH36pT7Z0BRTqLhbk= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b h1:PxfKdU9lEEDYjdIzOtC4qFWgkU2rGHdKlKowJSMN9h0= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -710,11 +969,9 @@ golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c h1:pkQiBZBvdos9qq4wBAHqlzuZHEXo07pqV06ef90u1WI= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -726,15 +983,13 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -743,26 +998,38 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502175342-a43fa875dd82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190514135907-3a4b5fb9f71f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190522044717-8097e1b27ff5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190602015325-4c4f7f33c9ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190812073006-9eafafc0a87e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191002063906-3421d5a6bb1c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200120151820-655fe14d7479/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -771,43 +1038,41 @@ golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200817155316-9781c653f443/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200916030750-2334cc1a136f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200922070232-aee5d888a860/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201117170446-d9b008d0a637/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201202213521-69691e467435/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210603125802-9665404d3644 h1:CA1DEQ4NdKphKeL70tvsWNdT5oFh1lOjihRcEDROi0I= golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220517195934-5e4e11fc645e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f h1:v4INt8xihDGvnrfjMDVXGxw9wrfxYyCjk0KbXjhR55s= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -815,7 +1080,6 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= @@ -823,35 +1087,33 @@ golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190706070813-72ffa07ba3db/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191205215504-7b8c8591a921/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= @@ -877,29 +1139,23 @@ golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201110201400-7099162a900a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.10 h1:QjFRCZxdOhBJ/UNgnBZLbNV13DlbnK0quyivTnXJM20= golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0= -gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= -gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e/go.mod h1:kS+toOQn6AQKjmKJ7gzohV1XkqsFehRA2FbsbkopSuQ= +google.golang.org/api v0.0.0-20160322025152-9bf6e6e569ff/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.6.1-0.20190607001116-5213b8090861/go.mod h1:btoxGiFvQNVUZQ8W08zLtrVS08CNpINPEfxXxgJL1Q4= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -920,25 +1176,24 @@ google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34q google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8= +google.golang.org/api v0.47.0 h1:sQLWZQvP6jPGIP4JGPkJu4zHswrv81iobiyszr3b/0I= google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= -google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= -google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= -google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= -google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= -google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= -google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/cloud v0.0.0-20151119220103-975617b05ea8/go.mod h1:0H1ncTHf11KCFhTc/+EFRbzSCOZx+VUbRMk55Yv5MYk= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190522204451-c2c4e71fbf69/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= @@ -947,6 +1202,7 @@ google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvx google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200117163144-32f20d992d24/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= @@ -967,6 +1223,7 @@ google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= @@ -976,20 +1233,9 @@ google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c h1:wtujag7C+4D6KMoulW9YauvK2lgdvCMS260jsqqBXr0= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= -google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= -google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= -google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= -google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= -google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= -google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1014,10 +1260,8 @@ google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.38.0 h1:/9BgsAsa5nWe26HqOlvlgJnqBuktYOLCgjCPqsa56W0= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= -google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= -google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -1034,37 +1278,45 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20141024133853-64131543e789/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= +gopkg.in/cheggaaa/pb.v1 v1.0.27/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/gcfg.v1 v1.2.0/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= +gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/warnings.v0 v0.1.1/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= +gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -1072,59 +1324,47 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.17.0/go.mod h1:npsyOePkeP0CPwyGfXDHxvypiYMJxBWAMpQxCaJ4ZxI= -k8s.io/api v0.19.3/go.mod h1:VF+5FT1B74Pw3KxMdKyinLo+zynBaMBiAfGMuldcNDs= -k8s.io/api v0.25.4/go.mod h1:IG2+RzyPQLllQxnhzD8KQNEu4c4YvyDTpSMztf4A0OQ= -k8s.io/apimachinery v0.17.0/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= -k8s.io/apimachinery v0.19.3/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA= -k8s.io/apimachinery v0.25.4/go.mod h1:jaF9C/iPNM1FuLl7Zuy5b9v+n35HGSh6AQ4HYRkCqwo= -k8s.io/apiserver v0.17.0/go.mod h1:ABM+9x/prjINN6iiffRVNCBR2Wk7uY4z+EtEGZD48cg= -k8s.io/client-go v0.17.0/go.mod h1:TYgR6EUHs6k45hb6KWjVD6jFZvJV4gHDikv/It0xz+k= -k8s.io/client-go v0.19.3/go.mod h1:+eEMktZM+MG0KO+PTkci8xnbCZHvj9TqR6Q1XDUIJOM= -k8s.io/client-go v0.25.4 h1:3RNRDffAkNU56M/a7gUfXaEzdhZlYhoW8dgViGy5fn8= -k8s.io/client-go v0.25.4/go.mod h1:8trHCAC83XKY0wsBIpbirZU4NTUpbuhc2JnI7OruGZw= -k8s.io/cloud-provider v0.17.0/go.mod h1:Ze4c3w2C0bRsjkBUoHpFi+qWe3ob1wI2/7cUn+YQIDE= -k8s.io/code-generator v0.0.0-20191121015212-c4c8f8345c7e/go.mod h1:DVmfPQgxQENqDIzVR2ddLXMH34qeszkKSdH/N+s+38s= -k8s.io/component-base v0.17.0/go.mod h1:rKuRAokNMY2nn2A6LP/MiwpoaMRHpfRnrPaUJJj1Yoc= -k8s.io/csi-translation-lib v0.17.0/go.mod h1:HEF7MEz7pOLJCnxabi45IPkhSsE/KmxPQksuCrHKWls= -k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/gengo v0.0.0-20190822140433-26a664648505/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo= +k8s.io/api v0.20.4/go.mod h1:++lNL1AJMkDymriNniQsWRkMDzRaX2Y/POTUi8yvqYQ= +k8s.io/api v0.20.6/go.mod h1:X9e8Qag6JV/bL5G6bU8sdVRltWKmdHsFUGS3eVndqE8= +k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= +k8s.io/apimachinery v0.20.4/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= +k8s.io/apimachinery v0.20.6/go.mod h1:ejZXtW1Ra6V1O5H8xPBGz+T3+4gfkTCeExAHKU57MAc= +k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU= +k8s.io/apiserver v0.20.4/go.mod h1:Mc80thBKOyy7tbvFtB4kJv1kbdD0eIH8k8vianJcbFM= +k8s.io/apiserver v0.20.6/go.mod h1:QIJXNt6i6JB+0YQRNcS0hdRHJlMhflFmsBDeSgT1r8Q= +k8s.io/client-go v0.20.1/go.mod h1:/zcHdt1TeWSd5HoUe6elJmHSQ6uLLgp4bIJHVEuy+/Y= +k8s.io/client-go v0.20.4/go.mod h1:LiMv25ND1gLUdBeYxBIwKpkSC5IsozMMmOOeSJboP+k= +k8s.io/client-go v0.20.6/go.mod h1:nNQMnOvEUEsOzRRFIIkdmYOjAZrC8bgq0ExboWSU1I0= +k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk= +k8s.io/component-base v0.20.4/go.mod h1:t4p9EdiagbVCJKrQ1RsA5/V4rFQNDfRlevJajlGwgjI= +k8s.io/component-base v0.20.6/go.mod h1:6f1MPBAeI+mvuts3sIdtpjljHWBQ2cIy38oBIWMYnrM= +k8s.io/cri-api v0.17.3/go.mod h1:X1sbHmuXhwaHs9xxYffLqJogVsnI+f6cPRcgPel7ywM= +k8s.io/cri-api v0.20.1/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI= +k8s.io/cri-api v0.20.4/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI= +k8s.io/cri-api v0.20.6/go.mod h1:ew44AjNXwyn1s0U4xCKGodU7J1HzBeZ1MpGrpa5r8Yc= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= -k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= -k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= -k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= -k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.70.1 h1:7aaoSdahviPmR+XkS7FyxlkkXs6tHISSG03RxleQAVQ= -k8s.io/klog/v2 v2.70.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= -k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= -k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e h1:KLHHjkdQFomZy8+06csTWZ0m1343QqxZhR2LJ1OxCYM= +k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= +k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 h1:MQ8BAZPZlWk3S9K4a9NCkIFQtZShWqoha7snGixVgEA= k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1/go.mod h1:C/N6wCaBHeBHkHUesQOQy2/MZqGgMAFPqGsGQLdbZBU= -k8s.io/legacy-cloud-providers v0.17.0/go.mod h1:DdzaepJ3RtRy+e5YhNtrCYwlgyK87j/5+Yfp0L9Syp8= -k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= -k8s.io/utils v0.0.0-20200729134348-d5654de09c73/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= +k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw= -modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= -modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= -modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= -modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= -sigs.k8s.io/kustomize/kyaml v0.11.0 h1:9KhiCPKaVyuPcgOLJXkvytOvjMJLoxpjodiycb4gHsA= -sigs.k8s.io/kustomize/kyaml v0.11.0/go.mod h1:GNMwjim4Ypgp/MueD3zXHLRJEjz7RvtPae0AwlvEMFM= -sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= -sigs.k8s.io/structured-merge-diff v1.0.1-0.20191108220359-b1b620dd3f06/go.mod h1:/ULNhyfzRopfcjskuui0cTITekDduZ7ycKN3oUT9R18= -sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= +sigs.k8s.io/kustomize/kyaml v0.13.6 h1:eF+wsn4J7GOAXlvajv6OknSunxpcOBQQqsnPxObtkGs= +sigs.k8s.io/kustomize/kyaml v0.13.6/go.mod h1:yHP031rn1QX1lr/Xd934Ri/xdVNG8BE2ECa78Ht/kEg= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= +sigs.k8s.io/structured-merge-diff/v4 v4.0.3/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/test/integration/private_zonal_with_networking/testdata/TestPrivateZonalWithNetworking.json b/test/integration/private_zonal_with_networking/testdata/TestPrivateZonalWithNetworking.json index 84eec1d693..b2b0009907 100755 --- a/test/integration/private_zonal_with_networking/testdata/TestPrivateZonalWithNetworking.json +++ b/test/integration/private_zonal_with_networking/testdata/TestPrivateZonalWithNetworking.json @@ -5,6 +5,7 @@ "enabled": true }, "gcpFilestoreCsiDriverConfig": {}, + "gkeBackupAgentConfig": {}, "horizontalPodAutoscaling": {}, "httpLoadBalancing": {}, "kubernetesDashboard": { @@ -233,6 +234,7 @@ "gke-CLUSTER_NAME", "gke-CLUSTER_NAME-default-node-pool" ], + "windowsNodeConfig" : {}, "workloadMetadataConfig": { "mode": "GKE_METADATA" } diff --git a/test/integration/simple_autopilot_private_non_default_sa/simple_autopilot_private_non_default_sa_test.go b/test/integration/simple_autopilot_private_non_default_sa/simple_autopilot_private_non_default_sa_test.go new file mode 100644 index 0000000000..4692ddac24 --- /dev/null +++ b/test/integration/simple_autopilot_private_non_default_sa/simple_autopilot_private_non_default_sa_test.go @@ -0,0 +1,42 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package simple_autopilot_private_non_default_sa + +import ( + "testing" + + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft" + "github.com/stretchr/testify/assert" + "github.com/terraform-google-modules/terraform-google-kubernetes-engine/test/integration/utils" +) + +func TestSimpleAutopilotPrivateNonDefaultSA(t *testing.T) { + projectID := utils.GetTestProjectFromSetup(t, 1) + bpt := tft.NewTFBlueprintTest(t, tft.WithVars(map[string]interface{}{"project_id": projectID})) + + bpt.DefineVerify(func(assert *assert.Assertions) { + bpt.DefaultVerify(assert) + + location := bpt.GetStringOutput("location") + clusterName := bpt.GetStringOutput("cluster_name") + sa := bpt.GetStringOutput("service_account") + + op := gcloud.Runf(t, "container clusters describe %s --zone %s --project %s", clusterName, location, projectID) + assert.True(op.Get("autopilot.enabled").Bool(), "should be autopilot") + assert.Equal(sa, op.Get("autoscaling.autoprovisioningNodePoolDefaults.serviceAccount").String(), "should have custom SA") + }) + bpt.Test() +} diff --git a/test/integration/simple_regional_with_gateway_api/controls/gcloud.rb b/test/integration/simple_regional_with_gateway_api/controls/gcloud.rb new file mode 100644 index 0000000000..14722175bd --- /dev/null +++ b/test/integration/simple_regional_with_gateway_api/controls/gcloud.rb @@ -0,0 +1,199 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +project_id = attribute('project_id') +location = attribute('location') +cluster_name = attribute('cluster_name') + +control "gcloud" do + title "Google Compute Engine GKE configuration" + describe command("gcloud --project=#{project_id} container clusters --zone=#{location} describe #{cluster_name} --format=json") do + its(:exit_status) { should eq 0 } + its(:stderr) { should eq '' } + + let!(:data) do + if subject.exit_status == 0 + JSON.parse(subject.stdout) + else + {} + end + end + + describe "cluster" do + it "is running" do + expect(data['status']).to eq 'RUNNING' + end + + it "is regional" do + expect(data['location']).to match(/^.*[1-9]$/) + end + + it "uses public nodes and master endpoint" do + expect(data['privateClusterConfig']['enablePrivateEndpoint']).to eq nil + expect(data['privateClusterConfig']['enablePrivateNodes']).to eq nil + end + + it "has the expected addon settings" do + expect(data['addonsConfig']).to include( + "horizontalPodAutoscaling" => {}, + "httpLoadBalancing" => {}, + "kubernetesDashboard" => { + "disabled" => true, + }, + "networkPolicyConfig" => { + "disabled" => true, + }, + ) + end + + it "has gateway api enabled" do + expect(data['networkConfig']).to include( + "gatewayApiConfig" => { + "channel" => "CHANNEL_STANDARD", + }, + ) + end + + it "has the expected databaseEncryption config" do + expect(data['databaseEncryption']).to eq({ + "state" => 'DECRYPTED', + }) + end + + it "has the expected shieldedNodes config" do + expect(data['shieldedNodes']).to eq({ + "enabled" => true, + }) + end + + it "has the expected binaryAuthorization config" do + expect(data['binaryAuthorization']).to eq({ + "evaluationMode" => "PROJECT_SINGLETON_POLICY_ENFORCE", + }) + end + end + + describe "default node pool" do + let(:default_node_pool) { data['nodePools'].select { |p| p['name'] == "default-pool" }.first } + + it "exists" do + expect(data['nodePools']).to include( + including( + "name" => "default-pool", + ) + ) + end + end + + describe "node pool" do + let(:node_pools) { data['nodePools'].reject { |p| p['name'] == "default-pool" } } + + it "has autoscaling enabled" do + expect(node_pools).to include( + including( + "autoscaling" => including( + "enabled" => true, + ), + ) + ) + end + + it "has the expected minimum node count" do + expect(node_pools).to include( + including( + "autoscaling" => including( + "minNodeCount" => 1, + ), + ) + ) + end + + it "has the expected maximum node count" do + expect(node_pools).to include( + including( + "autoscaling" => including( + "maxNodeCount" => 100, + ), + ) + ) + end + + it "is the expected machine type" do + expect(node_pools).to include( + including( + "config" => including( + "machineType" => "e2-medium", + ), + ) + ) + end + + it "has the expected disk size" do + expect(node_pools).to include( + including( + "config" => including( + "diskSizeGb" => 100, + ), + ) + ) + end + + it "has the expected labels" do + expect(node_pools).to include( + including( + "config" => including( + "labels" => including( + "cluster_name" => cluster_name, + "node_pool" => "default-node-pool", + ), + ), + ) + ) + end + + it "has the expected network tags" do + expect(node_pools).to include( + including( + "config" => including( + "tags" => match_array([ + "gke-#{cluster_name}", + "gke-#{cluster_name}-default-node-pool", + ]), + ), + ) + ) + end + + it "has autorepair enabled" do + expect(node_pools).to include( + including( + "management" => including( + "autoRepair" => true, + ), + ) + ) + end + + it "has autoupgrade enabled" do + expect(node_pools).to include( + including( + "management" => including( + "autoUpgrade" => true, + ), + ) + ) + end + end + end +end diff --git a/test/integration/simple_regional_with_gateway_api/inspec.yml b/test/integration/simple_regional_with_gateway_api/inspec.yml new file mode 100644 index 0000000000..e91bbc6ca9 --- /dev/null +++ b/test/integration/simple_regional_with_gateway_api/inspec.yml @@ -0,0 +1,31 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: simple_regional_with_gateway_api +attributes: + - name: project_id + required: true + type: string + - name: location + required: true + type: string + - name: cluster_name + required: true + type: string + - name: kubernetes_endpoint + required: true + type: string + - name: client_token + required: true + type: string diff --git a/test/integration/simple_zonal/controls/acm.rb b/test/integration/simple_zonal/controls/acm.rb index 45663f176d..225f2bf963 100644 --- a/test/integration/simple_zonal/controls/acm.rb +++ b/test/integration/simple_zonal/controls/acm.rb @@ -1,4 +1,4 @@ -# Copyright 2019 Google LLC +# Copyright 2019-2023 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -48,5 +48,12 @@ expect(namespace).not_to be nil end end + + describe "gatekeeper-system namespace" do + let(:namespace) { client.get_namespace("gatekeeper-system") } + it "should exist" do + expect(namespace).not_to be nil + end + end end end diff --git a/test/integration/utils/utils.go b/test/integration/utils/utils.go new file mode 100644 index 0000000000..3c7097ed0b --- /dev/null +++ b/test/integration/utils/utils.go @@ -0,0 +1,30 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "testing" + + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft" +) + +func GetTestProjectFromSetup(t *testing.T, idx int) string { + setup := tft.NewTFBlueprintTest(t) + projectIDs := setup.GetTFSetupOutputListVal("project_ids") + if len(projectIDs)-1 < idx { + t.Fatalf("project_ids has %d elements, index of %d is invalid", len(projectIDs), idx) + } + return projectIDs[idx] +} diff --git a/test/setup/versions.tf b/test/setup/versions.tf index e416115b03..7a57560994 100644 --- a/test/setup/versions.tf +++ b/test/setup/versions.tf @@ -23,7 +23,7 @@ terraform { } google-beta = { source = "hashicorp/google-beta" - version = "~> 4.3" + version = "~> 4.45" } random = { source = "hashicorp/random" diff --git a/variables.tf b/variables.tf index a691783262..764abfdb13 100644 --- a/variables.tf +++ b/variables.tf @@ -170,6 +170,16 @@ variable "node_pools_labels" { } } +variable "node_pools_resource_labels" { + type = map(map(string)) + description = "Map of maps containing resource labels by node-pool name" + + default = { + all = {} + default-node-pool = {} + } +} + variable "node_pools_metadata" { type = map(map(string)) description = "Map of maps containing node metadata by node-pool name" @@ -181,6 +191,17 @@ variable "node_pools_metadata" { } } +variable "node_pools_linux_node_configs_sysctls" { + type = map(map(string)) + description = "Map of maps containing linux node config sysctls by node-pool name" + + # Default is being set in variables_defaults.tf + default = { + all = {} + default-node-pool = {} + } +} + variable "enable_cost_allocation" { type = bool description = "Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery" @@ -212,6 +233,8 @@ variable "cluster_autoscaling" { min_memory_gb = number max_memory_gb = number gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) + auto_repair = bool + auto_upgrade = bool }) default = { enabled = false @@ -220,6 +243,8 @@ variable "cluster_autoscaling" { max_memory_gb = 0 min_memory_gb = 0 gpu_resources = [] + auto_repair = true + auto_upgrade = true } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } @@ -377,6 +402,12 @@ variable "release_channel" { default = null } +variable "gateway_api_channel" { + type = string + description = "The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`." + default = null +} + variable "add_cluster_firewall_rules" { type = bool description = "Create additional firewall rules" @@ -411,6 +442,20 @@ variable "shadow_firewall_rules_priority" { type = number description = "The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000." default = 999 + validation { + condition = var.shadow_firewall_rules_priority < 1000 + error_message = "The shadow firewall rule priority must be lower than auto-created one(1000)." + } +} + +variable "shadow_firewall_rules_log_config" { + type = object({ + metadata = string + }) + description = "The log_config for shadow firewall rules. You can set this variable to `null` to disable logging." + default = { + metadata = "INCLUDE_ALL_METADATA" + } } @@ -497,7 +542,7 @@ variable "node_metadata" { validation { condition = contains(["GKE_METADATA", "GCE_METADATA", "UNSPECIFIED", "GKE_METADATA_SERVER", "EXPOSE"], var.node_metadata) - error_message = "The node_metadata value must be one of GKE_METADATA, GCE_METADATA or UNSPECIFIED." + error_message = "The node_metadata value must be one of GKE_METADATA, GCE_METADATA, UNSPECIFIED, GKE_METADATA_SERVER or EXPOSE." } } @@ -519,6 +564,18 @@ variable "cluster_dns_domain" { default = "" } +variable "gce_pd_csi_driver" { + type = bool + description = "Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver." + default = true +} + +variable "gke_backup_agent_config" { + type = bool + description = "Whether Backup for GKE agent is enabled for this cluster." + default = false +} + variable "timeouts" { type = map(string) description = "Timeout for cluster operations." @@ -528,3 +585,27 @@ variable "timeouts" { error_message = "Only create, update, delete timeouts can be specified." } } + +variable "monitoring_enable_managed_prometheus" { + type = bool + description = "Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled." + default = false +} + +variable "monitoring_enabled_components" { + type = list(string) + description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration." + default = [] +} + +variable "logging_enabled_components" { + type = list(string) + description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." + default = [] +} + +variable "enable_kubernetes_alpha" { + type = bool + description = "Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days." + default = false +} diff --git a/variables_defaults.tf b/variables_defaults.tf index b570f5f850..e4f3004771 100644 --- a/variables_defaults.tf +++ b/variables_defaults.tf @@ -34,6 +34,20 @@ locals { var.node_pools_labels ) + node_pools_resource_labels = merge( + { all = {} }, + { default-node-pool = {} }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : {}] + ), + zipmap( + [for node_pool in var.windows_node_pools : node_pool["name"]], + [for node_pool in var.windows_node_pools : {}] + ), + var.node_pools_resource_labels + ) + node_pools_metadata = merge( { all = {} }, { default-node-pool = {} }, @@ -89,4 +103,14 @@ locals { ), var.node_pools_oauth_scopes ) + + node_pools_linux_node_configs_sysctls = merge( + { all = {} }, + { default-node-pool = {} }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : {}] + ), + var.node_pools_linux_node_configs_sysctls + ) } diff --git a/versions.tf b/versions.tf index e08a8f3fd5..1d828ffcbe 100644 --- a/versions.tf +++ b/versions.tf @@ -21,7 +21,7 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 4.36.0, < 5.0" + version = ">= 4.51.0, < 5.0" } kubernetes = { source = "hashicorp/kubernetes" @@ -29,6 +29,6 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine/v24.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine/v25.0.0" } }