diff --git a/README.md b/README.md index 9b42c20..9d71ba2 100644 --- a/README.md +++ b/README.md @@ -177,54 +177,64 @@ Available targets: | Name | Version | |------|---------| -| terraform | >= 0.13.0, < 0.14.0 | -| aws | ~> 3.0 | -| local | ~> 1.3 | -| template | ~> 2.0 | +| terraform | >= 0.13.0 | +| aws | >= 3.0 | +| local | >= 1.3 | +| random | >= 2.0 | +| template | >= 2.0 | ## Providers | Name | Version | |------|---------| -| aws | ~> 3.0 | +| aws | >= 3.0 | +| random | >= 2.0 | ## Inputs | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | additional\_tag\_map | Additional tags for appending to tags\_as\_list\_of\_maps. Not added to `tags`. | `map(string)` | `{}` | no | -| ami\_release\_version | AMI version of the EKS Node Group. Defaults to latest version for Kubernetes version | `string` | `null` | no | -| ami\_type | Type of Amazon Machine Image (AMI) associated with the EKS Node Group. Defaults to `AL2_x86_64`. Valid values: `AL2_x86_64`, `AL2_x86_64_GPU`. Terraform will only perform drift detection if a configuration value is provided | `string` | `"AL2_x86_64"` | no | +| after\_cluster\_joining\_userdata | Additional `bash` commands to execute on each worker node after joining the EKS cluster (after executing the `bootstrap.sh` script). For more info, see https://kubedex.com/90-days-of-aws-eks-in-production | `string` | `""` | no | +| ami\_image\_id | AMI to use. Ignored of `launch_template_id` is supplied. | `string` | `null` | no | +| ami\_release\_version | AMI version to use, e.g. "1.16.13-20200821" (no "v"). Defaults to latest version for Kubernetes version. | `string` | `null` | no | +| ami\_type | Type of Amazon Machine Image (AMI) associated with the EKS Node Group.
Defaults to `AL2_x86_64`. Valid values: `AL2_x86_64`, `AL2_x86_64_GPU`, and `AL2_ARM_64`. | `string` | `"AL2_x86_64"` | no | | attributes | Additional attributes (e.g. `1`) | `list(string)` | `[]` | no | -| before\_cluster\_joining\_userdata | Additional commands to execute on each worker node before joining the EKS cluster (before executing the `bootstrap.sh` script). For more info, see https://kubedex.com/90-days-of-aws-eks-in-production | `string` | `""` | no | +| before\_cluster\_joining\_userdata | Additional `bash` commands to execute on each worker node before joining the EKS cluster (before executing the `bootstrap.sh` script). For more info, see https://kubedex.com/90-days-of-aws-eks-in-production | `string` | `""` | no | +| bootstrap\_additional\_options | Additional options to bootstrap.sh. DO NOT include `--kubelet-additional-args`, use `kubelet_additional_args` var instead. | `string` | `""` | no | | cluster\_name | The name of the EKS cluster | `string` | n/a | yes | | context | Single object for setting entire context at once.
See description of individual variables for details.
Leave string and numeric variables as `null` to use default value.
Individual variable settings (non-null) override settings in context object,
except for attributes, tags, and additional\_tag\_map, which are merged. | 
object({
    enabled             = bool
    namespace           = string
    environment         = string
    stage               = string
    name                = string
    delimiter           = string
    attributes          = list(string)
    tags                = map(string)
    additional_tag_map  = map(string)
    regex_replace_chars = string
    label_order         = list(string)
    id_length_limit     = number
  })
| 
{
  "additional_tag_map": {},
  "attributes": [],
  "delimiter": null,
  "enabled": true,
  "environment": null,
  "id_length_limit": null,
  "label_order": [],
  "name": null,
  "namespace": null,
  "regex_replace_chars": null,
  "stage": null,
  "tags": {}
}
| no | +| create\_before\_destroy | Set true in order to create the new node group before destroying the old one.
If false, the old node group will be destroyed first, causing downtime.
Changing this setting will always cause node group to be replaced. | `bool` | `false` | no | | delimiter | Delimiter to be used between `namespace`, `environment`, `stage`, `name` and `attributes`.
Defaults to `-` (hyphen). Set to `""` to use no delimiter at all. | `string` | `null` | no | -| desired\_size | Desired number of worker nodes (external changes ignored) | `number` | n/a | yes | -| disk\_size | Disk size in GiB for worker nodes. Defaults to 20. Terraform will only perform drift detection if a configuration value is provided | `number` | `20` | no | +| desired\_size | Initial desired number of worker nodes (external changes ignored) | `number` | n/a | yes | +| disk\_size | Disk size in GiB for worker nodes. Defaults to 20. Ignored it `launch_template_id` is supplied.
Terraform will only perform drift detection if a configuration value is provided. | `number` | `20` | no | | ec2\_ssh\_key | SSH key name that should be used to access the worker nodes | `string` | `null` | no | -| enable\_cluster\_autoscaler | Whether to enable node group to scale the Auto Scaling Group | `bool` | `false` | no | +| enable\_cluster\_autoscaler | Set true to allow Kubernetes Cluster Auto Scaler to scale the node group | `bool` | `false` | no | | enabled | Set to false to prevent the module from creating any resources | `bool` | `null` | no | | environment | Environment, e.g. 'uw2', 'us-west-2', OR 'prod', 'staging', 'dev', 'UAT' | `string` | `null` | no | | existing\_workers\_role\_policy\_arns | List of existing policy ARNs that will be attached to the workers default role on creation | `list(string)` | `[]` | no | -| existing\_workers\_role\_policy\_arns\_count | Count of existing policy ARNs that will be attached to the workers default role on creation. Needed to prevent Terraform error `count can't be computed` | `number` | `0` | no | +| existing\_workers\_role\_policy\_arns\_count | Obsolete and ignored. Allowed for backward compatibility. | `number` | `0` | no | | id\_length\_limit | Limit `id` to this many characters.
Set to `0` for unlimited length.
Set to `null` for default, which is `0`.
Does not affect `id_full`. | `number` | `null` | no | | instance\_types | Set of instance types associated with the EKS Node Group. Defaults to ["t3.medium"]. Terraform will only perform drift detection if a configuration value is provided | `list(string)` | 
[
  "t3.medium"
]
| no | -| kubernetes\_labels | Key-value mapping of Kubernetes labels. Only labels that are applied with the EKS API are managed by this argument. Other Kubernetes labels applied to the EKS Node Group will not be managed | `map(string)` | `{}` | no | +| kubelet\_additional\_options | Additional flags to pass to kubelet.
DO NOT include `--node-labels` or `--node-taints`,
use `kubernetes_labels` and `kubernetes_taints` to specify those." | `string` | `""` | no | +| kubernetes\_labels | Key-value mapping of Kubernetes labels. Only labels that are applied with the EKS API are managed by this argument.
Other Kubernetes labels applied to the EKS Node Group will not be managed. | `map(string)` | `{}` | no | +| kubernetes\_taints | Key-value mapping of Kubernetes taints. | `map(string)` | `{}` | no | | kubernetes\_version | Kubernetes version. Defaults to EKS Cluster Kubernetes version. Terraform will only perform drift detection if a configuration value is provided | `string` | `null` | no | | label\_order | The naming order of the id output and Name tag.
Defaults to ["namespace", "environment", "stage", "name", "attributes"].
You can omit any of the 5 elements, but at least one must be present. | `list(string)` | `null` | no | -| launch\_template\_id | The ID of a custom launch template to use for the EKS node group. | `string` | `null` | no | -| launch\_template\_version | A specific version of the above specific launch template | `string` | `null` | no | +| launch\_template\_name | The name (not ID) of a custom launch template to use for the EKS node group. If provided, it must specify the AMI image id. | `string` | `null` | no | +| launch\_template\_version | The version of the specified launch template to use. Defaults to latest version. | `string` | `null` | no | | max\_size | Maximum number of worker nodes | `number` | n/a | yes | | min\_size | Minimum number of worker nodes | `number` | n/a | yes | | module\_depends\_on | Can be any value desired. Module will wait for this value to be computed before creating node group. | `any` | `null` | no | | name | Solution name, e.g. 'app' or 'jenkins' | `string` | `null` | no | | namespace | Namespace, which could be your organization name or abbreviation, e.g. 'eg' or 'cp' | `string` | `null` | no | | regex\_replace\_chars | Regex to replace chars with empty string in `namespace`, `environment`, `stage` and `name`.
If not set, `"/[^a-zA-Z0-9-]/"` is used to remove all characters other than hyphens, letters and digits. | `string` | `null` | no | +| resources\_to\_tag | List of auto-launched resource types to tag. Valid types are "instance", "volume", "elastic-gpu", "spot-instances-request". | `list(string)` | `[]` | no | | source\_security\_group\_ids | Set of EC2 Security Group IDs to allow SSH access (port 22) from on the worker nodes. If you specify `ec2_ssh_key`, but do not specify this configuration when you create an EKS Node Group, port 22 on the worker nodes is opened to the Internet (0.0.0.0/0) | `list(string)` | `[]` | no | | stage | Stage, e.g. 'prod', 'staging', 'dev', OR 'source', 'build', 'test', 'deploy', 'release' | `string` | `null` | no | | subnet\_ids | A list of subnet IDs to launch resources in | `list(string)` | n/a | yes | | tags | Additional tags (e.g. `map('BusinessUnit','XYZ')` | `map(string)` | `{}` | no | +| userdata\_override | Many features of this module rely on the `bootstrap.sh` provided with Amazon Linux, and this module
may generate "user data" that expects to find that script. If you want to use an AMI that is not
compatible with the Amazon Linux `bootstrap.sh` initialization, then use `userdata_override` to provide
your own (Base64 encoded) user data. Use "" to prevent any user data from being set.

Setting `userdata_override` disables `kubernetes_taints`, `kubelet_additional_options`,
`before_cluster_joining_userdata`, `after_cluster_joining_userdata`, and `bootstrap_additional_options`. | `string` | `null` | no | ## Outputs diff --git a/ami.tf b/ami.tf new file mode 100644 index 0000000..be0a347 --- /dev/null +++ b/ami.tf @@ -0,0 +1,43 @@ + +locals { + // "amazon-eks-gpu-node-", + arch_label_map = { + "AL2_x86_64" : "", + "AL2_x86_64_GPU" : "-gpu", + "AL2_ARM_64" : "-arm64", + } + + // Kubernetes version priority (first one to be set wins) + // 1. prefix of var.ami_release_version + // 2. var.kubernetes_version + // 3. data.eks_cluster.this.kubernetes_version + need_cluster_kubernetes_version = local.need_ami_id && length(compact([var.ami_release_version, var.kubernetes_version])) == 0 + + ami_kubernetes_version = local.need_ami_id ? (local.need_cluster_kubernetes_version ? data.aws_eks_cluster.this[0].version : + regex("^(\\d+\\.\\d+)", coalesce(var.ami_release_version, var.kubernetes_version))[0] + ) : "" + + // Note: the expression: + // length(compact([x])) > 0 + // is a clean way to evaluate `x` and return false if x is null or an empty string. + // All string functions return an error when an argument is null, so the alternative + // (x != null) && (x != "") + // length(compact([var.ami_release_version])) > 0 + // (var.ami_release_version != null) && (var.ami_release_version != "") + ami_version_regex = local.need_ami_id ? (length(compact([var.ami_release_version])) > 0 ? + replace(var.ami_release_version, "/^(\\d+\\.\\d+)\\.\\d+-(\\d+)$/", "$1-v$2") : + "${local.ami_kubernetes_version}-*" + ) : "" + + ami_regex = local.need_ami_id ? format("amazon-eks%s-node-%s", local.arch_label_map[var.ami_type], local.ami_version_regex) : "" +} + +data "aws_ami" "selected" { + count = local.enabled && local.need_ami_id ? 1 : 0 + + most_recent = true + name_regex = local.ami_regex + + owners = ["amazon"] +} + diff --git a/docs/terraform.md b/docs/terraform.md index 59f7845..2f11fb0 100644 --- a/docs/terraform.md +++ b/docs/terraform.md @@ -3,54 +3,64 @@ | Name | Version | |------|---------| -| terraform | >= 0.13.0, < 0.14.0 | -| aws | ~> 3.0 | -| local | ~> 1.3 | -| template | ~> 2.0 | +| terraform | >= 0.13.0 | +| aws | >= 3.0 | +| local | >= 1.3 | +| random | >= 2.0 | +| template | >= 2.0 | ## Providers | Name | Version | |------|---------| -| aws | ~> 3.0 | +| aws | >= 3.0 | +| random | >= 2.0 | ## Inputs | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | additional\_tag\_map | Additional tags for appending to tags\_as\_list\_of\_maps. Not added to `tags`. | `map(string)` | `{}` | no | -| ami\_release\_version | AMI version of the EKS Node Group. Defaults to latest version for Kubernetes version | `string` | `null` | no | -| ami\_type | Type of Amazon Machine Image (AMI) associated with the EKS Node Group. Defaults to `AL2_x86_64`. Valid values: `AL2_x86_64`, `AL2_x86_64_GPU`. Terraform will only perform drift detection if a configuration value is provided | `string` | `"AL2_x86_64"` | no | +| after\_cluster\_joining\_userdata | Additional `bash` commands to execute on each worker node after joining the EKS cluster (after executing the `bootstrap.sh` script). For more info, see https://kubedex.com/90-days-of-aws-eks-in-production | `string` | `""` | no | +| ami\_image\_id | AMI to use. Ignored of `launch_template_id` is supplied. | `string` | `null` | no | +| ami\_release\_version | AMI version to use, e.g. "1.16.13-20200821" (no "v"). Defaults to latest version for Kubernetes version. | `string` | `null` | no | +| ami\_type | Type of Amazon Machine Image (AMI) associated with the EKS Node Group.
Defaults to `AL2_x86_64`. Valid values: `AL2_x86_64`, `AL2_x86_64_GPU`, and `AL2_ARM_64`. | `string` | `"AL2_x86_64"` | no | | attributes | Additional attributes (e.g. `1`) | `list(string)` | `[]` | no | -| before\_cluster\_joining\_userdata | Additional commands to execute on each worker node before joining the EKS cluster (before executing the `bootstrap.sh` script). For more info, see https://kubedex.com/90-days-of-aws-eks-in-production | `string` | `""` | no | +| before\_cluster\_joining\_userdata | Additional `bash` commands to execute on each worker node before joining the EKS cluster (before executing the `bootstrap.sh` script). For more info, see https://kubedex.com/90-days-of-aws-eks-in-production | `string` | `""` | no | +| bootstrap\_additional\_options | Additional options to bootstrap.sh. DO NOT include `--kubelet-additional-args`, use `kubelet_additional_args` var instead. | `string` | `""` | no | | cluster\_name | The name of the EKS cluster | `string` | n/a | yes | | context | Single object for setting entire context at once.
See description of individual variables for details.
Leave string and numeric variables as `null` to use default value.
Individual variable settings (non-null) override settings in context object,
except for attributes, tags, and additional\_tag\_map, which are merged. | 
object({
    enabled             = bool
    namespace           = string
    environment         = string
    stage               = string
    name                = string
    delimiter           = string
    attributes          = list(string)
    tags                = map(string)
    additional_tag_map  = map(string)
    regex_replace_chars = string
    label_order         = list(string)
    id_length_limit     = number
  })
| 
{
  "additional_tag_map": {},
  "attributes": [],
  "delimiter": null,
  "enabled": true,
  "environment": null,
  "id_length_limit": null,
  "label_order": [],
  "name": null,
  "namespace": null,
  "regex_replace_chars": null,
  "stage": null,
  "tags": {}
}
| no | +| create\_before\_destroy | Set true in order to create the new node group before destroying the old one.
If false, the old node group will be destroyed first, causing downtime.
Changing this setting will always cause node group to be replaced. | `bool` | `false` | no | | delimiter | Delimiter to be used between `namespace`, `environment`, `stage`, `name` and `attributes`.
Defaults to `-` (hyphen). Set to `""` to use no delimiter at all. | `string` | `null` | no | -| desired\_size | Desired number of worker nodes (external changes ignored) | `number` | n/a | yes | -| disk\_size | Disk size in GiB for worker nodes. Defaults to 20. Terraform will only perform drift detection if a configuration value is provided | `number` | `20` | no | +| desired\_size | Initial desired number of worker nodes (external changes ignored) | `number` | n/a | yes | +| disk\_size | Disk size in GiB for worker nodes. Defaults to 20. Ignored it `launch_template_id` is supplied.
Terraform will only perform drift detection if a configuration value is provided. | `number` | `20` | no | | ec2\_ssh\_key | SSH key name that should be used to access the worker nodes | `string` | `null` | no | -| enable\_cluster\_autoscaler | Whether to enable node group to scale the Auto Scaling Group | `bool` | `false` | no | +| enable\_cluster\_autoscaler | Set true to allow Kubernetes Cluster Auto Scaler to scale the node group | `bool` | `false` | no | | enabled | Set to false to prevent the module from creating any resources | `bool` | `null` | no | | environment | Environment, e.g. 'uw2', 'us-west-2', OR 'prod', 'staging', 'dev', 'UAT' | `string` | `null` | no | | existing\_workers\_role\_policy\_arns | List of existing policy ARNs that will be attached to the workers default role on creation | `list(string)` | `[]` | no | -| existing\_workers\_role\_policy\_arns\_count | Count of existing policy ARNs that will be attached to the workers default role on creation. Needed to prevent Terraform error `count can't be computed` | `number` | `0` | no | +| existing\_workers\_role\_policy\_arns\_count | Obsolete and ignored. Allowed for backward compatibility. | `number` | `0` | no | | id\_length\_limit | Limit `id` to this many characters.
Set to `0` for unlimited length.
Set to `null` for default, which is `0`.
Does not affect `id_full`. | `number` | `null` | no | | instance\_types | Set of instance types associated with the EKS Node Group. Defaults to ["t3.medium"]. Terraform will only perform drift detection if a configuration value is provided | `list(string)` | 
[
  "t3.medium"
]
| no | -| kubernetes\_labels | Key-value mapping of Kubernetes labels. Only labels that are applied with the EKS API are managed by this argument. Other Kubernetes labels applied to the EKS Node Group will not be managed | `map(string)` | `{}` | no | +| kubelet\_additional\_options | Additional flags to pass to kubelet.
DO NOT include `--node-labels` or `--node-taints`,
use `kubernetes_labels` and `kubernetes_taints` to specify those." | `string` | `""` | no | +| kubernetes\_labels | Key-value mapping of Kubernetes labels. Only labels that are applied with the EKS API are managed by this argument.
Other Kubernetes labels applied to the EKS Node Group will not be managed. | `map(string)` | `{}` | no | +| kubernetes\_taints | Key-value mapping of Kubernetes taints. | `map(string)` | `{}` | no | | kubernetes\_version | Kubernetes version. Defaults to EKS Cluster Kubernetes version. Terraform will only perform drift detection if a configuration value is provided | `string` | `null` | no | | label\_order | The naming order of the id output and Name tag.
Defaults to ["namespace", "environment", "stage", "name", "attributes"].
You can omit any of the 5 elements, but at least one must be present. | `list(string)` | `null` | no | -| launch\_template\_id | The ID of a custom launch template to use for the EKS node group. | `string` | `null` | no | -| launch\_template\_version | A specific version of the above specific launch template | `string` | `null` | no | +| launch\_template\_name | The name (not ID) of a custom launch template to use for the EKS node group. If provided, it must specify the AMI image id. | `string` | `null` | no | +| launch\_template\_version | The version of the specified launch template to use. Defaults to latest version. | `string` | `null` | no | | max\_size | Maximum number of worker nodes | `number` | n/a | yes | | min\_size | Minimum number of worker nodes | `number` | n/a | yes | | module\_depends\_on | Can be any value desired. Module will wait for this value to be computed before creating node group. | `any` | `null` | no | | name | Solution name, e.g. 'app' or 'jenkins' | `string` | `null` | no | | namespace | Namespace, which could be your organization name or abbreviation, e.g. 'eg' or 'cp' | `string` | `null` | no | | regex\_replace\_chars | Regex to replace chars with empty string in `namespace`, `environment`, `stage` and `name`.
If not set, `"/[^a-zA-Z0-9-]/"` is used to remove all characters other than hyphens, letters and digits. | `string` | `null` | no | +| resources\_to\_tag | List of auto-launched resource types to tag. Valid types are "instance", "volume", "elastic-gpu", "spot-instances-request". | `list(string)` | `[]` | no | | source\_security\_group\_ids | Set of EC2 Security Group IDs to allow SSH access (port 22) from on the worker nodes. If you specify `ec2_ssh_key`, but do not specify this configuration when you create an EKS Node Group, port 22 on the worker nodes is opened to the Internet (0.0.0.0/0) | `list(string)` | `[]` | no | | stage | Stage, e.g. 'prod', 'staging', 'dev', OR 'source', 'build', 'test', 'deploy', 'release' | `string` | `null` | no | | subnet\_ids | A list of subnet IDs to launch resources in | `list(string)` | n/a | yes | | tags | Additional tags (e.g. `map('BusinessUnit','XYZ')` | `map(string)` | `{}` | no | +| userdata\_override | Many features of this module rely on the `bootstrap.sh` provided with Amazon Linux, and this module
may generate "user data" that expects to find that script. If you want to use an AMI that is not
compatible with the Amazon Linux `bootstrap.sh` initialization, then use `userdata_override` to provide
your own (Base64 encoded) user data. Use "" to prevent any user data from being set.

Setting `userdata_override` disables `kubernetes_taints`, `kubelet_additional_options`,
`before_cluster_joining_userdata`, `after_cluster_joining_userdata`, and `bootstrap_additional_options`. | `string` | `null` | no | ## Outputs diff --git a/examples/complete/fixtures.us-east-2.tfvars b/examples/complete/fixtures.us-east-2.tfvars index e2ca148..3233dce 100644 --- a/examples/complete/fixtures.us-east-2.tfvars +++ b/examples/complete/fixtures.us-east-2.tfvars @@ -10,7 +10,7 @@ stage = "test" name = "eks-node-group" -kubernetes_version = "1.15" +kubernetes_version = "1.17" oidc_provider_enabled = true @@ -30,4 +30,6 @@ disk_size = 20 kubernetes_labels = {} -before_cluster_joining_userdata = "echo foo" +before_cluster_joining_userdata = <<-EOT + printf "\n\n###\nExample output from before_cluster_joining_userdata\n###\n\n" + EOT diff --git a/examples/complete/versions.tf b/examples/complete/versions.tf index da21f48..761968f 100644 --- a/examples/complete/versions.tf +++ b/examples/complete/versions.tf @@ -1,11 +1,10 @@ terraform { - required_version = ">= 0.13.0, < 0.14.0" + required_version = ">= 0.13.0" required_providers { - aws = "~> 3.0" - template = "~> 2.0" - null = "~> 2.0" - local = "~> 1.3" - kubernetes = "~> 1.11" + aws = ">= 3.0" + template = ">= 2.0" + local = ">= 1.3" + random = ">= 2.0" } } diff --git a/main.tf b/main.tf index e12361e..ddd2796 100644 --- a/main.tf +++ b/main.tf @@ -1,32 +1,75 @@ locals { enabled = module.this.enabled - node_group_tags = merge( + # The heavy use of the ternary operator `? :` is because it is one of the few ways to avoid + # evaluating expressions. The unused expression is not evaluated and so it does not have to be valid. + # This allows us to refer to resources that are only conditionally created and avoid creating + # dependencies on them that would not be avoided by using expressions like `join("",expr)`. + # + # The expression + # length(compact([var.launch_template_version])) > 0 + # is a shorter way of accomplishing the same test as + # var.launch_template_version != null && var.launch_template_version != "" + # and as an idiom has the added benefit of being extensible: + # length(compact([x, y])) > 0 + # is the same as + # x != null && x != "" && y != null && y != "" + + configured_launch_template_name = var.launch_template_name == null ? "" : var.launch_template_name + configured_launch_template_version = length(local.configured_launch_template_name) > 0 && length(compact([var.launch_template_version])) > 0 ? var.launch_template_version : "" + + configured_ami_image_id = var.ami_image_id == null ? "" : var.ami_image_id + + # See https://aws.amazon.com/blogs/containers/introducing-launch-template-and-custom-ami-support-in-amazon-eks-managed-node-groups/ + features_require_ami = local.need_bootstrap + need_ami_id = local.features_require_ami && length(local.configured_ami_image_id) == 0 + + features_require_launch_template = length(var.resources_to_tag) > 0 || local.need_userdata || local.features_require_ami + generate_launch_template = local.features_require_launch_template && length(local.configured_launch_template_name) == 0 + use_launch_template = local.features_require_launch_template || length(local.configured_launch_template_name) > 0 + + launch_template_id = local.use_launch_template ? (length(local.configured_launch_template_name) > 0 ? data.aws_launch_template.this[0].id : aws_launch_template.default[0].id) : "" + launch_template_version = local.use_launch_template ? ( + length(local.configured_launch_template_version) > 0 ? local.configured_launch_template_version : + ( + length(local.configured_launch_template_name) > 0 ? data.aws_launch_template.this[0].latest_version : aws_launch_template.default[0].latest_version + ) + ) : "" + + launch_template_ami = length(local.configured_ami_image_id) == 0 ? (local.features_require_ami ? data.aws_ami.selected[0].image_id : "") : local.configured_ami_image_id + + autoscaler_enabled_tags = { + "k8s.io/cluster-autoscaler/${var.cluster_name}" = "owned" + "k8s.io/cluster-autoscaler/enabled" = "true" + } + autoscaler_kubernetes_label_tags = { + for label, value in var.kubernetes_labels : format("k8s.io/cluster-autoscaler/node-template/label/%v", label) => value + } + autoscaler_kubernetes_taints_tags = { + for label, value in var.kubernetes_taints : format("k8s.io/cluster-autoscaler/node-template/taint/%v", label) => value + } + autoscaler_tags = merge(local.autoscaler_enabled_tags, local.autoscaler_kubernetes_label_tags, local.autoscaler_kubernetes_taints_tags) + + node_tags = merge( module.label.tags, { "kubernetes.io/cluster/${var.cluster_name}" = "owned" - }, - { - "k8s.io/cluster-autoscaler/${var.cluster_name}" = "owned" - }, - { - "k8s.io/cluster-autoscaler/enabled" = "${var.enable_cluster_autoscaler}" } ) + node_group_tags = merge(local.node_tags, var.enable_cluster_autoscaler ? local.autoscaler_tags : {}) + aws_policy_prefix = format("arn:%s:iam::aws:policy", join("", data.aws_partition.current.*.partition)) - userdata_vars = { - before_cluster_joining_userdata = var.before_cluster_joining_userdata - } + get_cluster_data = local.enabled && (local.need_cluster_kubernetes_version || local.need_bootstrap) +} - # Use a custom launch_template if one was passed as an input - # Otherwise, use the default in this project - launch_template = { - id = coalesce(var.launch_template_id, aws_launch_template.default[0].id) - latest_version = coalesce(var.launch_template_version, aws_launch_template.default[0].latest_version) - } +data "aws_eks_cluster" "this" { + count = local.get_cluster_data ? 1 : 0 + name = var.cluster_name } + + module "label" { source = "git::https://github.com/cloudposse/terraform-null-label.git?ref=tags/0.19.2" @@ -117,14 +160,15 @@ resource "aws_iam_role_policy_attachment" "amazon_ec2_container_registry_read_on } resource "aws_iam_role_policy_attachment" "existing_policies_for_eks_workers_role" { - count = local.enabled ? var.existing_workers_role_policy_arns_count : 0 - policy_arn = var.existing_workers_role_policy_arns[count.index] + for_each = local.enabled ? toset(var.existing_workers_role_policy_arns) : [] + policy_arn = each.value role = join("", aws_iam_role.default.*.name) } resource "aws_launch_template" "default" { # We'll use this default if we aren't provided with a launch template during invocation - count = (local.enabled && (var.launch_template_id == null)) ? 1 : 0 + count = (local.enabled && local.generate_launch_template) ? 1 : 0 + block_device_mappings { device_name = "/dev/xvda" @@ -133,48 +177,139 @@ resource "aws_launch_template" "default" { } } + name_prefix = module.label.id + update_default_version = true + instance_type = var.instance_types[0] + image_id = local.launch_template_ami == "" ? null : local.launch_template_ami dynamic "tag_specifications" { - for_each = ["instance", "volume", "elastic-gpu"] + for_each = var.resources_to_tag content { resource_type = tag_specifications.value - tags = local.node_group_tags + tags = local.node_tags } } - user_data = base64encode(templatefile("${path.module}/userdata.tpl", local.userdata_vars)) + # See https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html + # and https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html + # Note in particular: + # If any containers that you deploy to the node group use the Instance Metadata Service Version 2, + # then make sure to set the Metadata response hop limit to 2 in your launch template. + metadata_options { + http_put_response_hop_limit = 2 + # Despite being documented as "Optional", `http_endpoint` is required when `http_put_response_hop_limit` is set. + # We set it to the default setting of "enabled". + http_endpoint = "enabled" + } + + user_data = local.userdata + tags = local.node_group_tags } +data "aws_launch_template" "this" { + count = local.enabled && length(local.configured_launch_template_name) > 0 ? 1 : 0 + + name = local.configured_launch_template_name +} + +resource "random_pet" "cbd" { + count = local.enabled && var.create_before_destroy ? 1 : 0 + + separator = module.label.delimiter + length = 1 + + keepers = { + ami_type = var.ami_type + disk_size = local.use_launch_template ? null : var.disk_size + instance_types = join(",", local.use_launch_template ? [] : var.instance_types) + node_role_arn = join("", aws_iam_role.default.*.arn) + + ec2_ssh_key = var.ec2_ssh_key == null ? "" : var.ec2_ssh_key + source_security_group_ids = join(",", var.source_security_group_ids) + subnet_ids = join(",", var.subnet_ids) + + launch_template_id = local.launch_template_id + } + + depends_on = [var.module_depends_on] +} + + +# Support keeping 2 node groups in sync by extracting common variable settings +locals { + ng = { + cluster_name = var.cluster_name + node_role_arn = join("", aws_iam_role.default.*.arn) + subnet_ids = var.subnet_ids + disk_size = local.use_launch_template ? null : var.disk_size + instance_types = local.use_launch_template ? null : var.instance_types + ami_type = local.launch_template_ami == "" ? var.ami_type : null + labels = var.kubernetes_labels + release_version = local.launch_template_ami == "" ? var.ami_release_version : null + version = length(compact([local.launch_template_ami, var.ami_release_version])) == 0 ? var.kubernetes_version : null + + tags = local.node_group_tags + + scaling_config = { + desired_size = var.desired_size + max_size = var.max_size + min_size = var.min_size + } + + ec2_ssh_key = var.ec2_ssh_key == null ? "" : var.ec2_ssh_key + source_security_group_ids = var.source_security_group_ids + } +} + +# Because create_before_destroy is such a dramatic change, we want to make it optional. +# Because lifecycle must be static, the only way to make it optional is to create +# two nearly identical resources and only enable the correct one. +# See https://github.com/hashicorp/terraform/issues/24188 +# +# WARNING TO MAINTAINERS: both node groups should be kept exactly in sync +# except for count, lifecycle, and node_group_name. resource "aws_eks_node_group" "default" { - count = local.enabled ? 1 : 0 - cluster_name = var.cluster_name + count = local.enabled && ! var.create_before_destroy ? 1 : 0 node_group_name = module.label.id - node_role_arn = join("", aws_iam_role.default.*.arn) - subnet_ids = var.subnet_ids - ami_type = var.ami_type - labels = var.kubernetes_labels - release_version = var.ami_release_version - version = var.kubernetes_version - tags = local.node_group_tags + lifecycle { + create_before_destroy = false + ignore_changes = [scaling_config[0].desired_size] + } + + # From here to end of resource should be identical in both node groups + cluster_name = local.ng.cluster_name + node_role_arn = local.ng.node_role_arn + subnet_ids = local.ng.subnet_ids + disk_size = local.ng.disk_size + instance_types = local.ng.instance_types + ami_type = local.ng.ami_type + labels = local.ng.labels + release_version = local.ng.release_version + version = local.ng.version + + tags = local.ng.tags scaling_config { - desired_size = var.desired_size - max_size = var.max_size - min_size = var.min_size + desired_size = local.ng.scaling_config.desired_size + max_size = local.ng.scaling_config.max_size + min_size = local.ng.scaling_config.min_size } - launch_template { - id = local.launch_template.id - version = local.launch_template.latest_version + dynamic "launch_template" { + for_each = local.use_launch_template ? ["true"] : [] + content { + id = local.launch_template_id + version = local.launch_template_version + } } dynamic "remote_access" { - for_each = var.ec2_ssh_key != null && var.ec2_ssh_key != "" ? ["true"] : [] + for_each = length(local.ng.ec2_ssh_key) > 0 ? ["true"] : [] content { - ec2_ssh_key = var.ec2_ssh_key - source_security_group_ids = var.source_security_group_ids + ec2_ssh_key = local.ng.ec2_ssh_key + source_security_group_ids = local.ng.source_security_group_ids } } @@ -185,13 +320,71 @@ resource "aws_eks_node_group" "default" { aws_iam_role_policy_attachment.amazon_eks_worker_node_autoscaler_policy, aws_iam_role_policy_attachment.amazon_eks_cni_policy, aws_iam_role_policy_attachment.amazon_ec2_container_registry_read_only, + aws_launch_template.default, # Also allow calling module to create an explicit dependency # This is useful in conjunction with terraform-aws-eks-cluster to ensure # the cluster is fully created and configured before creating any node groups var.module_depends_on ] +} + +# WARNING TO MAINTAINERS: both node groups should be kept exactly in sync +# except for count, lifecycle, and node_group_name. +resource "aws_eks_node_group" "cbd" { + count = local.enabled && var.create_before_destroy ? 1 : 0 + node_group_name = format("%v%v%v", module.label.id, module.label.delimiter, join("", random_pet.cbd.*.id)) lifecycle { - ignore_changes = [scaling_config[0].desired_size] + create_before_destroy = true + ignore_changes = [scaling_config[0].desired_size] + } + + # From here to end of resource should be identical in both node groups + cluster_name = local.ng.cluster_name + node_role_arn = local.ng.node_role_arn + subnet_ids = local.ng.subnet_ids + disk_size = local.ng.disk_size + instance_types = local.ng.instance_types + ami_type = local.ng.ami_type + labels = local.ng.labels + release_version = local.ng.release_version + version = local.ng.version + + tags = local.ng.tags + + scaling_config { + desired_size = local.ng.scaling_config.desired_size + max_size = local.ng.scaling_config.max_size + min_size = local.ng.scaling_config.min_size + } + + dynamic "launch_template" { + for_each = local.use_launch_template ? ["true"] : [] + content { + id = local.launch_template_id + version = local.launch_template_version + } + } + + dynamic "remote_access" { + for_each = length(local.ng.ec2_ssh_key) > 0 ? ["true"] : [] + content { + ec2_ssh_key = local.ng.ec2_ssh_key + source_security_group_ids = local.ng.source_security_group_ids + } } + + # Ensure that IAM Role permissions are created before and deleted after EKS Node Group handling. + # Otherwise, EKS will not be able to properly delete EC2 Instances and Elastic Network Interfaces. + depends_on = [ + aws_iam_role_policy_attachment.amazon_eks_worker_node_policy, + aws_iam_role_policy_attachment.amazon_eks_worker_node_autoscaler_policy, + aws_iam_role_policy_attachment.amazon_eks_cni_policy, + aws_iam_role_policy_attachment.amazon_ec2_container_registry_read_only, + aws_launch_template.default, + # Also allow calling module to create an explicit dependency + # This is useful in conjunction with terraform-aws-eks-cluster to ensure + # the cluster is fully created and configured before creating any node groups + var.module_depends_on + ] } diff --git a/outputs.tf b/outputs.tf index 25acb88..60389bd 100644 --- a/outputs.tf +++ b/outputs.tf @@ -10,20 +10,20 @@ output "eks_node_group_role_name" { output "eks_node_group_id" { description = "EKS Cluster name and EKS Node Group name separated by a colon" - value = join("", aws_eks_node_group.default.*.id) + value = join("", aws_eks_node_group.default.*.id, aws_eks_node_group.cbd.*.id) } output "eks_node_group_arn" { description = "Amazon Resource Name (ARN) of the EKS Node Group" - value = join("", aws_eks_node_group.default.*.arn) + value = join("", aws_eks_node_group.default.*.arn, aws_eks_node_group.cbd.*.arn) } output "eks_node_group_resources" { description = "List of objects containing information about underlying resources of the EKS Node Group" - value = local.enabled ? aws_eks_node_group.default.*.resources : [] + value = local.enabled ? (var.create_before_destroy ? aws_eks_node_group.cbd.*.resources : aws_eks_node_group.default.*.resources) : [] } output "eks_node_group_status" { description = "Status of the EKS Node Group" - value = join("", aws_eks_node_group.default.*.status) + value = join("", aws_eks_node_group.default.*.status, aws_eks_node_group.cbd.*.status) } diff --git a/userdata.tf b/userdata.tf new file mode 100644 index 0000000..ecffd0e --- /dev/null +++ b/userdata.tf @@ -0,0 +1,51 @@ +# The userdata is built from the `userdata.tpl` file. It is limited to ~16k bytes, +# so comments about the userdata (~1k bytes) are here, not in the tpl file. +# +# userdata for EKS worker nodes to configure Kubernetes applications on EC2 instances +# In multipart MIME format so EKS can append to it. See: +# https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html#launch-template-user-data +# https://www.w3.org/Protocols/rfc1341/7_2_Multipart.html +# If you just provide a #!/bin/bash script like you can do when you provide the entire userdata you get +# an error at deploy time: Ec2LaunchTemplateInvalidConfiguration: User data was not in the MIME multipart format +# +# See also: +# https://aws.amazon.com/premiumsupport/knowledge-center/execute-user-data-ec2/ +# https://docs.aws.amazon.com/eks/latest/userguide/launch-workers.html +# https://aws.amazon.com/blogs/opensource/improvements-eks-worker-node-provisioning/ +# https://github.com/awslabs/amazon-eks-ami/blob/master/files/bootstrap.sh +# + +locals { + kubelet_label_settings = [for k, v in var.kubernetes_labels : format("%v=%v", k, v)] + kubelet_taint_settings = [for k, v in var.kubernetes_taints : format("%v=%v", k, v)] + kubelet_label_args = (length(local.kubelet_label_settings) == 0 ? "" : + "--node-labels=${join(",", local.kubelet_label_settings)}" + ) + kubelet_taint_args = (length(local.kubelet_taint_settings) == 0 ? "" : + "--register-with-taints=${join(",", local.kubelet_taint_settings)}" + ) + + kubelet_extra_args = join(" ", compact([local.kubelet_label_args, local.kubelet_taint_args, var.kubelet_additional_options])) + + userdata_vars = { + before_cluster_joining_userdata = var.before_cluster_joining_userdata == null ? "" : var.before_cluster_joining_userdata + kubelet_extra_args = local.kubelet_extra_args + bootstrap_extra_args = var.bootstrap_additional_options == null ? "" : var.bootstrap_additional_options + after_cluster_joining_userdata = var.after_cluster_joining_userdata == null ? "" : var.after_cluster_joining_userdata + } + + cluster_data = { + cluster_endpoint = local.get_cluster_data ? data.aws_eks_cluster.this[0].endpoint : null + certificate_authority_data = local.get_cluster_data ? data.aws_eks_cluster.this[0].certificate_authority[0].data : null + cluster_name = local.get_cluster_data ? data.aws_eks_cluster.this[0].name : null + } + + need_bootstrap = length(compact([local.kubelet_taint_args, var.kubelet_additional_options, + local.userdata_vars.bootstrap_extra_args, + local.userdata_vars.after_cluster_joining_userdata] + )) > 0 + + need_userdata = (var.userdata_override == null) && (length(local.userdata_vars.before_cluster_joining_userdata) > 0) || local.need_bootstrap + + userdata = local.need_userdata ? base64encode(templatefile("${path.module}/userdata.tpl", merge(local.userdata_vars, local.cluster_data))) : var.userdata_override +} \ No newline at end of file diff --git a/userdata.tpl b/userdata.tpl index 0a6d7fe..181eabb 100644 --- a/userdata.tpl +++ b/userdata.tpl @@ -1,10 +1,22 @@ MIME-Version: 1.0 -Content-Type: multipart/mixed; boundary="//" +Content-Type: multipart/mixed; boundary="/:/+++" ---// +--/:/+++ Content-Type: text/x-shellscript; charset="us-ascii" #!/bin/bash +# In multipart MIME format to support EKS appending to it + ${before_cluster_joining_userdata} ---//-- +%{ if length(kubelet_extra_args) > 0 } +export KUBELET_EXTRA_ARGS="${kubelet_extra_args}" +%{ endif } +%{ if length(kubelet_extra_args) > 0 || length (bootstrap_extra_args) > 0 || length (after_cluster_joining_userdata) > 0 } + +/etc/eks/bootstrap.sh --apiserver-endpoint '${cluster_endpoint}' --b64-cluster-ca '${certificate_authority_data}' ${bootstrap_extra_args} '${cluster_name}' + +${after_cluster_joining_userdata} +%{ endif } + +--/:/+++-- diff --git a/variables.tf b/variables.tf index f3273cc..0e875e0 100644 --- a/variables.tf +++ b/variables.tf @@ -1,6 +1,6 @@ variable "enable_cluster_autoscaler" { type = bool - description = "Whether to enable node group to scale the Auto Scaling Group" + description = "Set true to allow Kubernetes Cluster Auto Scaler to scale the node group" default = false } @@ -9,6 +9,16 @@ variable "cluster_name" { description = "The name of the EKS cluster" } +variable "create_before_destroy" { + type = bool + default = false + description = <<-EOT + Set true in order to create the new node group before destroying the old one. + If false, the old node group will be destroyed first, causing downtime. + Changing this setting will always cause node group to be replaced. + EOT +} + variable "ec2_ssh_key" { type = string description = "SSH key name that should be used to access the worker nodes" @@ -17,7 +27,7 @@ variable "ec2_ssh_key" { variable "desired_size" { type = number - description = "Desired number of worker nodes (external changes ignored)" + description = "Initial desired number of worker nodes (external changes ignored)" } variable "max_size" { @@ -44,18 +54,30 @@ variable "existing_workers_role_policy_arns" { variable "existing_workers_role_policy_arns_count" { type = number default = 0 - description = "Count of existing policy ARNs that will be attached to the workers default role on creation. Needed to prevent Terraform error `count can't be computed`" + description = "Obsolete and ignored. Allowed for backward compatibility." } variable "ami_type" { type = string - description = "Type of Amazon Machine Image (AMI) associated with the EKS Node Group. Defaults to `AL2_x86_64`. Valid values: `AL2_x86_64`, `AL2_x86_64_GPU`. Terraform will only perform drift detection if a configuration value is provided" + description = <<-EOT + Type of Amazon Machine Image (AMI) associated with the EKS Node Group. + Defaults to `AL2_x86_64`. Valid values: `AL2_x86_64`, `AL2_x86_64_GPU`, and `AL2_ARM_64`. + EOT default = "AL2_x86_64" + validation { + condition = ( + contains(["AL2_x86_64", "AL2_x86_64_GPU", "AL2_ARM_64"], var.ami_type) + ) + error_message = "Var ami_type must be one of \"AL2_x86_64\", \"AL2_x86_64_GPU\", and \"AL2_ARM_64\"." + } } variable "disk_size" { type = number - description = "Disk size in GiB for worker nodes. Defaults to 20. Terraform will only perform drift detection if a configuration value is provided" + description = <<-EOT + Disk size in GiB for worker nodes. Defaults to 20. Ignored it `launch_template_id` is supplied. + Terraform will only perform drift detection if a configuration value is provided. + EOT default = 20 } @@ -74,20 +96,64 @@ variable "instance_types" { variable "kubernetes_labels" { type = map(string) - description = "Key-value mapping of Kubernetes labels. Only labels that are applied with the EKS API are managed by this argument. Other Kubernetes labels applied to the EKS Node Group will not be managed" + description = <<-EOT + Key-value mapping of Kubernetes labels. Only labels that are applied with the EKS API are managed by this argument. + Other Kubernetes labels applied to the EKS Node Group will not be managed. + EOT + default = {} +} + +variable "kubernetes_taints" { + type = map(string) + description = "Key-value mapping of Kubernetes taints." default = {} } +variable "kubelet_additional_options" { + type = string + description = <<-EOT + Additional flags to pass to kubelet. + DO NOT include `--node-labels` or `--node-taints`, + use `kubernetes_labels` and `kubernetes_taints` to specify those." + EOT + default = "" + validation { + condition = (length(compact([var.kubelet_additional_options])) == 0 ? true : + length(regexall("--node-labels", var.kubelet_additional_options)) == 0 && + length(regexall("--node-taints", var.kubelet_additional_options)) == 0 + ) + error_message = "Var kubelet_additional_options must not contain \"--node-labels\" or \"--node-taints\". Use `kubernetes_labels` and `kubernetes_taints` to specify labels and taints." + } +} + +variable "ami_image_id" { + type = string + description = "AMI to use. Ignored of `launch_template_id` is supplied." + default = null +} + variable "ami_release_version" { type = string - description = "AMI version of the EKS Node Group. Defaults to latest version for Kubernetes version" + description = "AMI version to use, e.g. \"1.16.13-20200821\" (no \"v\"). Defaults to latest version for Kubernetes version." default = null + validation { + condition = ( + length(compact([var.ami_release_version])) == 0 ? true : length(regexall("^\\d+\\.\\d+\\.\\d+-\\d+$", var.ami_release_version)) == 1 + ) + error_message = "Var ami_release_version, if supplied, must be like \"1.16.13-20200821\" (no \"v\")." + } } variable "kubernetes_version" { type = string description = "Kubernetes version. Defaults to EKS Cluster Kubernetes version. Terraform will only perform drift detection if a configuration value is provided" default = null + validation { + condition = ( + length(compact([var.kubernetes_version])) == 0 ? true : length(regexall("^\\d+\\.\\d+$", var.kubernetes_version)) == 1 + ) + error_message = "Var kubernetes_version, if supplied, must be like \"1.16\" (no patch level)." + } } variable "source_security_group_ids" { @@ -102,20 +168,59 @@ variable "module_depends_on" { description = "Can be any value desired. Module will wait for this value to be computed before creating node group." } -variable "launch_template_id" { - type = string - description = "The ID of a custom launch template to use for the EKS node group." +variable "launch_template_name" { + type = string + // Note: the aws_launch_template data source only accepts name, not ID, to specify the launch template, so we cannot support ID as input. + description = "The name (not ID) of a custom launch template to use for the EKS node group. If provided, it must specify the AMI image id." default = null } variable "launch_template_version" { type = string - description = "A specific version of the above specific launch template" + description = "The version of the specified launch template to use. Defaults to latest version." default = null } +variable "resources_to_tag" { + type = list(string) + description = "List of auto-launched resource types to tag. Valid types are \"instance\", \"volume\", \"elastic-gpu\", \"spot-instances-request\"." + default = [] + validation { + condition = ( + length(compact([for r in var.resources_to_tag : r if ! contains(["instance", "volume", "elastic-gpu", "spot-instances-request"], r)])) == 0 + ) + error_message = "Invalid resource type in `resources_to_tag`. Valid types are \"instance\", \"volume\", \"elastic-gpu\", \"spot-instances-request\"." + } +} + variable "before_cluster_joining_userdata" { type = string default = "" - description = "Additional commands to execute on each worker node before joining the EKS cluster (before executing the `bootstrap.sh` script). For more info, see https://kubedex.com/90-days-of-aws-eks-in-production" + description = "Additional `bash` commands to execute on each worker node before joining the EKS cluster (before executing the `bootstrap.sh` script). For more info, see https://kubedex.com/90-days-of-aws-eks-in-production" +} + +variable "after_cluster_joining_userdata" { + type = string + default = "" + description = "Additional `bash` commands to execute on each worker node after joining the EKS cluster (after executing the `bootstrap.sh` script). For more info, see https://kubedex.com/90-days-of-aws-eks-in-production" +} + +variable "bootstrap_additional_options" { + type = string + default = "" + description = "Additional options to bootstrap.sh. DO NOT include `--kubelet-additional-args`, use `kubelet_additional_args` var instead." +} + +variable "userdata_override" { + type = string + default = null + description = <<-EOT + Many features of this module rely on the `bootstrap.sh` provided with Amazon Linux, and this module + may generate "user data" that expects to find that script. If you want to use an AMI that is not + compatible with the Amazon Linux `bootstrap.sh` initialization, then use `userdata_override` to provide + your own (Base64 encoded) user data. Use "" to prevent any user data from being set. + + Setting `userdata_override` disables `kubernetes_taints`, `kubelet_additional_options`, + `before_cluster_joining_userdata`, `after_cluster_joining_userdata`, and `bootstrap_additional_options`. + EOT } diff --git a/versions.tf b/versions.tf index e62ead0..761968f 100644 --- a/versions.tf +++ b/versions.tf @@ -1,9 +1,10 @@ terraform { - required_version = ">= 0.13.0, < 0.14.0" + required_version = ">= 0.13.0" required_providers { - aws = "~> 3.0" - template = "~> 2.0" - local = "~> 1.3" + aws = ">= 3.0" + template = ">= 2.0" + local = ">= 1.3" + random = ">= 2.0" } }