(feat) managed node groups (#1)

aws-controller.tf

@@ -1,3 +1,13 @@
+resource "time_sleep" "wait" {
+  create_duration = "10s"
+  depends_on = [
+    aws_eks_cluster.cp,
+    aws_eks_node_group.ng,
+    aws_autoscaling_group.ng,
+    kubernetes_config_map.aws-auth,
+  ]
+}
+
 provider "helm" {
   alias = "aws-controller"
   kubernetes {
@@ -10,7 +20,7 @@ provider "helm" {
 
 module "alb-ingress" {
   source       = "./modules/alb-ingress"
-  depends_on   = [kubernetes_config_map.aws-auth]
+  depends_on   = [time_sleep.wait]
   providers    = { helm = helm.aws-controller }
   enabled      = local.node_groups_enabled
   cluster_name = aws_eks_cluster.cp.name
@@ -20,7 +30,7 @@ module "alb-ingress" {
 
 module "app-mesh" {
   source       = "./modules/app-mesh"
-  depends_on   = [kubernetes_config_map.aws-auth]
+  depends_on   = [time_sleep.wait]
   providers    = { helm = helm.aws-controller }
   enabled      = local.app_mesh_enabled
   cluster_name = aws_eks_cluster.cp.name
@@ -30,7 +40,7 @@ module "app-mesh" {
 
 module "cluster-autoscaler" {
   source       = "./modules/cluster-autoscaler"
-  depends_on   = [kubernetes_config_map.aws-auth]
+  depends_on   = [time_sleep.wait]
   providers    = { helm = helm.aws-controller }
   enabled      = local.node_groups_enabled
   cluster_name = aws_eks_cluster.cp.name
@@ -40,7 +50,7 @@ module "cluster-autoscaler" {
 
 module "container-insights" {
   source       = "./modules/container-insights"
-  depends_on   = [kubernetes_config_map.aws-auth]
+  depends_on   = [time_sleep.wait]
   providers    = { helm = helm.aws-controller }
   enabled      = local.container_insights_enabled
   cluster_name = aws_eks_cluster.cp.name

examples/complete/main.tf

@@ -15,6 +15,7 @@ module "eks" {
   name                       = var.name
   tags                       = var.tags
   kubernetes_version         = var.kubernetes_version
+  managed_node_groups        = var.managed_node_groups
   node_groups                = var.node_groups
   container_insights_enabled = true
   app_mesh_enabled           = true

examples/complete/outputs.tf

@@ -1,19 +1,19 @@
 output "eks" {
-  value       = module.eks.cluster
   description = "The generated AWS EKS cluster"
+  value       = module.eks.cluster
 }
 
 output "kubeconfig" {
-  value       = module.eks.kubeconfig
   description = "Bash script to update the kubeconfig file for the EKS cluster"
+  value       = module.eks.kubeconfig
 }
 
 output "kubecli" {
-  value       = module.irsa.kubecli
   description = "The kubectl command to attach annotations of IAM role for service account"
+  value       = module.irsa.kubecli
 }
 
 output "features" {
-  value       = module.eks.features
   description = "Features configurations of the AWS EKS cluster"
+  value       = module.eks.features
 }

examples/complete/tc2.tfvars

@@ -6,7 +6,7 @@ tags = {
 }
 kubernetes_version = "1.17"
 node_groups = {
-  default = {
+  mixed = {
     min_size      = 1
     max_size      = 3
     desired_size  = 2

examples/complete/tc3.tfvars

@@ -5,12 +5,20 @@ tags = {
   test = "tc3"
 }
 kubernetes_version = "1.17"
-node_groups = {
+managed_node_groups = {
   default = {
     min_size      = 1
     max_size      = 3
     desired_size  = 1
     instance_type = "t3.large"
+  }
+}
+node_groups = {
+  spot = {
+    min_size      = 1
+    max_size      = 3
+    desired_size  = 1
+    instance_type = "t3.large"
     instances_distribution = {
       spot_allocation_strategy = "lowest-price"
       spot_max_price           = "0.03"

examples/complete/tc5.tfvars

@@ -0,0 +1,15 @@
+aws_region = "ap-northeast-2"
+name       = "eks-tc5"
+tags = {
+  env  = "dev"
+  test = "tc5"
+}
+kubernetes_version = "1.17"
+managed_node_groups = {
+  default = {
+    min_size      = 1
+    max_size      = 3
+    desired_size  = 1
+    instance_type = "t3.large"
+  }
+}

examples/complete/variables.tf

@@ -22,13 +22,19 @@ variable "subnets" {
 variable "kubernetes_version" {
   description = "The target version of kubernetes"
   type        = string
-  default     = "1.14"
+  default     = "1.17"
 }
 
 variable "node_groups" {
   description = "Node groups definition"
   type        = map
-  default     = {}
+  default     = null
+}
+
+variable "managed_node_groups" {
+  description = "Amazon managed node groups definition"
+  type        = map
+  default     = null
 }
 
 ### description

labels.tf

@@ -31,12 +31,12 @@ locals {
     "kubernetes.io/role/internal-elb" = "1"
   }
   eks-autoscaler-tag = {
+    "k8s.io/cluster-autoscaler/enabled"                = "true"
     format("k8s.io/cluster-autoscaler/%s", local.name) = "owned"
   }
   eks-tag = merge(
     {
-      "eks:cluster-name"   = local.name
-      "eks:nodegroup-name" = local.name
+      "eks:cluster-name" = local.name
     },
     local.eks-owned-tag,
     local.eks-autoscaler-tag,

main.tf

@@ -2,9 +2,10 @@
 
 ## features
 locals {
-  node_groups_enabled        = (var.node_groups != null && length(var.node_groups) > 0) ? true : false
-  app_mesh_enabled           = (local.node_groups_enabled && var.app_mesh_enabled) ? true : false
-  container_insights_enabled = (local.node_groups_enabled && var.container_insights_enabled) ? true : false
+  node_groups_enabled         = (var.node_groups != null ? ((length(var.node_groups) > 0) ? true : false) : false)
+  managed_node_groups_enabled = (var.managed_node_groups != null ? ((length(var.managed_node_groups) > 0) ? true : false) : false)
+  app_mesh_enabled            = ((local.node_groups_enabled || local.managed_node_groups_enabled) && var.app_mesh_enabled) ? true : false
+  container_insights_enabled  = ((local.node_groups_enabled || local.managed_node_groups_enabled) && var.container_insights_enabled) ? true : false
 }
 
 ## control plane (cp)
@@ -59,7 +60,7 @@ data "aws_eks_cluster_auth" "cp" {
 ## node groups (ng)
 # security/policy
 resource "aws_iam_role" "ng" {
-  count = local.node_groups_enabled ? 1 : 0
+  count = local.node_groups_enabled || local.managed_node_groups_enabled ? 1 : 0
   name  = format("%s-ng", local.name)
   tags  = merge(local.default-tags, var.tags)
   assume_role_policy = jsonencode({
@@ -81,19 +82,19 @@ resource "aws_iam_instance_profile" "ng" {
 }
 
 resource "aws_iam_role_policy_attachment" "eks-ng" {
-  count      = local.node_groups_enabled ? 1 : 0
+  count      = local.node_groups_enabled || local.managed_node_groups_enabled ? 1 : 0
   policy_arn = format("arn:%s:iam::aws:policy/AmazonEKSWorkerNodePolicy", data.aws_partition.current.partition)
   role       = aws_iam_role.ng.0.name
 }
 
 resource "aws_iam_role_policy_attachment" "eks-cni" {
-  count      = local.node_groups_enabled ? 1 : 0
+  count      = local.node_groups_enabled || local.managed_node_groups_enabled ? 1 : 0
   policy_arn = format("arn:%s:iam::aws:policy/AmazonEKS_CNI_Policy", data.aws_partition.current.partition)
   role       = aws_iam_role.ng.0.name
 }
 
 resource "aws_iam_role_policy_attachment" "ecr-read" {
-  count      = local.node_groups_enabled ? 1 : 0
+  count      = local.node_groups_enabled || local.managed_node_groups_enabled ? 1 : 0
   policy_arn = format("arn:%s:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly", data.aws_partition.current.partition)
   role       = aws_iam_role.ng.0.name
 }
@@ -217,7 +218,12 @@ resource "aws_autoscaling_group" "ng" {
   }
 
   dynamic "tag" {
-    for_each = local.eks-tag
+    for_each = merge(
+      local.eks-tag,
+      {
+        "eks:nodegroup-name" = join("-", [aws_eks_cluster.cp.name, each.key])
+      }
+    )
     content {
       key                 = tag.key
       value               = tag.value
@@ -239,6 +245,37 @@ resource "aws_autoscaling_group" "ng" {
   ]
 }
 
+## managed node groups
+resource "aws_eks_node_group" "ng" {
+  for_each        = var.managed_node_groups != null ? var.managed_node_groups : {}
+  cluster_name    = aws_eks_cluster.cp.name
+  node_group_name = join("-", [aws_eks_cluster.cp.name, each.key])
+  node_role_arn   = aws_iam_role.ng.0.arn
+  subnet_ids      = local.subnet_ids
+  disk_size       = lookup(each.value, "disk_size", "20")
+  instance_types  = [lookup(each.value, "instance_type", "m5.xlarge")]
+  version         = aws_eks_cluster.cp.version
+  tags            = merge(local.default-tags, var.tags)
+
+  scaling_config {
+    max_size     = lookup(each.value, "max_size", 3)
+    min_size     = lookup(each.value, "min_size", 1)
+    desired_size = lookup(each.value, "desired_size", 1)
+  }
+
+  lifecycle {
+    create_before_destroy = true
+    ignore_changes        = [scaling_config[0].desired_size]
+  }
+
+  depends_on = [
+    aws_iam_role.ng,
+    aws_iam_role_policy_attachment.eks-ng,
+    aws_iam_role_policy_attachment.eks-cni,
+    aws_iam_role_policy_attachment.ecr-read,
+  ]
+}
+
 resource "aws_iam_openid_connect_provider" "oidc" {
   client_id_list  = ["sts.amazonaws.com"]
   thumbprint_list = ["9e99a48a9960b14926bb7f3b02e22da2b0ab7280"]
@@ -260,7 +297,7 @@ provider "kubernetes" {
 }
 
 resource "kubernetes_config_map" "aws-auth" {
-  count      = local.node_groups_enabled ? 1 : 0
+  count      = (local.managed_node_groups_enabled ? 0 : (local.node_groups_enabled ? 1 : 0))
   depends_on = [aws_eks_cluster.cp, aws_autoscaling_group.ng]
   metadata {
     name      = "aws-auth"

outputs.tf

@@ -1,59 +1,60 @@
 # output variables 
 
 output "name" {
-  value       = local.name
   description = "The EKS cluster name"
-}
-
-output "features" {
-  value = {
-    "app_mesh_enabled"           = local.app_mesh_enabled
-    "container_insights_enabled" = local.container_insights_enabled
-    "node_groups_enabled"        = local.node_groups_enabled
-  }
-  description = "Features configurations for the EKS "
+  value       = local.name
 }
 
 output "cluster" {
-  value       = aws_eks_cluster.cp
   description = "The EKS cluster attributes"
+  value       = aws_eks_cluster.cp
 }
 
 output "role" {
+  description = "The generated role of the EKS node group"
   value = (local.node_groups_enabled ? zipmap(
     ["name", "arn"],
     [aws_iam_role.ng.0.name, aws_iam_role.ng.0.arn]
   ) : null)
-  description = "The generated role of the EKS node group"
 }
 
 output "oidc" {
+  description = "The OIDC provider attributes for IAM Role for ServiceAccount"
   value = zipmap(
     ["url", "arn"],
     [local.oidc["url"], local.oidc["arn"]]
   )
-  description = "The OIDC provider attributes for IAM Role for ServiceAccount"
 }
 
 output "tags" {
+  description = "The generated tags for EKS integration"
   value = {
     "shared"       = local.eks-shared-tag
     "owned"        = local.eks-owned-tag
     "elb"          = local.eks-elb-tag
     "internal-elb" = local.eks-internal-elb-tag
   }
-  description = "The generated tags for EKS integration"
 }
 
 data "aws_region" "current" {}
 
 output "kubeconfig" {
+  description = "Bash script to update kubeconfig file"
   value = join(" ", [
     "bash -e",
     format("%s/script/update-kubeconfig.sh", path.module),
     format("-r %s", data.aws_region.current.name),
     format("-n %s", aws_eks_cluster.cp.name),
     "-k kubeconfig",
   ])
-  description = "Bash script to update kubeconfig file"
+}
+
+output "features" {
+  description = "Features configurations for the EKS "
+  value = {
+    "app_mesh_enabled"            = local.app_mesh_enabled
+    "container_insights_enabled"  = local.container_insights_enabled
+    "managed_node_groups_enabled" = local.managed_node_groups_enabled
+    "node_groups_enabled"         = local.node_groups_enabled
+  }
 }

variables.tf

@@ -18,6 +18,12 @@ variable "node_groups" {
   default     = null
 }
 
+variable "managed_node_groups" {
+  description = "Amazon managed node groups definition"
+  type        = map
+  default     = null
+}
+
 ### feature
 variable "enabled_cluster_log_types" {
   description = "A list of the desired control plane logging to enable"