From 062e10910465a4384a6922b96c29365e2b03c71a Mon Sep 17 00:00:00 2001 From: Piotr Stawarski Date: Wed, 5 Jun 2024 12:37:52 +0200 Subject: [PATCH] cleanups --- README.md | 105 ++++++++++++++++++++++++---- aws-auth.tf | 10 +-- compute.tf | 6 +- examples/basic/main.tf | 75 ++++++++++++++++++++ examples/basic/outputs.tf | 9 +++ examples/basic/override.tf | 5 ++ examples/basic/terraform.tf | 32 +++++++++ examples/basic/update-kubeconfig.sh | 12 ++++ main.tf | 21 +++--- outputs.tf | 2 +- role-cluster.tf | 2 +- role-fargate.tf | 2 +- role-node.tf | 6 +- variables.tf | 22 ++++++ 14 files changed, 273 insertions(+), 36 deletions(-) create mode 100644 examples/basic/main.tf create mode 100644 examples/basic/outputs.tf create mode 100644 examples/basic/override.tf create mode 100644 examples/basic/terraform.tf create mode 100755 examples/basic/update-kubeconfig.sh diff --git a/README.md b/README.md index f6cb368..ceb21b0 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Meet **OPSd**. The unique and effortless way of managing cloud infrastructure. -# terraform-module-template +# terraform-module-aws-kubernetes ## Introduction @@ -11,12 +11,24 @@ What does the module provide? ## Usage ```hcl -module "module_name" { - source = "github.com/opsd-io/module_name?ref=v0.0.1" - - # Variables - variable_1 = "foo" - variable_2 = "bar" +module "kubernetes" { + source = "github.com/opsd-io/terraform-module-aws-kubernetes" + name = "basic-k8s-example" + + subnet_ids = [ + for subnet in module.network.public_subnet_groups["public1"] : subnet.id + ] + node_group_subnet_ids = [ + for subnet in module.network.private_subnet_groups["nodes1"] : subnet.id + ] + + node_groups = { + main = { + max_size = 9 + desired_size = 1 + disk_size = 8 + } + } } ``` @@ -27,11 +39,18 @@ module "module_name" { | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 1.3.1 | +| [terraform](#requirement\_terraform) | >= 1.5.5 | +| [aws](#requirement\_aws) | ~> 5.0 | +| [kubernetes](#requirement\_kubernetes) | ~> 2.0 | +| [tls](#requirement\_tls) | ~> 4.0 | ## Providers -No providers. +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | ~> 5.0 | +| [kubernetes](#provider\_kubernetes) | ~> 2.0 | +| [tls](#provider\_tls) | ~> 4.0 | ## Modules @@ -39,15 +58,77 @@ No modules. ## Resources -No resources. +| Name | Type | +|------|------| +| [aws_cloudwatch_log_group.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource | +| [aws_eks_cluster.main](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_cluster) | resource | +| [aws_eks_fargate_profile.main](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_fargate_profile) | resource | +| [aws_eks_node_group.main](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_node_group) | resource | +| [aws_iam_openid_connect_provider.eks_cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_openid_connect_provider) | resource | +| [aws_iam_role.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role.fargate](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role.node](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role_policy_attachment.ec2_container_registry_readonly](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.eks_cluster_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.eks_cni_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.eks_fargate_pod_execution_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.eks_worker_node_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [kubernetes_config_map.aws_auth](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/config_map) | resource | +| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | +| [aws_iam_policy_document.assume_role_cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.assume_role_fargate](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.assume_role_node](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source | +| [tls_certificate.oidc_issuer](https://registry.terraform.io/providers/hashicorp/tls/latest/docs/data-sources/certificate) | data source | ## Inputs -No inputs. +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [auth\_map\_accounts](#input\_auth\_map\_accounts) | Maps IAM ARN from these accounts to username. | `list(string)` | 
[
  "current"
]
| no | +| [auth\_map\_roles](#input\_auth\_map\_roles) | Maps an IAM role to a username and set of groups. | 
list(object({
    arn      = string
    username = optional(string)
    groups   = optional(list(string))
  }))
| `[]` | no | +| [auth\_map\_users](#input\_auth\_map\_users) | Maps an IAM user to a static username and set of groups. | 
list(object({
    arn      = string
    username = optional(string)
    groups   = optional(list(string))
  }))
| `[]` | no | +| [cluster\_log\_retention\_in\_days](#input\_cluster\_log\_retention\_in\_days) | Specifies the number of days you want to retain log events. | `number` | `7` | no | +| [common\_tags](#input\_common\_tags) | A map of tags to assign to every resource in this module. | `map(string)` | `{}` | no | +| [ec2\_ssh\_key](#input\_ec2\_ssh\_key) | The EC2 Key Pair name that provides access to the worker nodes. | `string` | `null` | no | +| [enabled\_cluster\_log\_types](#input\_enabled\_cluster\_log\_types) | List of the desired control plane logging to enable. | `list(string)` | 
[
  "api",
  "authenticator",
  "controllerManager",
  "scheduler"
]
| no | +| [encryption\_key\_arn](#input\_encryption\_key\_arn) | ARN of the KMS customer master key for secrets encryption. | `string` | `null` | no | +| [endpoint\_private\_access](#input\_endpoint\_private\_access) | Whether the Amazon EKS private API server endpoint is enabled. | `bool` | `false` | no | +| [endpoint\_public\_access](#input\_endpoint\_public\_access) | Whether the Amazon EKS public API server endpoint is enabled. | `bool` | `true` | no | +| [fargate\_profiles](#input\_fargate\_profiles) | Map of EKS Fargate Profile definitions. | 
map(object({
    subnet_ids = optional(set(string))
    namespace  = string
    labels     = optional(map(string))
  }))
| `{}` | no | +| [fargate\_role\_arns](#input\_fargate\_role\_arns) | Additional IAM role ARNs of Fargate Profiles managed externally. | `list(string)` | `[]` | no | +| [fargate\_subnet\_ids](#input\_fargate\_subnet\_ids) | Identifiers of private EC2 Subnets to associate with the EKS Fargate Profiles. | `set(string)` | `[]` | no | +| [k8s\_version](#input\_k8s\_version) | Desired Kubernetes master version. | `string` | `null` | no | +| [masters\_role\_arns](#input\_masters\_role\_arns) | List of IAM role to set as system:masters. Shortcut for auth\_map\_roles. | `list(string)` | `[]` | no | +| [name](#input\_name) | The name of the cluster. | `string` | n/a | yes | +| [node\_group\_subnet\_ids](#input\_node\_group\_subnet\_ids) | Identifiers of EC2 Subnets to associate with the EKS Node Groups. | `set(string)` | `[]` | no | +| [node\_groups](#input\_node\_groups) | Map of EKS Node Group definitions. | 
map(object({
    subnet_ids    = optional(set(string))
    ami_type      = optional(string, "AL2_x86_64")
    capacity_type = optional(string, "ON_DEMAND")
    instance_type = optional(string, "t3.medium")
    disk_size     = optional(number, 20)
    min_size      = optional(number, 0)
    max_size      = optional(number, 1)
    desired_size  = optional(number, 0)
    labels        = optional(map(string))
    taints = optional(list(object({
      key    = string
      value  = string
      effect = string # Valid values: NO_SCHEDULE, NO_EXECUTE, PREFER_NO_SCHEDULE.
    })), [])
  }))
| `{}` | no | +| [nodes\_role\_arns](#input\_nodes\_role\_arns) | Additional IAM role ARNs of Node Groups managed externally. | `list(string)` | `[]` | no | +| [public\_access\_cidrs](#input\_public\_access\_cidrs) | List of CIDR blocks that can access the Amazon EKS public API server endpoint when enabled. | `list(string)` | 
[
  "0.0.0.0/0"
]
| no | +| [security\_group\_ids](#input\_security\_group\_ids) | List of security group IDs to allow communication between your worker nodes and the Kubernetes control plane. | `set(string)` | `[]` | no | +| [source\_security\_group\_ids](#input\_source\_security\_group\_ids) | Set of EC2 Security Group IDs to allow SSH access (port 22) from on the worker nodes. | `set(string)` | `null` | no | +| [subnet\_ids](#input\_subnet\_ids) | List of subnet IDs. Must be in at least two different availability zones. | `set(string)` | n/a | yes | ## Outputs -No outputs. +| Name | Description | +|------|-------------| +| [arn](#output\_arn) | The ARN of the cluster. | +| [cluster\_ca](#output\_cluster\_ca) | Decoded CA certificate of the cluster. | +| [cluster\_role\_arn](#output\_cluster\_role\_arn) | The ARN of the IAM role that provides permissions for the Kubernetes control plane. | +| [cluster\_role\_name](#output\_cluster\_role\_name) | The name of the IAM role that provides permissions for the Kubernetes control plane. | +| [endpoint](#output\_endpoint) | Endpoint for your Kubernetes API server. | +| [fargate\_role\_arn](#output\_fargate\_role\_arn) | The ARN of the IAM Role that provides permissions for the EKS Fargate Profile. | +| [fargate\_role\_name](#output\_fargate\_role\_name) | The name of the IAM Role that provides permissions for the EKS Fargate Profile. | +| [id](#output\_id) | The ID of the cluster. | +| [name](#output\_name) | The name of the cluster. | +| [node\_role\_arn](#output\_node\_role\_arn) | The ARN of the IAM Role that provides permissions for the EKS Node Group. | +| [node\_role\_name](#output\_node\_role\_name) | The name of the IAM Role that provides permissions for the EKS Node Group. | +| [oidc\_issuer](#output\_oidc\_issuer) | Issuer URL for the OpenID Connect identity provider. | +| [openid\_arn](#output\_openid\_arn) | The ARN assigned by AWS for IAM OpenID Connect of the cluster. | +| [openid\_sub](#output\_openid\_sub) | The URL of the identity provider. Corresponds to the iss claim. | +| [region](#output\_region) | The region of of the cluster. | +| [version](#output\_version) | The Kubernetes master version. | ## Examples of usage diff --git a/aws-auth.tf b/aws-auth.tf index c345f75..dfdbecd 100644 --- a/aws-auth.tf +++ b/aws-auth.tf @@ -3,8 +3,7 @@ locals { current = data.aws_caller_identity.current.account_id } - # TODO: merge with extra roles from vars - nodes_role_arns = [aws_iam_role.node.arn] + nodes_role_arns = concat(var.nodes_role_arns, [aws_iam_role.node.arn]) nodes_map = [for arn in local.nodes_role_arns : { rolearn = arn username = "system:node:{{EC2PrivateDNSName}}" @@ -14,8 +13,7 @@ locals { ] }] - # TODO: merge with extra roles from vars - fargate_role_arns = [aws_iam_role.fargate.arn] + fargate_role_arns = concat(var.fargate_role_arns, [aws_iam_role.fargate.arn]) fargate_map = [for arn in local.fargate_role_arns : { rolearn = arn username = "system:node:{{SessionName}}" @@ -26,9 +24,7 @@ locals { ] }] - # TODO: merge with extra roles from vars - masters_role_arns = ["arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/devops"] - masters_map = [for arn in local.masters_role_arns : { + masters_map = [for arn in var.masters_role_arns : { rolearn = arn username = "master:{{AccountID}}:{{SessionName}}" groups = [ diff --git a/compute.tf b/compute.tf index a1af1e1..2dc7516 100644 --- a/compute.tf +++ b/compute.tf @@ -58,14 +58,14 @@ resource "aws_eks_node_group" "main" { # only if node_group_name_prefix is in use, but (local.fixed_names == false) does not work.. create_before_destroy = true ignore_changes = [ - # scaling_config[0].desired_size, # needed for autoscaling + scaling_config[0].desired_size, # needed for autoscaling to work ] } depends_on = [ aws_eks_cluster.main, aws_iam_role.node, - aws_iam_role_policy_attachment.AmazonEKSWorkerNodePolicy, + aws_iam_role_policy_attachment.eks_worker_node_policy, kubernetes_config_map.aws_auth, ] @@ -91,7 +91,7 @@ resource "aws_eks_fargate_profile" "main" { depends_on = [ aws_eks_cluster.main, aws_iam_role.fargate, - aws_iam_role_policy_attachment.AmazonEKSFargatePodExecutionRolePolicy, + aws_iam_role_policy_attachment.eks_fargate_pod_execution_role_policy, kubernetes_config_map.aws_auth, ] diff --git a/examples/basic/main.tf b/examples/basic/main.tf new file mode 100644 index 0000000..4d161a8 --- /dev/null +++ b/examples/basic/main.tf @@ -0,0 +1,75 @@ +module "network" { + source = "github.com/opsd-io/terraform-module-aws-network" + + vpc_name = "k8s-test-vpc" + cidr_block = "10.100.0.0/16" + + public_subnet_groups = { + "public1" = { + availability_zones = { + "a" = { cidr_block = "10.100.1.0/24", nat_gateway = true } + "b" = { cidr_block = "10.100.2.0/24" } + "c" = { cidr_block = "10.100.3.0/24" } + } + } + } + private_subnet_groups = { + "nodes1" = { + nat_group_name = "public1" + availability_zones = { + "a" = { cidr_block = "10.100.101.0/24" } + "b" = { cidr_block = "10.100.102.0/24" } + "c" = { cidr_block = "10.100.103.0/24" } + } + } + "fargate1" = { + nat_group_name = "public1" + availability_zones = { + "a" = { cidr_block = "10.100.201.0/24" } + "b" = { cidr_block = "10.100.202.0/24" } + "c" = { cidr_block = "10.100.203.0/24" } + } + } + } +} + +module "kubernetes" { + source = "github.com/opsd-io/terraform-module-aws-kubernetes" + name = "basic-k8s-example" + + subnet_ids = [ + for subnet in module.network.public_subnet_groups["public1"] : subnet.id + ] + + node_group_subnet_ids = [ + for subnet in module.network.private_subnet_groups["nodes1"] : subnet.id + ] + node_groups = { + main = { + max_size = 9 + desired_size = 1 + disk_size = 8 + } + } + + fargate_subnet_ids = [ + for subnet in module.network.private_subnet_groups["fargate1"] : subnet.id + ] + fargate_profiles = { + "default-namespace" = { # default is bad, better put it "far" + namespace = "default" + } + "amazonaws-components" = { # this should put CoreDNS on fargate + namespace = "*" + labels = { "eks.amazonaws.com/component" = "*" } + } + } +} + +module "autoscaler" { + source = "github.com/opsd-io/terraform-module-eks-autoscaler" + cluster_region = module.kubernetes.region + cluster_name = module.kubernetes.name + openid_arn = module.kubernetes.openid_arn + openid_sub = module.kubernetes.openid_sub +} diff --git a/examples/basic/outputs.tf b/examples/basic/outputs.tf new file mode 100644 index 0000000..9702544 --- /dev/null +++ b/examples/basic/outputs.tf @@ -0,0 +1,9 @@ +# output "network" { +# description = "The network module outputs." +# value = module.network +# } + +output "kubernetes" { + description = "The kubernetes module outputs." + value = module.kubernetes +} diff --git a/examples/basic/override.tf b/examples/basic/override.tf new file mode 100644 index 0000000..eb9cd7e --- /dev/null +++ b/examples/basic/override.tf @@ -0,0 +1,5 @@ +# Make sure we're using working version (from local directory, not git). + +module "kubernetes" { + source = "./../.." +} diff --git a/examples/basic/terraform.tf b/examples/basic/terraform.tf new file mode 100644 index 0000000..90b87e3 --- /dev/null +++ b/examples/basic/terraform.tf @@ -0,0 +1,32 @@ +terraform { + required_version = ">= 1.5.5" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.0" + } + } +} + +provider "aws" { + region = "eu-central-1" +} + +data "aws_eks_cluster_auth" "this" { + name = module.kubernetes.name +} + +provider "kubernetes" { + host = module.kubernetes.endpoint + cluster_ca_certificate = module.kubernetes.cluster_ca + token = data.aws_eks_cluster_auth.this.token +} + +provider "helm" { + kubernetes { + host = module.kubernetes.endpoint + cluster_ca_certificate = module.kubernetes.cluster_ca + token = data.aws_eks_cluster_auth.this.token + } +} diff --git a/examples/basic/update-kubeconfig.sh b/examples/basic/update-kubeconfig.sh new file mode 100755 index 0000000..aa40fca --- /dev/null +++ b/examples/basic/update-kubeconfig.sh @@ -0,0 +1,12 @@ +#!/bin/sh -eux + +aws sts get-caller-identity + +aws eks update-kubeconfig \ + --region "eu-central-1" \ + --name "basic-k8s-example" \ + --alias "k8s-example" + +kubectl version + +kubectl auth whoami diff --git a/main.tf b/main.tf index 757d2af..dc365b3 100644 --- a/main.tf +++ b/main.tf @@ -1,10 +1,19 @@ terraform { - required_version = ">= 1.3.1" + required_version = ">= 1.5.5" + required_providers { aws = { source = "hashicorp/aws" version = "~> 5.0" } + kubernetes = { + source = "hashicorp/kubernetes" + version = "~> 2.0" + } + tls = { + source = "hashicorp/tls" + version = "~> 4.0" + } } } @@ -16,10 +25,6 @@ data "aws_region" "current" { # no arguments } -data "aws_eks_cluster_auth" "main" { - name = aws_eks_cluster.main.name -} - resource "aws_eks_cluster" "main" { name = var.name version = var.k8s_version @@ -55,9 +60,9 @@ resource "aws_eks_cluster" "main" { } depends_on = [ - aws_iam_role_policy_attachment.AmazonEKSClusterPolicy, - aws_iam_role_policy_attachment.AmazonEKSWorkerNodePolicy, - aws_iam_role_policy_attachment.AmazonEKSFargatePodExecutionRolePolicy, + aws_iam_role_policy_attachment.eks_cluster_policy, + aws_iam_role_policy_attachment.eks_worker_node_policy, + aws_iam_role_policy_attachment.eks_fargate_pod_execution_role_policy, aws_cloudwatch_log_group.cluster, ] } diff --git a/outputs.tf b/outputs.tf index 6eba015..cd214e0 100644 --- a/outputs.tf +++ b/outputs.tf @@ -24,7 +24,7 @@ output "version" { } output "region" { - description = "The region of the state storage resources." + description = "The region of of the cluster." value = data.aws_region.current.name } diff --git a/role-cluster.tf b/role-cluster.tf index 7184def..bd24fd0 100644 --- a/role-cluster.tf +++ b/role-cluster.tf @@ -16,7 +16,7 @@ resource "aws_iam_role" "cluster" { assume_role_policy = data.aws_iam_policy_document.assume_role_cluster.json } -resource "aws_iam_role_policy_attachment" "AmazonEKSClusterPolicy" { +resource "aws_iam_role_policy_attachment" "eks_cluster_policy" { policy_arn = "arn:aws:iam::aws:policy/AmazonEKSClusterPolicy" role = aws_iam_role.cluster.name } diff --git a/role-fargate.tf b/role-fargate.tf index cb05f26..64b25f0 100644 --- a/role-fargate.tf +++ b/role-fargate.tf @@ -16,7 +16,7 @@ resource "aws_iam_role" "fargate" { assume_role_policy = data.aws_iam_policy_document.assume_role_fargate.json } -resource "aws_iam_role_policy_attachment" "AmazonEKSFargatePodExecutionRolePolicy" { +resource "aws_iam_role_policy_attachment" "eks_fargate_pod_execution_role_policy" { policy_arn = "arn:aws:iam::aws:policy/AmazonEKSFargatePodExecutionRolePolicy" role = aws_iam_role.fargate.name } diff --git a/role-node.tf b/role-node.tf index 1125a2e..4591efc 100644 --- a/role-node.tf +++ b/role-node.tf @@ -16,19 +16,19 @@ resource "aws_iam_role" "node" { assume_role_policy = data.aws_iam_policy_document.assume_role_node.json } -resource "aws_iam_role_policy_attachment" "AmazonEKSWorkerNodePolicy" { +resource "aws_iam_role_policy_attachment" "eks_worker_node_policy" { policy_arn = "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy" role = aws_iam_role.node.name } -resource "aws_iam_role_policy_attachment" "AmazonEC2ContainerRegistryReadOnly" { +resource "aws_iam_role_policy_attachment" "ec2_container_registry_readonly" { policy_arn = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly" role = aws_iam_role.node.name } # The AmazonEKS_CNI_Policy policy must be attached to either this role # or to a different role that is mapped to the aws-node Kubernetes service account. -resource "aws_iam_role_policy_attachment" "AmazonEKS_CNI_Policy" { +resource "aws_iam_role_policy_attachment" "eks_cni_policy" { policy_arn = "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy" role = aws_iam_role.node.name } diff --git a/variables.tf b/variables.tf index e1cd40a..9cd1523 100644 --- a/variables.tf +++ b/variables.tf @@ -96,6 +96,7 @@ variable "node_group_subnet_ids" { } variable "node_groups" { + description = "Map of EKS Node Group definitions." type = map(object({ subnet_ids = optional(set(string)) ami_type = optional(string, "AL2_x86_64") @@ -124,6 +125,7 @@ variable "fargate_subnet_ids" { } variable "fargate_profiles" { + description = "Map of EKS Fargate Profile definitions." type = map(object({ subnet_ids = optional(set(string)) namespace = string @@ -159,3 +161,23 @@ variable "auth_map_users" { })) default = [] } + +variable "nodes_role_arns" { + description = "Additional IAM role ARNs of Node Groups managed externally." + type = list(string) + default = [] +} + +variable "fargate_role_arns" { + description = "Additional IAM role ARNs of Fargate Profiles managed externally." + type = list(string) + default = [] +} + +variable "masters_role_arns" { + description = "List of IAM role to set as system:masters. Shortcut for auth_map_roles." + type = list(string) + default = [ + # "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/devops", + ] +}