Skip to content

Commit 51459cf

Browse files
arunlalparunlalp
committed
Add Karpenter Terraform configuration
Introduces a new Terraform configuration for Karpenter under infra/karpenter, including backend, main, variables, outputs, and provider version files. Updates dev tfvars to use the new Karpenter module and variables. This sets up Karpenter deployment and integration with EKS, including IAM roles, Helm chart, and example manifests.
1 parent 4b34493 commit 51459cf

File tree

6 files changed

+578
-77
lines changed

6 files changed

+578
-77
lines changed

‎infra/karpenter/backend.tf‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
terraform {
2+
backend "s3" {
3+
bucket = "techies-terraform"
4+
key = "dev/karpenter/karpenter.tfstate"
5+
region = "us-east-1"
6+
use_lockfile = true
7+
}
8+
}

‎infra/karpenter/main.tf‎

Lines changed: 269 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,269 @@
1+
provider "aws" {
2+
region = "us-west-2"
3+
alias = "virginia"
4+
}
5+
6+
provider "aws" {
7+
region = "us-east-1"
8+
alias = "east1"
9+
}
10+
11+
data "aws_security_groups" "security_groups" {
12+
filter {
13+
name = "tag:aws:eks:cluster-name"
14+
values = [var.cluster_name]
15+
}
16+
}
17+
18+
resource "aws_ec2_tag" "security_groups" {
19+
for_each = toset(data.aws_security_groups.security_groups.ids)
20+
21+
resource_id = each.value
22+
key = "karpenter.sh/discovery"
23+
value = var.cluster_name
24+
}
25+
26+
data "aws_eks_cluster" "cert_authority_data" {
27+
provider = aws.virginia
28+
name = var.cluster_name
29+
}
30+
31+
data "aws_eks_cluster_auth" "cluster_auth" {
32+
provider = aws.virginia
33+
name = data.aws_eks_cluster.cert_authority_data.name
34+
}
35+
36+
provider "helm" {
37+
kubernetes = {
38+
host = data.aws_eks_cluster.cert_authority_data.endpoint
39+
cluster_ca_certificate = base64decode(data.aws_eks_cluster.cert_authority_data.certificate_authority[0].data)
40+
41+
exec = {
42+
api_version = "client.authentication.k8s.io/v1beta1"
43+
command = "aws"
44+
args = ["eks", "get-token", "--cluster-name", data.aws_eks_cluster.cert_authority_data.name, "--region", local.region]
45+
}
46+
}
47+
}
48+
49+
50+
provider "kubectl" {
51+
apply_retry_count = 5
52+
host = data.aws_eks_cluster.cert_authority_data.endpoint
53+
cluster_ca_certificate = base64decode(data.aws_eks_cluster.cert_authority_data.certificate_authority[0].data)
54+
load_config_file = false
55+
56+
exec {
57+
api_version = "client.authentication.k8s.io/v1beta1"
58+
command = "aws"
59+
# This requires the awscli to be installed locally where Terraform is executed
60+
args = ["eks", "get-token", "--cluster-name", data.aws_eks_cluster.cert_authority_data.name]
61+
}
62+
}
63+
64+
data "aws_availability_zones" "available" {}
65+
data "aws_ecrpublic_authorization_token" "token" {
66+
provider = aws.east1
67+
}
68+
69+
locals {
70+
name = "ex-${basename(path.cwd)}"
71+
region = "us-west-2"
72+
73+
vpc_cidr = "10.0.0.0/16"
74+
azs = slice(data.aws_availability_zones.available.names, 0, 3)
75+
76+
tags = {
77+
Example = local.name
78+
}
79+
}
80+
81+
82+
# Karpenter
83+
84+
module "karpenter" {
85+
source = "../../modules/karpenter"
86+
87+
create = var.create
88+
tags = var.tags
89+
cluster_name = var.cluster_name
90+
create_iam_role = var.create_iam_role
91+
iam_role_name = var.iam_role_name
92+
iam_role_use_name_prefix = var.iam_role_use_name_prefix
93+
iam_role_path = var.iam_role_path
94+
iam_role_description = var.iam_role_description
95+
iam_role_max_session_duration = var.iam_role_max_session_duration
96+
iam_role_permissions_boundary_arn = var.iam_role_permissions_boundary_arn
97+
iam_role_tags = var.iam_role_tags
98+
iam_policy_name = var.iam_policy_name
99+
iam_policy_use_name_prefix = var.iam_policy_use_name_prefix
100+
iam_policy_path = var.iam_policy_path
101+
iam_policy_description = var.iam_policy_description
102+
iam_policy_statements = var.iam_policy_statements
103+
iam_role_policies = var.iam_role_policies
104+
ami_id_ssm_parameter_arns = var.ami_id_ssm_parameter_arns
105+
enable_pod_identity = var.enable_pod_identity
106+
enable_v1_permissions = var.enable_v1_permissions
107+
enable_irsa = var.enable_irsa
108+
irsa_oidc_provider_arn = var.irsa_oidc_provider_arn
109+
irsa_namespace_service_accounts = var.irsa_namespace_service_accounts
110+
irsa_assume_role_condition_test = var.irsa_assume_role_condition_test
111+
create_pod_identity_association = var.create_pod_identity_association
112+
namespace = var.namespace
113+
service_account = var.service_account
114+
enable_spot_termination = var.enable_spot_termination
115+
queue_name = var.queue_name
116+
queue_managed_sse_enabled = var.queue_managed_sse_enabled
117+
queue_kms_master_key_id = var.queue_kms_master_key_id
118+
queue_kms_data_key_reuse_period_seconds = var.queue_kms_data_key_reuse_period_seconds
119+
create_node_iam_role = var.create_node_iam_role
120+
cluster_ip_family = var.cluster_ip_family
121+
node_iam_role_arn = var.node_iam_role_arn
122+
node_iam_role_name = var.node_iam_role_name
123+
node_iam_role_use_name_prefix = var.node_iam_role_use_name_prefix
124+
node_iam_role_path = var.node_iam_role_path
125+
node_iam_role_description = var.node_iam_role_description
126+
node_iam_role_max_session_duration = var.node_iam_role_max_session_duration
127+
node_iam_role_permissions_boundary = var.node_iam_role_permissions_boundary
128+
node_iam_role_attach_cni_policy = var.node_iam_role_attach_cni_policy
129+
node_iam_role_additional_policies = var.node_iam_role_additional_policies
130+
node_iam_role_tags = var.node_iam_role_tags
131+
create_access_entry = var.create_access_entry
132+
access_entry_type = var.access_entry_type
133+
create_instance_profile = var.create_instance_profile
134+
rule_name_prefix = var.rule_name_prefix
135+
}
136+
137+
module "karpenter_disabled" {
138+
source = "../../modules/karpenter"
139+
140+
create = false
141+
}
142+
143+
# Karpenter Helm chart & manifests
144+
145+
resource "helm_release" "karpenter" {
146+
namespace = "kube-system"
147+
name = "karpenter"
148+
repository = "oci://public.ecr.aws/karpenter"
149+
repository_username = data.aws_ecrpublic_authorization_token.token.user_name
150+
repository_password = data.aws_ecrpublic_authorization_token.token.password
151+
chart = "karpenter"
152+
version = "1.3.0"
153+
wait = false
154+
155+
values = [
156+
<<-EOT
157+
serviceAccount:
158+
name: karpenter
159+
settings:
160+
clusterName: ${data.aws_eks_cluster.cert_authority_data.name}
161+
clusterEndpoint: ${data.aws_eks_cluster.cert_authority_data.endpoint}
162+
interruptionQueue: ${module.karpenter.queue_name}
163+
EOT
164+
]
165+
}
166+
167+
resource "kubectl_manifest" "karpenter_node_class" {
168+
yaml_body = <<-YAML
169+
apiVersion: karpenter.k8s.aws/v1
170+
kind: EC2NodeClass
171+
metadata:
172+
name: default
173+
spec:
174+
blockDeviceMappings:
175+
- deviceName: /dev/xvda
176+
ebs:
177+
volumeSize: 80Gi
178+
volumeType: gp3
179+
deleteOnTermination: true
180+
181+
amiSelectorTerms:
182+
- alias: bottlerocket@latest
183+
184+
role: ${module.karpenter.node_iam_role_name}
185+
subnetSelectorTerms:
186+
- tags:
187+
karpenter.sh/discovery: ${data.aws_eks_cluster.cert_authority_data.name}
188+
securityGroupSelectorTerms:
189+
- tags:
190+
karpenter.sh/discovery: ${data.aws_eks_cluster.cert_authority_data.name}
191+
tags:
192+
karpenter.sh/discovery: ${data.aws_eks_cluster.cert_authority_data.name}
193+
YAML
194+
195+
depends_on = [
196+
helm_release.karpenter
197+
]
198+
}
199+
200+
resource "kubectl_manifest" "karpenter_node_pool" {
201+
yaml_body = <<-YAML
202+
apiVersion: karpenter.sh/v1
203+
kind: NodePool
204+
metadata:
205+
name: default
206+
spec:
207+
template:
208+
spec:
209+
nodeClassRef:
210+
group: karpenter.k8s.aws
211+
kind: EC2NodeClass
212+
name: default
213+
requirements:
214+
- key: "karpenter.k8s.aws/instance-category"
215+
operator: In
216+
values: ["c", "m", "r"]
217+
- key: "karpenter.k8s.aws/instance-cpu"
218+
operator: In
219+
values: ["4", "8", "16", "32"]
220+
- key: "karpenter.k8s.aws/instance-hypervisor"
221+
operator: In
222+
values: ["nitro"]
223+
- key: "karpenter.k8s.aws/instance-generation"
224+
operator: Gt
225+
values: ["2"]
226+
limits:
227+
cpu: 1000
228+
disruption:
229+
consolidationPolicy: WhenEmpty
230+
consolidateAfter: 30s
231+
YAML
232+
233+
depends_on = [
234+
kubectl_manifest.karpenter_node_class
235+
]
236+
}
237+
238+
# Example deployment using the [pause image](https://www.ianlewis.org/en/almighty-pause-container)
239+
# and starts with zero replicas
240+
resource "kubectl_manifest" "karpenter_example_deployment" {
241+
yaml_body = <<-YAML
242+
apiVersion: apps/v1
243+
kind: Deployment
244+
metadata:
245+
name: inflate
246+
spec:
247+
replicas: 0
248+
selector:
249+
matchLabels:
250+
app: inflate
251+
template:
252+
metadata:
253+
labels:
254+
app: inflate
255+
spec:
256+
terminationGracePeriodSeconds: 0
257+
containers:
258+
- name: inflate
259+
image: public.ecr.aws/eks-distro/kubernetes/pause:3.7
260+
resources:
261+
requests:
262+
cpu: 1
263+
YAML
264+
265+
depends_on = [
266+
helm_release.karpenter
267+
]
268+
}
269+

‎infra/karpenter/outputs.tf‎

Whitespace-only changes.

0 commit comments

Comments
 (0)