Terraform drift detection cron job

.github/workflows/terraform-drift-detection.yaml

@@ -0,0 +1,67 @@
+name: Detect infrastructure drift
+on:
+  schedule:
+    - cron: '0 * * * *'  # run once an hour
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  detect-drift:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: terraform/production
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: arn:aws:iam::588562868276:role/GitHubActionsReadonlyRole-prod
+          aws-region: us-east-1
+
+      - name: Install Terraform
+        uses: hashicorp/setup-terraform@v3
+        with:
+          terraform_version: "~1.5.2"
+          terraform_wrapper: false
+
+      - name: Initialize Terraform
+        run: terraform init
+
+      - name: Run Terraform Plan
+        run: terraform plan -lock=false -detailed-exitcode -no-color -input=false -out=tfplan > tfplan_output.txt 2>&1
+        env:
+          TF_VAR_eks_cluster_role: "arn:aws:iam::588562868276:role/GitHubActionsReadonlyRole-prod"
+
+      - name: Send Slack alert on drift
+        if: failure()
+        run: |
+          # Post message
+          curl -X POST \
+          -H "Content-type: application/json" \
+          -H "Authorization: Bearer ${{ secrets.SLACK_BOT_TOKEN }}" \
+          -d '{
+                "channel": "spack-alerts",
+                "text": ":rotating_light: :rotating_light: :rotating_light: Infrastructure drift detected! :rotating_light: :rotating_light: :rotating_light:"
+              }' \
+          https://slack.com/api/chat.postMessage
+
+          # Upload TF plan stdout
+          curl -F file=@tfplan_output.txt \
+          -F channels=spack-alerts \
+          -F title="tfplan_output.txt" \
+          -F filetype="text" \
+          -H "Authorization: Bearer ${{ secrets.SLACK_BOT_TOKEN }}" \
+          https://slack.com/api/files.upload
+
+          # Upload TF plan binary file
+          curl -F file=@tfplan \
+          -F channels=spack-alerts \
+          -F title="tfplan" \
+          -F filetype="binary" \
+          -H "Authorization: Bearer ${{ secrets.SLACK_BOT_TOKEN }}" \
+          https://slack.com/api/files.upload

terraform/modules/spack/eks.tf

@@ -28,6 +28,13 @@ module "eks" {
       username = "admin"
       groups   = ["system:masters"]
     },
+    {
+      rolearn  = aws_iam_role.github_actions.arn,
+      username = "github-actions",
+      # See ./github_actions_iam.tf for ClusterRole/ClusterRoleBinding
+      # for the permissions given to this group
+      groups = ["github-actions"],
+    }
   ]
   # This is required for DNS resolution to work on Windows nodes.
   # See info about aws-auth configmap here - https://docs.aws.amazon.com/eks/latest/userguide/windows-support.html#enable-windows-support

terraform/modules/spack/github_actions_iam.tf

@@ -0,0 +1,92 @@
+locals {
+  iam_role_name = "GitHubActionsReadonlyRole-${var.deployment_name}"
+}
+
+resource "aws_iam_role" "github_actions" {
+  name        = local.iam_role_name
+  description = "Managed by Terraform. IAM Role that a GitHub Actions runner can assume to authenticate with AWS."
+
+  assume_role_policy = jsonencode({
+    "Version" : "2012-10-17",
+    "Statement" : [
+      {
+        "Effect" : "Allow",
+        "Principal" : {
+          "Federated" : var.github_actions_oidc_arn
+        },
+        "Action" : "sts:AssumeRoleWithWebIdentity",
+        "Condition" : {
+          "StringLike" : {
+            "token.actions.githubusercontent.com:sub" : "repo:spack/spack-infrastructure:ref:refs/heads/main",
+            "token.actions.githubusercontent.com:aud" : "sts.amazonaws.com"
+          }
+        }
+      },
+      {
+        "Action" : "sts:AssumeRole",
+        "Principal" : {
+          # Unfortunately, we need to do this until https://github.com/hashicorp/terraform-provider-aws/issues/27034 is resolved.
+          # This trust statement allows the role to assume itself, which is necessary for the GitHub Actions session user to run terraform plan.
+          "AWS" : "arn:aws:sts::${data.aws_caller_identity.current.account_id}:assumed-role/GitHubActionsReadonlyRole-${var.deployment_name}/GitHubActions"
+        },
+        "Effect" : "Allow",
+      },
+    ]
+  })
+
+  # The `ReadOnlyAccess` managed policy doesn't include secretsmanager, so we explicitly grant it here.
+  inline_policy {
+    name = "read-secrets"
+    policy = jsonencode({
+      "Version" : "2012-10-17",
+      "Statement" : [
+        {
+          "Effect" : "Allow",
+          "Action" : [
+            "secretsmanager:GetSecretValue"
+          ],
+          "Resource" : "*"
+        }
+      ]
+    })
+  }
+}
+
+# This policy grants the GitHub Actions role read-only access to most resources in the AWS account.
+# There are some exceptions, such as secretsmanager (see inline_policy above)
+resource "aws_iam_role_policy_attachment" "github_actions" {
+  role       = aws_iam_role.github_actions.name
+  policy_arn = "arn:aws:iam::aws:policy/ReadOnlyAccess"
+}
+
+# This ClusterRole and ClusterRoleBinding allow for read-only access to the
+# Kubernetes cluster. This allows the GitHub Actions role to run a `terraform plan`,
+# but crucially, not a `terraform apply` or other mutable actions.
+resource "kubectl_manifest" "github_actions_clusterrole" {
+  yaml_body = <<YAML
+    apiVersion: rbac.authorization.k8s.io/v1
+    kind: ClusterRole
+    metadata:
+      name: github-actions-oidc
+    rules:
+    - apiGroups: ["*"]
+      resources: ["*"]
+      verbs: ["get", "list", "watch"]
+  YAML
+}
+resource "kubectl_manifest" "github_actions_clusterrolebinding" {
+  yaml_body = <<YAML
+    apiVersion: rbac.authorization.k8s.io/v1
+    kind: ClusterRoleBinding
+    metadata:
+      name: github-actions-oidc
+    subjects:
+    - kind: Group
+      name: github-actions
+      apiGroup: rbac.authorization.k8s.io
+    roleRef:
+      kind: ClusterRole
+      name: ${kubectl_manifest.github_actions_clusterrole.name}
+      apiGroup: rbac.authorization.k8s.io
+    YAML
+}

terraform/modules/spack/variables.tf

@@ -8,6 +8,11 @@ variable "gitlab_url" {
   type        = string
 }
 
+variable "github_actions_oidc_arn" {
+  description = "ARN of the GitHub Actions OIDC provider."
+  type        = string
+}
+
 variable "vpc_cidr" {
   description = "CIDR for the VPC."
   type        = string

terraform/production/github_actions_oidc.tf

@@ -0,0 +1,9 @@
+data "tls_certificate" "github_actions" {
+  url = "https://token.actions.githubusercontent.com"
+}
+
+resource "aws_iam_openid_connect_provider" "github_actions" {
+  url             = "https://token.actions.githubusercontent.com"
+  client_id_list  = ["sts.amazonaws.com"]
+  thumbprint_list = [data.tls_certificate.github_actions.certificates.0.sha1_fingerprint]
+}

terraform/production/main.tf

@@ -41,6 +41,16 @@ terraform {
   }
 }
 
+variable "eks_cluster_role" {
+  type        = string
+  description = "Role for Terraform to assume that grants access to the EKS cluster."
+  default     = null
+}
+
+locals {
+  eks_cluster_role = coalesce(var.eks_cluster_role, module.production_cluster.cluster_access_role_arn)
+}
+
 provider "aws" {
   region = "us-east-1"
 }
@@ -57,7 +67,7 @@ provider "helm" {
         "get-token",
         "--region", "us-east-1",
         "--cluster-name", module.production_cluster.cluster_name,
-        "--role", module.production_cluster.cluster_access_role_arn
+        "--role", local.eks_cluster_role
       ]
     }
   }
@@ -74,7 +84,7 @@ provider "kubectl" {
       "get-token",
       "--region", "us-east-1",
       "--cluster-name", module.production_cluster.cluster_name,
-      "--role", module.production_cluster.cluster_access_role_arn
+      "--role", local.eks_cluster_role
     ]
   }
 }
@@ -90,7 +100,7 @@ provider "kubernetes" {
       "get-token",
       "--region", "us-east-1",
       "--cluster-name", module.production_cluster.cluster_name,
-      "--role", module.production_cluster.cluster_access_role_arn
+      "--role", local.eks_cluster_role
     ]
   }
 }
@@ -156,6 +166,7 @@ module "production_cluster" {
   opensearch_instance_type = "r6g.xlarge.search"
   opensearch_volume_size   = 500
 
-
   ses_email_domain = "spack.io"
+
+  github_actions_oidc_arn = aws_iam_openid_connect_provider.github_actions.arn
 }

terraform/staging/github_actions_oidc.tf

@@ -0,0 +1,3 @@
+data "aws_iam_openid_connect_provider" "github_actions" {
+  url = "https://token.actions.githubusercontent.com"
+}

terraform/staging/main.tf

@@ -41,6 +41,16 @@ terraform {
   }
 }
 
+variable "eks_cluster_role" {
+  type        = string
+  description = "Role for Terraform to assume that grants access to the EKS cluster."
+  default     = null
+}
+
+locals {
+  eks_cluster_role = coalesce(var.eks_cluster_role, module.staging_cluster.cluster_access_role_arn)
+}
+
 provider "aws" {
   region = "us-west-2"
 }
@@ -57,7 +67,7 @@ provider "helm" {
         "get-token",
         "--region", "us-west-2",
         "--cluster-name", module.staging_cluster.cluster_name,
-        "--role", module.staging_cluster.cluster_access_role_arn
+        "--role", local.eks_cluster_role,
       ]
     }
   }
@@ -74,7 +84,7 @@ provider "kubectl" {
       "get-token",
       "--region", "us-west-2",
       "--cluster-name", module.staging_cluster.cluster_name,
-      "--role", module.staging_cluster.cluster_access_role_arn
+      "--role", local.eks_cluster_role,
     ]
   }
 }
@@ -90,7 +100,7 @@ provider "kubernetes" {
       "get-token",
       "--region", "us-west-2",
       "--cluster-name", module.staging_cluster.cluster_name,
-      "--role", module.staging_cluster.cluster_access_role_arn
+      "--role", local.eks_cluster_role,
     ]
   }
 }
@@ -157,4 +167,6 @@ module "staging_cluster" {
   opensearch_volume_size   = 100
 
   ses_email_domain = "staging.spack.io"
+
+  github_actions_oidc_arn = data.aws_iam_openid_connect_provider.github_actions.arn
 }