initial tf apply fails, but re-plan and re-apply fix it

[rocoll]

Terraform Version Details

I run terraform from inside the viya4-iac-aws container.

Command definition:

$ alias terraform

alias terraform='docker container run --rm --group-add root --user 1000:977 -v /home/cloud-user/.aws:/.aws -v /home/cloud-user/.ssh:/.ssh -v /home/cloud-user/viya4-iac-aws:/workspace --entrypoint terraform viya4-iac-aws'

TF version:

$ terraform version

Terraform v1.10.5
on linux_amd64
+ provider registry.terraform.io/hashicorp/aws v5.100.0
+ provider registry.terraform.io/hashicorp/cloudinit v2.3.7
+ provider registry.terraform.io/hashicorp/external v2.3.5
+ provider registry.terraform.io/hashicorp/kubernetes v2.38.0
+ provider registry.terraform.io/hashicorp/local v2.5.3
+ provider registry.terraform.io/hashicorp/null v3.2.4
+ provider registry.terraform.io/hashicorp/random v3.7.2
+ provider registry.terraform.io/hashicorp/time v0.13.1
+ provider registry.terraform.io/hashicorp/tls v4.1.0

Terraform Variable File Details

Nothing special here.

# REQUIRED VARIABLES
# Necessary for use by the IAC
# --------------------------------------
# - Prefix is used for naming resources for easy identification
# - Location is the geo region where resources will be placed
#
prefix                                  = "crest-p41756"
location                                = "us-east-1" 

# ACCESS, IDENTITY, and AUTHENTICATION
# Who is doing what where
# --------------------------------------
#
aws_profile                             = "default"            # or whatever you named it
ssh_public_key                          = "~/.ssh/id_rsa.pub"
create_static_kubeconfig                = true

# CIDR
# Specify public access CIDR to allow ingress traffic to the EKS cluster
# --------------------------------------
# - Define access from RACE VMWARE and RACE Azure clients networks
#
default_public_access_cidrs         = ["149.173.0.0/16", "52.226.102.80/32", "52.226.102.81/32"]

# TAGS
# Optional metadata associated with AWS resources
# --------------------------------------
# - Resourceowner makes it easy to find associated resources
# - Project_Name and GEL_Project are for tracking
# - Chronos (old) and Smart Parking (new) are SAS IT programs to auto-shutdown resources
# - GEL Smart Parking Exemption: RITM0988495
#
tags = { "resourceowner"          = "crest-p41756", 
        "project_name"           = "PSGEL297", 
        "gel_project"            = "PSGEL297", 
        "disable_chronos"        = "True", 
        "smart_parking_disabled" = "True" 
    }

# EXTERNAL POSTGRES SERVER
# --------------------------------------
# - if defined, creates an External Postgres Server in AWS, else use internal Crunchy
#
#postgres_servers = {
#  default = {},
#}

## Cluster config
kubernetes_version                      = "1.31"

default_nodepool_node_count             = 2
default_nodepool_vm_type                = "m7i-flex.2xlarge"
default_nodepool_custom_data            = ""

## General
efs_performance_mode                    = "generalPurpose"
storage_type                            = "standard"

## Cluster Node Pools config
node_pools = {
cas = {
    "vm_type" = "m7i-flex.2xlarge"
    "cpu_type"     = "AL2_x86_64"
    "os_disk_type" = "gp2"
    "os_disk_size" = 200
    "os_disk_iops" = 0
    "min_nodes" = 4
    "max_nodes" = 5
    "node_taints" = ["workload.sas.com/class=cas:NoSchedule"]
    "node_labels" = {
    "workload.sas.com/class" = "cas"
    }
    "custom_data" = ""
    "metadata_http_endpoint"               = "enabled"
    "metadata_http_tokens"                 = "required"
    "metadata_http_put_response_hop_limit" = 1
},
compute = {
    "vm_type" = "m7i-flex.8xlarge"
    "cpu_type"     = "AL2_x86_64"
    "os_disk_type" = "gp2"
    "os_disk_size" = 200
    "os_disk_iops" = 0
    "min_nodes" = 1
    "max_nodes" = 5
    "node_taints" = ["workload.sas.com/class=compute:NoSchedule"]
    "node_labels" = {
    "workload.sas.com/class"        = "compute"
    "launcher.sas.com/prepullImage" = "sas-programming-environment"
    }
    "custom_data" = ""
    "metadata_http_endpoint"               = "enabled"
    "metadata_http_tokens"                 = "required"
    "metadata_http_put_response_hop_limit" = 1
},
connect = {
    "vm_type" = "m7i-flex.8xlarge"
    "cpu_type"     = "AL2_x86_64"
    "os_disk_type" = "gp2"
    "os_disk_size" = 200
    "os_disk_iops" = 0
    "min_nodes" = 1
    "max_nodes" = 5
    "node_taints" = ["workload.sas.com/class=connect:NoSchedule"]
    "node_labels" = {
    "workload.sas.com/class"        = "connect"
    "launcher.sas.com/prepullImage" = "sas-programming-environment"
    }
    "custom_data" = ""
    "metadata_http_endpoint"               = "enabled"
    "metadata_http_tokens"                 = "required"
    "metadata_http_put_response_hop_limit" = 1
},
stateless = {
    "vm_type" = "m7i-flex.4xlarge"
    "cpu_type"     = "AL2_x86_64"
    "os_disk_type" = "gp2"
    "os_disk_size" = 200
    "os_disk_iops" = 0
    "min_nodes" = 1
    "max_nodes" = 5
    "node_taints" = ["workload.sas.com/class=stateless:NoSchedule"]
    "node_labels" = {
    "workload.sas.com/class" = "stateless"
    }
    "custom_data" = ""
    "metadata_http_endpoint"               = "enabled"
    "metadata_http_tokens"                 = "required"
    "metadata_http_put_response_hop_limit" = 1
},
stateful = {
    "vm_type" = "m7i-flex.4xlarge"
    "cpu_type"     = "AL2_x86_64"
    "os_disk_type" = "gp2"
    "os_disk_size" = 200
    "os_disk_iops" = 0
    "min_nodes" = 1
    "max_nodes" = 3
    "node_taints" = ["workload.sas.com/class=stateful:NoSchedule"]
    "node_labels" = {
    "workload.sas.com/class" = "stateful"
    }
    "custom_data" = ""
    "metadata_http_endpoint"               = "enabled"
    "metadata_http_tokens"                 = "required"
    "metadata_http_put_response_hop_limit" = 1
}
}

# Jump Server
create_jump_vm                        = true
jump_vm_admin                         = "jumpuser"
jump_vm_type                          = "t3.small"

# NFS Server
# required ONLY when storage_type is "standard" to create NFS Server VM
create_nfs_public_ip                  = false
nfs_vm_admin                          = "nfsuser"
nfs_vm_type                           = "m7i-flex.xlarge"

Steps to Reproduce

Pretty much every first time I run "terraform apply".

Expected Behavior

I expect TF to complete provisioning AWS resources successfully.

Actual Behavior

The "terraform apply" fails with error:

╷
│ Error: Unauthorized
│ 
│   with module.kubeconfig.kubernetes_service_account.kubernetes_sa[0],
│   on modules/kubeconfig/main.tf line 66, in resource "kubernetes_service_account" "kubernetes_sa":
│   66: resource "kubernetes_service_account" "kubernetes_sa" {
│ 
╵

Additional Context

I've looked into a little bit and it seems that TF is running multiple tasks concurrently. When it gets to this point, one step is dependent on a resource (the k8s service acct) that doesn't exist yet. So TF errors out.

The workaround/fix for this is to simply re-plan (specifying the current tfstate file) and then re-apply:

# Note that I'm running TF from inside the viya4-iac-aws container
terraform plan \
     -input=false \
     -var-file=/workspace/${MY_PREFIX}.tfvars \
     -state /workspace/${MY_PREFIX}.tfstate \
     -out /workspace/${MY_PREFIX}.tfplan

To which it says that it will:

Plan: 6 to add, 2 to change, 0 to destroy.

And then "terraform apply" runs successfully after that and I have all expected AWS resources provisioned by the IAC.

Possibly related to Issue 345 because the internal discussions about that also note circumstances where required resources don't yet exist when the dependent objects need them. It seems there might be justification to consider re-structuring the "terraform apply" process to include multiple runs instead of just one so that required resources can be strictly provisioned before the dependent objects actually need them.

References

Issue 345

Code of Conduct

• I agree to follow this project's Code of Conduct

[rocoll]

The re-plan will Add 6 resources and Change 2 resources - specifically:

terraform plan \
>     -input=false \
>     -var-file=/workspace/${MY_PREFIX}.tfvars \
>     -state /workspace/${MY_PREFIX}.tfstate \
>     -out /workspace/${MY_PREFIX}.tfplan

<snip>

Terraform will perform the following actions:

  # data.cloudinit_config.jump[0] will be read during apply
  # (depends on a resource or a module with changes pending)
 <= data "cloudinit_config" "jump" {
      + base64_encode = true
      + boundary      = (known after apply)
      + gzip          = true
      + id            = (known after apply)
      + rendered      = (known after apply)

      + part {
          + content      = <<-EOT
                #cloud-config
                system_info:
                  default_user:
                    name: jumpuser
                
                #
                # First we'll update the repo and then update the OS.
                #
                package_update: true
                package_upgrade: true
                
                #
                # Install packages
                #
                packages:
                  - nfs-common
                
                #
                # Update /etc/fstab
                #
                mounts:
                  - ["192.168.58.112:/export","/viya-share","nfs","rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,noresvport","0","0"]
                
                #
                # Add nfs mounts
                #
                runcmd:
                  - if ! [ -z "192.168.58.112" ]
                  - then
                      #
                      # mount the nfs
                      #
                  -   while [ `df -h | grep "192.168.58.112:/export" | wc -l` -eq 0 ]; do sleep 5 && mount -a ; done
                      # Create pvs folder and adjust perms and owner only if the folder doesn't exist
                  -   if ! [ -d "/viya-share/pvs" ]
                  -   then
                        #
                        # Change permissions and owner
                        #
                  -     mkdir -p /viya-share/pvs
                  -     chmod 777 /viya-share -R
                  -     chown -R nobody:nogroup /viya-share
                  -   fi
                  - fi
            EOT
          + content_type = "text/cloud-config"
        }
    }

  # kubernetes_config_map.sas_iac_buildinfo will be created
  + resource "kubernetes_config_map" "sas_iac_buildinfo" {
      + data = (known after apply)
      + id   = (known after apply)

      + metadata {
          + generation       = (known after apply)
          + name             = "sas-iac-buildinfo"
          + namespace        = "kube-system"
          + resource_version = (known after apply)
          + uid              = (known after apply)
        }
    }

  # terraform_data.run_command[0] will be created
  + resource "terraform_data" "run_command" {
      + id = (known after apply)
    }

  # module.jump[0].data.aws_ami.ubuntu will be read during apply
  # (depends on a resource or a module with changes pending)
 <= data "aws_ami" "ubuntu" {
      + architecture          = (known after apply)
      + arn                   = (known after apply)
      + block_device_mappings = (known after apply)
      + boot_mode             = (known after apply)
      + creation_date         = (known after apply)
      + deprecation_time      = (known after apply)
      + description           = (known after apply)
      + ena_support           = (known after apply)
      + hypervisor            = (known after apply)
      + id                    = (known after apply)
      + image_id              = (known after apply)
      + image_location        = (known after apply)
      + image_owner_alias     = (known after apply)
      + image_type            = (known after apply)
      + imds_support          = (known after apply)
      + kernel_id             = (known after apply)
      + last_launched_time    = (known after apply)
      + most_recent           = true
      + name                  = (known after apply)
      + owner_id              = (known after apply)
      + owners                = [
          + "099720109477",
        ]
      + platform              = (known after apply)
      + platform_details      = (known after apply)
      + product_codes         = (known after apply)
      + public                = (known after apply)
      + ramdisk_id            = (known after apply)
      + root_device_name      = (known after apply)
      + root_device_type      = (known after apply)
      + root_snapshot_id      = (known after apply)
      + sriov_net_support     = (known after apply)
      + state                 = (known after apply)
      + state_reason          = (known after apply)
      + tags                  = (known after apply)
      + tpm_support           = (known after apply)
      + usage_operation       = (known after apply)
      + virtualization_type   = (known after apply)

      + filter {
          + name   = "architecture"
          + values = [
              + "x86_64",
            ]
        }
      + filter {
          + name   = "name"
          + values = [
              + "ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-server-*",
            ]
        }
      + filter {
          + name   = "root-device-type"
          + values = [
              + "ebs",
            ]
        }
      + filter {
          + name   = "virtualization-type"
          + values = [
              + "hvm",
            ]
        }
    }

  # module.jump[0].aws_instance.vm will be updated in-place
  ~ resource "aws_instance" "vm" {
        id                                   = "i-05c3b297e58acfd29"
      ~ public_dns                           = "ec2-3-82-73-63.compute-1.amazonaws.com" -> (known after apply)
      ~ public_ip                            = "3.82.73.63" -> (known after apply)
      ~ tags                                 = {
            "Name"                   = "crest-p41756-jump-vm"
            "disable_chronos"        = "True"
            "gel_project"            = "PSGEL297"
            "project_name"           = "PSGEL297"
            "resourceowner"          = "crest-p41756"
            "smart_parking_disabled" = "True"
          - "stoptime"               = "30 17 * * 1-5 US/Eastern" -> null
        }
      ~ tags_all                             = {
          - "stoptime"               = "30 17 * * 1-5 US/Eastern" -> null
            # (6 unchanged elements hidden)
        }
      ~ user_data                            = "f20b4049704c3c823f2514b7da5a3eca36b7734f" -> (known after apply)
        # (36 unchanged attributes hidden)

        # (8 unchanged blocks hidden)
    }

  # module.kubeconfig.data.kubernetes_secret.sa_secret[0] will be read during apply
  # (depends on a resource or a module with changes pending)
 <= data "kubernetes_secret" "sa_secret" {
      + data      = (sensitive value)
      + id        = (known after apply)
      + immutable = (known after apply)
      + type      = (known after apply)

      + metadata {
          + generation       = (known after apply)
          + name             = "crest-p41756-sa-secret"
          + namespace        = "kube-system"
          + resource_version = (known after apply)
          + uid              = (known after apply)
        }
    }

  # module.kubeconfig.kubernetes_cluster_role_binding.kubernetes_crb[0] will be created
  + resource "kubernetes_cluster_role_binding" "kubernetes_crb" {
      + id = (known after apply)

      + metadata {
          + generation       = (known after apply)
          + name             = "crest-p41756-cluster-admin-crb"
          + resource_version = (known after apply)
          + uid              = (known after apply)
        }

      + role_ref {
          + api_group = "rbac.authorization.k8s.io"
          + kind      = "ClusterRole"
          + name      = "cluster-admin"
        }

      + subject {
          + api_group = (known after apply)
          + kind      = "ServiceAccount"
          + name      = "crest-p41756-cluster-admin-sa"
          + namespace = "kube-system"
        }
    }

  # module.kubeconfig.kubernetes_secret.sa_secret[0] will be created
  + resource "kubernetes_secret" "sa_secret" {
      + data                           = (sensitive value)
      + id                             = (known after apply)
      + type                           = "kubernetes.io/service-account-token"
      + wait_for_service_account_token = true

      + metadata {
          + annotations      = {
              + "kubernetes.io/service-account.name" = "crest-p41756-cluster-admin-sa"
            }
          + generation       = (known after apply)
          + name             = "crest-p41756-sa-secret"
          + namespace        = "kube-system"
          + resource_version = (known after apply)
          + uid              = (known after apply)
        }
    }

  # module.kubeconfig.kubernetes_service_account.kubernetes_sa[0] will be created
  + resource "kubernetes_service_account" "kubernetes_sa" {
      + automount_service_account_token = true
      + default_secret_name             = (known after apply)
      + id                              = (known after apply)

      + metadata {
          + generation       = (known after apply)
          + name             = "crest-p41756-cluster-admin-sa"
          + namespace        = "kube-system"
          + resource_version = (known after apply)
          + uid              = (known after apply)
        }
    }

  # module.kubeconfig.local_file.kubeconfig will be created
  + resource "local_file" "kubeconfig" {
      + content              = (sensitive value)
      + content_base64sha256 = (known after apply)
      + content_base64sha512 = (known after apply)
      + content_md5          = (known after apply)
      + content_sha1         = (known after apply)
      + content_sha256       = (known after apply)
      + content_sha512       = (known after apply)
      + directory_permission = "0755"
      + file_permission      = "0644"
      + filename             = "/workspace/crest-p41756-eks-kubeconfig.conf"
      + id                   = (known after apply)
    }

  # module.nfs[0].aws_instance.vm will be updated in-place
  ~ resource "aws_instance" "vm" {
        id                                   = "i-053c21d074aaeb3cc"
      ~ tags                                 = {
            "Name"                   = "crest-p41756-nfs-server-vm"
            "disable_chronos"        = "True"
            "gel_project"            = "PSGEL297"
            "project_name"           = "PSGEL297"
            "resourceowner"          = "crest-p41756"
            "smart_parking_disabled" = "True"
          - "stoptime"               = "30 17 * * 1-5 US/Eastern" -> null
        }
      ~ tags_all                             = {
          - "stoptime"               = "30 17 * * 1-5 US/Eastern" -> null
            # (6 unchanged elements hidden)
        }
        # (39 unchanged attributes hidden)

        # (11 unchanged blocks hidden)
    }

Plan: 6 to add, 2 to change, 0 to destroy.

Changes to Outputs:
  + kube_config                     = (sensitive value)