updates for deployment via Unity Marketplace

common/aws.tf

@@ -2,7 +2,7 @@ locals {
   cost_tags = {
     ServiceArea = "аds"
     Proj = "${var.project}"
-    Venue = "${var.venue_prefix}${var.venue}"
+    Venue = "${var.deployment_name}-${var.venue}"
     Component = "${var.component_cost_name}"
     CreatedBy = "ads"
     Env = "${var.resource_prefix}"

common/variables.tf

@@ -9,7 +9,7 @@ variable "venue" {
   type        = string
 }
 
-variable "venue_prefix" {
+variable "deployment_name" {
   description = "Optional string to place before the venue name in resource names"
   type        = string
   default     = ""

dev_env/cognito/cognito_client.tf

@@ -3,7 +3,7 @@ data "aws_cognito_user_pools" "unity_user_pool" {
 }
 
 resource "aws_cognito_user_pool_client" "jupyter_cognito_client" {
-  name          = "${var.resource_prefix}-jupyter-${var.venue_prefix}${var.venue}-client"
+  name          = "${var.resource_prefix}-jupyter-${var.deployment_name}-${var.venue}-client"
   user_pool_id  = tolist(data.aws_cognito_user_pools.unity_user_pool.ids)[0]
 
   callback_urls = var.jupyter_base_url != null ? ["${var.jupyter_base_url}/${var.jupyter_base_path}/hub/oauth_callback"] : null

dev_env/jupyterhub/aws.tf

@@ -0,0 +1,17 @@
+locals {
+  cost_tags = {
+    ServiceArea = "аds"
+    Proj = "${var.project}"
+    Venue = "${var.deployment_name}-${var.venue}"
+    Component = "${var.component_cost_name}"
+    CreatedBy = "ads"
+    Env = "${var.resource_prefix}"
+    Stack = "${var.component_cost_name}"
+  }
+}
+
+provider "aws" {
+  default_tags {
+    tags = local.cost_tags
+  }
+}

dev_env/jupyterhub/common_variables.tf

@@ -0,0 +1,38 @@
+variable "project" {
+  description = "The name of the project matching the /unity/<{project>/<venue</project-name SSM parameter"
+  type        = string
+  default     = "unity"
+}
+
+variable "venue" {
+  description = "The name of the unity venue matching the /unity/<project>/<venue>/venue-name SSM parameter"
+  type        = string
+}
+
+variable "deployment_name" {
+  description = "Optional string to place before the venue name in resource names"
+  type        = string
+  default     = ""
+}
+
+variable "resource_prefix" {
+  description = "String used at the beginning of the names for all resources to identify them according to the UADS subsystem"
+  type        = string
+  default     = "uads"
+}
+
+variable "efs_identifier" {
+  description = "EFS file system to connect Jupyter shared storage with"
+  type        = string
+  # Example value:uads-development-efs-fs"
+}
+
+variable "installprefix" {
+  description = "Installation prefix"
+  type        = string
+  default     = ""
+}
+
+variable "tags" {
+  type = map(string)
+}

dev_env/jupyterhub/csi_driver.tf

@@ -23,6 +23,11 @@ module "ebs_csi_irsa_role" {
     "eks_addon" = "ebs-csi"
     "terraform" = "true"
   }
+
+  depends_on = [
+    module.eks,
+    null_resource.eks_post_deployment_actions
+  ]
 }
 
 resource "aws_eks_addon" "ebs-csi" {
@@ -36,7 +41,11 @@ resource "aws_eks_addon" "ebs-csi" {
     "terraform" = "true"
   }
 
-  depends_on = [ module.eks ]
+  depends_on = [
+    module.eks,
+    module.ebs_csi_irsa_role,
+    null_resource.eks_post_deployment_actions
+  ]
 }
 
 ################
@@ -65,6 +74,11 @@ module "efs_csi_irsa_role" {
     "eks_addon" = "efs-csi"
     "terraform" = "true"
   }
+
+  depends_on = [
+    module.eks,
+    null_resource.eks_post_deployment_actions
+  ]
 }
 
 resource "aws_eks_addon" "efs-csi" {
@@ -76,5 +90,9 @@ resource "aws_eks_addon" "efs-csi" {
     "terraform" = "true"
   }
 
-  depends_on = [ module.eks ]
+  depends_on = [
+    module.eks,
+    module.efs_csi_irsa_role,
+    null_resource.eks_post_deployment_actions
+  ]
 }

dev_env/jupyterhub/ebs.tf

@@ -14,4 +14,8 @@ resource "kubernetes_storage_class" "ebs_storage_class" {
     # How to add custom tags to ebs-csi deployed volumes
     # https://github.com/kubernetes-sigs/aws-ebs-csi-driver/blob/master/docs/tagging.md
     { for i, k in keys(local.cost_tags) : "tagSpecification_${i}" => "${k}=${local.cost_tags[k]}" })
+
+  depends_on = [
+    aws_eks_addon.ebs-csi
+  ]
 }

dev_env/jupyterhub/efs.tf

@@ -1,5 +1,5 @@
 resource "aws_security_group" "dev_support_efs_jupyter_sg" {
-   name = "${var.resource_prefix}-${var.venue_prefix}${var.venue}-efs-jupyter-sg"
+   name = "${var.resource_prefix}-${var.deployment_name}-${var.venue}-efs-jupyter-sg"
    description= "Allows inbound EFS traffic from Jupyter cluster"
    vpc_id = data.aws_ssm_parameter.vpc_id.value
 
@@ -38,10 +38,14 @@ resource "kubernetes_storage_class" "efs_storage_class" {
     name = "efs"
   }
   storage_provisioner = "efs.csi.aws.com"
-  reclaim_policy      = "Delete"
+  reclaim_policy      = "Retain"
 
   parameters = {
   }
+
+  depends_on = [
+    aws_eks_addon.efs-csi
+  ]
 }
 
 # Documentation on how to set up volume_handle:
@@ -50,7 +54,7 @@ resource "kubernetes_storage_class" "efs_storage_class" {
 # https://kubernetes.io/docs/concepts/storage/persistent-volumes/
 resource "kubernetes_persistent_volume" "dev_support_shared_volume" {
   metadata {
-    name = "${var.resource_prefix}-${var.venue_prefix}${var.venue}-dev-data"
+    name = "${var.resource_prefix}-${var.deployment_name}-${var.venue}-dev-data"
   }
 
   spec {
@@ -61,7 +65,7 @@ resource "kubernetes_persistent_volume" "dev_support_shared_volume" {
       storage = "100Gi"
     }
 
-    persistent_volume_reclaim_policy = "Delete"
+    persistent_volume_reclaim_policy = "Retain"
 
     persistent_volume_source {
       csi {
@@ -70,21 +74,22 @@ resource "kubernetes_persistent_volume" "dev_support_shared_volume" {
       }
     }
 
-    mount_options = [ "iam" ]
+    mount_options = [ "tls", "iam" ]
   }
 
   # Prevents a cycle with eks_cluster.jupyter_hub
   depends_on = [ 
     kubernetes_storage_class.efs_storage_class,
     aws_efs_mount_target.dev_support_efs_mt_1,
     aws_efs_mount_target.dev_support_efs_mt_2,
-    aws_eks_addon.efs-csi
+    aws_eks_addon.efs-csi,
+    module.efs_csi_irsa_role
   ]
 }
 
 resource "kubernetes_persistent_volume_claim" "dev_support_shared_volume_claim" {
   metadata {
-    name = "${var.resource_prefix}-${var.venue_prefix}${var.venue}-dev-data"
+    name = "${var.resource_prefix}-${var.deployment_name}-${var.venue}-dev-data"
     namespace = helm_release.jupyter_helm.namespace
   }
 
@@ -105,6 +110,7 @@ resource "kubernetes_persistent_volume_claim" "dev_support_shared_volume_claim"
   # Prevents a cycle with eks_cluster.jupyter_hub
   depends_on = [ 
     kubernetes_persistent_volume.dev_support_shared_volume,
-    aws_eks_addon.efs-csi
+    aws_eks_addon.efs-csi,
+    module.efs_csi_irsa_role
   ]
 }

dev_env/jupyterhub/eks_cluster.tf

@@ -8,43 +8,44 @@ data "aws_ssm_parameter" "ami_id" {
   name = "/mcp/amis/aml2-eks-1-30"
 }
 
-#data "external" "current_ip" {
-#  program = ["./get_ip.sh"]
-#}
-#
-#resource "aws_security_group" "mc_instance_k8s_api_access" {
-#  name        = "${var.resource_prefix}-${var.venue_prefix}${var.venue}-mc-sg"
-#  description = "Security group to allow access to K8s API from MC instance"
-#
-#  vpc_id = data.aws_ssm_parameter.vpc_id.value
-#
-#  tags = {
-#    Name = "${var.resource_prefix}-${var.venue_prefix}${var.venue}-mc-sg"
-#  }
-#
-#  # Allow all outbound traffic.
-#  egress {
-#    from_port   = 0
-#    to_port     = 0
-#    protocol    = "-1"
-#    cidr_blocks = ["0.0.0.0/0"]
-#  }
-#
-#  # Allow from variable defined input port
-#  ingress {
-#    from_port   = 0
-#    to_port     = 0
-#    protocol    = "-1"
-#    cidr_blocks = ["${data.external.current_ip.result.ip}/32"]
-#  }
-#
-#}
+data "external" "current_ip" {
+  program = ["${path.module}/get_ip.sh"]
+}
+
+resource "aws_security_group" "mc_instance_k8s_api_access" {
+  name        = "${var.resource_prefix}-${var.deployment_name}-${var.venue}-mc-sg"
+  description = "Security group to allow access to K8s API from MC instance"
+
+  vpc_id = data.aws_ssm_parameter.vpc_id.value
+
+  tags = {
+    Name = "${var.resource_prefix}-${var.deployment_name}-${var.venue}-mc-sg"
+  }
+
+  # Allow all outbound traffic.
+  egress {
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  # Allow from variable defined input port
+  ingress {
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["${data.external.current_ip.result.ip}/32"]
+  }
+
+}
 
 module "eks" {
   source  = "terraform-aws-modules/eks/aws"
-  version = "~> 20.0"
+  # pin to 20.16.0 because unity-proxy is pinned to hashicorp/aws 5.47.0
+  version = "20.16.0"
 
-  cluster_name    = "${var.resource_prefix}-${var.venue_prefix}${var.venue}-jupyter"
+  cluster_name    = "${var.resource_prefix}-${var.deployment_name}-${var.venue}-jupyter"
   cluster_version = "1.30"
 
   cluster_addons = {
@@ -66,19 +67,19 @@ module "eks" {
   enable_irsa = true
 
   create_iam_role = true
-  iam_role_name = "Unity-ADS-${var.venue_prefix}${var.venue}-EKSClusterRole"
+  iam_role_name = "Unity-ADS-${var.deployment_name}-${var.venue}-EKSClusterRole"
   iam_role_permissions_boundary = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:policy/mcp-tenantOperator-AMI-APIG"
 
   cluster_endpoint_public_access = true
   cluster_endpoint_private_access = true
   enable_cluster_creator_admin_permissions = true
 
   # add MC instance access to K8s API
-  #cluster_additional_security_group_ids = [aws_security_group.mc_instance_k8s_api_access.id]
+  cluster_additional_security_group_ids = [aws_security_group.mc_instance_k8s_api_access.id]
 
   eks_managed_node_group_defaults = {
     create_iam_role = true
-    iam_role_name = "Unity-ADS-${var.venue_prefix}${var.venue}-EKSNodeRole"
+    iam_role_name = "Unity-ADS-${var.deployment_name}-${var.venue}-EKSNodeRole"
     iam_role_permissions_boundary = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:policy/mcp-tenantOperator-AMI-APIG"
 
     ami_id          = data.aws_ssm_parameter.ami_id.value
@@ -100,6 +101,19 @@ module "eks" {
       min_size       = var.eks_node_min_size
       max_size       = var.eks_node_max_size
       desired_size   = var.eks_node_desired_size
+      block_device_mappings = {
+        xvda = {
+          device_name = "/dev/xvda"
+          ebs = {
+            volume_size           = 75
+            volume_type           = "gp3"
+            iops                  = 3000
+            throughput            = 150
+            encrypted             = true
+            delete_on_termination = true
+          }
+        }
+      }
     }
   }
 
@@ -108,7 +122,7 @@ module "eks" {
 resource "null_resource" "eks_post_deployment_actions" {
   depends_on = [module.eks]
   provisioner "local-exec" {
-    command = "./eks_post_deployment_actions.sh ${data.aws_region.current.name} ${module.eks.cluster_name}"
+    command = "${path.module}/eks_post_deployment_actions.sh ${data.aws_region.current.name} ${module.eks.cluster_name}"
   }
 }
 

dev_env/jupyterhub/eks_pre_destroy_actions.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+set -e
+
+# remove finalizers from EFS PVCs
+kubectl get pvc --all-namespaces --no-headers | grep ' efs ' | while read line; do
+  namespace=$(echo $line | awk '{print $1}')
+  pvc_name=$(echo $line | awk '{print $2}')
+  kubectl patch pvc ${pvc_name} -p '{"metadata":{"finalizers":null}}' -n ${namespace}
+done

dev_env/jupyterhub/frontend.tf

@@ -36,7 +36,7 @@ module "frontend" {
 
   project = var.project
   venue = var.venue
-  venue_prefix = var.venue_prefix
+  deployment_name = var.deployment_name
   resource_prefix = var.resource_prefix
   load_balancer_port = local.load_balancer_port
   jupyter_proxy_port = var.jupyter_proxy_port

dev_env/jupyterhub/get_ip.sh

@@ -1,4 +1,6 @@
 #!/bin/bash
-set -e
-IP=$(curl -s 'http://169.254.169.254/latest/meta-data/local-ipv4')
-jq -n --arg ip "$IP" '{"ip":$ip}'
+IP=$(curl --connect-timeout 1 -s 'http://169.254.169.254/latest/meta-data/local-ipv4')
+if [ "$IP" = "" ]; then
+  IP=127.0.0.1
+fi
+echo "{\"ip\": \"${IP}\"}"

dev_env/jupyterhub/health_check.tf

@@ -7,4 +7,9 @@ resource "aws_ssm_parameter" "jupyter_health_url" {
     landingPageUrl = "${module.frontend.jupyter_base_url}/${module.frontend.jupyter_base_path}/"
     componentName  = "Jupyterhub"
   })
+
+  provisioner "local-exec" {
+    when = destroy
+    command = "${path.module}/eks_pre_destroy_actions.sh"
+  }
 }