Merge pull request spack#693 from spack/opensearch-staging

terraform/modules/spack/opensearch.tf

@@ -1,28 +1,40 @@
 locals {
   domain_endpoint_name = "opensearch.${var.deployment_name == "prod" ? "" : "${var.deployment_name}."}spack.io"
-  cognito_enabled = var.deployment_name == "prod"
+  cognito_enabled      = var.deployment_name == "prod"
 }
 
-resource "aws_opensearch_domain" "spack" {
-  count = var.provision_opensearch_cluster ? 1 : 0
+resource "random_password" "opensearch_password" {
+  length  = 64
+  special = true
+}
 
+resource "aws_opensearch_domain" "spack" {
   domain_name = "spack${var.deployment_name == "prod" ? "" : "-${var.deployment_name}"}"
 
   engine_version = "OpenSearch_1.3"
 
   advanced_security_options {
     enabled                        = true
     internal_user_database_enabled = true
+
+    dynamic "master_user_options" {
+      for_each = var.deployment_name == "prod" ? [] : [1]
+
+      content {
+        master_user_name     = "admin"
+        master_user_password = random_password.opensearch_password.result
+      }
+    }
   }
 
   auto_tune_options {
-    desired_state       = "ENABLED"
+    desired_state       = var.deployment_name == "prod" ? "ENABLED" : "DISABLED"
     rollback_on_disable = "NO_ROLLBACK"
   }
 
   cluster_config {
     instance_count = 2
-    instance_type  = "r6g.xlarge.search"
+    instance_type  = var.opensearch_instance_type
     zone_awareness_config {
       availability_zone_count = 2
     }
@@ -40,36 +52,34 @@ resource "aws_opensearch_domain" "spack" {
     content {
       enabled          = true
       identity_pool_id = "us-east-1:ff2664d7-a403-42ba-8407-5d90b3eaa948" # TODO: encode this into terraform
-      role_arn         = aws_iam_role.opensearch_cognito_role[0].arn
+      role_arn         = aws_iam_role.opensearch_cognito_role.arn
       user_pool_id     = "us-east-1_k6YnDTVBT" # TODO: encode this into terraform
     }
   }
 
   domain_endpoint_options {
-    custom_endpoint_enabled = true
-    custom_endpoint = local.domain_endpoint_name
-    custom_endpoint_certificate_arn = aws_acm_certificate.opensearch[0].arn
-    enforce_https       = true
-    tls_security_policy = "Policy-Min-TLS-1-0-2019-07"
+    custom_endpoint_enabled         = true
+    custom_endpoint                 = local.domain_endpoint_name
+    custom_endpoint_certificate_arn = aws_acm_certificate.opensearch.arn
+    enforce_https                   = true
+    tls_security_policy             = "Policy-Min-TLS-1-0-2019-07"
   }
 
   ebs_options {
     ebs_enabled = true
     iops        = 3000
-    volume_size = 500
+    volume_size = var.opensearch_volume_size
     volume_type = "gp3"
   }
 
   encrypt_at_rest {
     enabled = true
-    # TODO: encode this KMS resource in terraform
-    kms_key_id = "arn:aws:kms:us-east-1:588562868276:key/6385d11c-f377-4778-96a6-5da54416b3cb"
   }
 
   log_publishing_options {
     # TODO: encode cloudwatch in terraform
     cloudwatch_log_group_arn = "arn:aws:logs:us-east-1:588562868276:log-group:/aws/OpenSearchService/domains/spack/application-logs"
-    enabled                  = true
+    enabled                  = var.deployment_name == "prod" ? true : false
     log_type                 = "ES_APPLICATION_LOGS"
   }
 
@@ -81,6 +91,8 @@ resource "aws_opensearch_domain" "spack" {
     automated_snapshot_start_hour = 0
   }
 
+  access_policies = data.aws_iam_policy_document.main.json
+
   lifecycle {
     prevent_destroy = true
   }
@@ -90,10 +102,23 @@ resource "aws_opensearch_domain" "spack" {
   ]
 }
 
+data "aws_iam_policy_document" "main" {
+  statement {
+    effect = "Allow"
+
+    principals {
+      type        = "AWS"
+      identifiers = ["*"]
+    }
+
+    actions   = ["es:ESHttp*"]
+    resources = ["arn:aws:es:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:domain/${"spack${var.deployment_name == "prod" ? "" : "-${var.deployment_name}"}"}/*"]
+  }
+}
+
+
 # Configure custom domain name
 resource "aws_acm_certificate" "opensearch" {
-  count = var.provision_opensearch_cluster ? 1 : 0
-
   domain_name       = local.domain_endpoint_name
   validation_method = "DNS"
 
@@ -103,13 +128,13 @@ resource "aws_acm_certificate" "opensearch" {
 }
 
 resource "aws_route53_record" "opensearch_validation" {
-  for_each = var.provision_opensearch_cluster ? {
-    for dvo in aws_acm_certificate.opensearch[0].domain_validation_options : dvo.domain_name => {
+  for_each = {
+    for dvo in aws_acm_certificate.opensearch.domain_validation_options : dvo.domain_name => {
       name   = dvo.resource_record_name
       record = dvo.resource_record_value
       type   = dvo.resource_record_type
     }
-  } : {}
+  }
 
   allow_overwrite = false
   name            = each.value.name
@@ -120,17 +145,13 @@ resource "aws_route53_record" "opensearch_validation" {
 }
 
 resource "aws_acm_certificate_validation" "opensearch" {
-  count = var.provision_opensearch_cluster ? 1 : 0
-
-  certificate_arn         = aws_acm_certificate.opensearch[0].arn
+  certificate_arn         = aws_acm_certificate.opensearch.arn
   validation_record_fqdns = [for record in aws_route53_record.opensearch_validation : record.fqdn]
 }
 
 resource "aws_route53_record" "opensearch" {
-  count = var.provision_opensearch_cluster ? 1 : 0
-
   name    = local.domain_endpoint_name
-  records = [aws_opensearch_domain.spack[0].endpoint]
+  records = [aws_opensearch_domain.spack.endpoint]
   ttl     = 300
   type    = "CNAME"
   zone_id = data.aws_route53_zone.spack_io.zone_id
@@ -142,8 +163,6 @@ data "aws_iam_policy" "amazon_opensearch_service_cognito_access" {
 }
 
 resource "aws_iam_role" "opensearch_cognito_role" {
-  count = var.provision_opensearch_cluster ? 1 : 0
-
   name        = "OpenSearchCognitoAccessRole-${var.deployment_name}"
   description = "IAM role that gives OpenSearch permissions to configure the Amazon Cognito user and identity pools and use them for authentication."
   assume_role_policy = jsonencode({
@@ -161,16 +180,12 @@ resource "aws_iam_role" "opensearch_cognito_role" {
 }
 
 resource "aws_iam_role_policy_attachment" "opensearch_congito_role_policy_attach" {
-  count = var.provision_opensearch_cluster ? 1 : 0
-
-  role       = aws_iam_role.opensearch_cognito_role[0].name
+  role       = aws_iam_role.opensearch_cognito_role.name
   policy_arn = data.aws_iam_policy.amazon_opensearch_service_cognito_access.arn
 }
 
 # Configure role needed for fluent-bit to post data to OpenSearch
 resource "aws_iam_role" "fluent_bit_role" {
-  count = var.provision_opensearch_cluster ? 1 : 0
-
   name        = "FluentBitRole-${var.deployment_name}"
   description = "IAM role that, when associated with a k8s service account, allows a fluent-bit pod to post logs to OpenSearch."
 
@@ -195,34 +210,30 @@ resource "aws_iam_role" "fluent_bit_role" {
 }
 
 resource "aws_iam_role_policy" "fluent_bit_policy" {
-  count = var.provision_opensearch_cluster ? 1 : 0
-
   name = "FluentBitPolicy-${var.deployment_name}"
-  role = aws_iam_role.fluent_bit_role[0].id
+  role = aws_iam_role.fluent_bit_role.id
   policy = jsonencode({
     "Version" : "2012-10-17",
     "Statement" : [
       {
         "Action" : [
           "es:ESHttp*"
         ],
-        "Resource" : aws_opensearch_domain.spack[0].arn,
+        "Resource" : aws_opensearch_domain.spack.arn,
         "Effect" : "Allow"
       }
     ]
   })
 }
 
 resource "kubectl_manifest" "fluent_bit_service_account" {
-  count = var.provision_opensearch_cluster ? 1 : 0
-
   yaml_body = <<-YAML
     apiVersion: v1
     kind: ServiceAccount
     metadata:
       name: fluent-bit
       namespace: fluent-bit
       annotations:
-        eks.amazonaws.com/role-arn: ${aws_iam_role.fluent_bit_role[0].arn}
+        eks.amazonaws.com/role-arn: ${aws_iam_role.fluent_bit_role.arn}
   YAML
 }

terraform/modules/spack/variables.tf

@@ -68,10 +68,14 @@ variable "analytics_db_credentials_secret" {
   type        = string
 }
 
+variable "opensearch_instance_type" {
+  description = "AWS OpenSearch instance type for the Spack OpenSearch cluster."
+  type        = string
+}
 
-variable "provision_opensearch_cluster" {
-  description = "Whether or not to provision an OpenSearch cluster for this deployment."
-  type        = bool
+variable "opensearch_volume_size" {
+  description = "AWS OpenSearch volume size for the Spack OpenSearch cluster."
+  type        = string
 }
 
 variable "ses_email_domain" {

terraform/production/main.tf

@@ -156,7 +156,9 @@ module "production_cluster" {
   # https://docs.gitlab.com/ee/administration/reference_architectures/3k_users.html#cluster-topology
   elasticache_instance_class = "cache.m6g.xlarge"
 
-  provision_opensearch_cluster = true
+  opensearch_instance_type = "r6g.xlarge.search"
+  opensearch_volume_size   = 500
+
 
   ses_email_domain = "spack.io"
 }

terraform/staging/main.tf

@@ -155,7 +155,9 @@ module "staging_cluster" {
 
   elasticache_instance_class = "cache.t4g.small"
 
-  provision_opensearch_cluster = false
+  # Use a cheap OpenSearch instance for staging deployment
+  opensearch_instance_type = "t3.small.search"
+  opensearch_volume_size   = 100
 
   ses_email_domain = "staging.spack.io"
 }