From 808a03a0d37204fa33a68be0240ec2a591b75370 Mon Sep 17 00:00:00 2001 From: Dan LaManna Date: Wed, 1 Nov 2023 11:34:46 -0400 Subject: [PATCH] Add opensearch deployment to staging --- terraform/modules/spack/opensearch.tf | 91 +++++++++++++++------------ terraform/modules/spack/variables.tf | 10 ++- terraform/production/main.tf | 4 +- terraform/staging/main.tf | 4 +- 4 files changed, 64 insertions(+), 45 deletions(-) diff --git a/terraform/modules/spack/opensearch.tf b/terraform/modules/spack/opensearch.tf index e63190fbd..c4c465e28 100644 --- a/terraform/modules/spack/opensearch.tf +++ b/terraform/modules/spack/opensearch.tf @@ -1,11 +1,14 @@ locals { domain_endpoint_name = "opensearch.${var.deployment_name == "prod" ? "" : "${var.deployment_name}."}spack.io" - cognito_enabled = var.deployment_name == "prod" + cognito_enabled = var.deployment_name == "prod" } -resource "aws_opensearch_domain" "spack" { - count = var.provision_opensearch_cluster ? 1 : 0 +resource "random_password" "opensearch_password" { + length = 64 + special = true +} +resource "aws_opensearch_domain" "spack" { domain_name = "spack${var.deployment_name == "prod" ? "" : "-${var.deployment_name}"}" engine_version = "OpenSearch_1.3" @@ -13,16 +16,25 @@ resource "aws_opensearch_domain" "spack" { advanced_security_options { enabled = true internal_user_database_enabled = true + + dynamic "master_user_options" { + for_each = var.deployment_name == "prod" ? [] : [1] + + content { + master_user_name = "admin" + master_user_password = random_password.opensearch_password.result + } + } } auto_tune_options { - desired_state = "ENABLED" + desired_state = var.deployment_name == "prod" ? "ENABLED" : "DISABLED" rollback_on_disable = "NO_ROLLBACK" } cluster_config { instance_count = 2 - instance_type = "r6g.xlarge.search" + instance_type = var.opensearch_instance_type zone_awareness_config { availability_zone_count = 2 } @@ -40,36 +52,34 @@ resource "aws_opensearch_domain" "spack" { content { enabled = true identity_pool_id = "us-east-1:ff2664d7-a403-42ba-8407-5d90b3eaa948" # TODO: encode this into terraform - role_arn = aws_iam_role.opensearch_cognito_role[0].arn + role_arn = aws_iam_role.opensearch_cognito_role.arn user_pool_id = "us-east-1_k6YnDTVBT" # TODO: encode this into terraform } } domain_endpoint_options { - custom_endpoint_enabled = true - custom_endpoint = local.domain_endpoint_name - custom_endpoint_certificate_arn = aws_acm_certificate.opensearch[0].arn - enforce_https = true - tls_security_policy = "Policy-Min-TLS-1-0-2019-07" + custom_endpoint_enabled = true + custom_endpoint = local.domain_endpoint_name + custom_endpoint_certificate_arn = aws_acm_certificate.opensearch.arn + enforce_https = true + tls_security_policy = "Policy-Min-TLS-1-0-2019-07" } ebs_options { ebs_enabled = true iops = 3000 - volume_size = 500 + volume_size = var.opensearch_volume_size volume_type = "gp3" } encrypt_at_rest { enabled = true - # TODO: encode this KMS resource in terraform - kms_key_id = "arn:aws:kms:us-east-1:588562868276:key/6385d11c-f377-4778-96a6-5da54416b3cb" } log_publishing_options { # TODO: encode cloudwatch in terraform cloudwatch_log_group_arn = "arn:aws:logs:us-east-1:588562868276:log-group:/aws/OpenSearchService/domains/spack/application-logs" - enabled = true + enabled = var.deployment_name == "prod" ? true : false log_type = "ES_APPLICATION_LOGS" } @@ -81,6 +91,8 @@ resource "aws_opensearch_domain" "spack" { automated_snapshot_start_hour = 0 } + access_policies = data.aws_iam_policy_document.main.json + lifecycle { prevent_destroy = true } @@ -90,10 +102,23 @@ resource "aws_opensearch_domain" "spack" { ] } +data "aws_iam_policy_document" "main" { + statement { + effect = "Allow" + + principals { + type = "AWS" + identifiers = ["*"] + } + + actions = ["es:ESHttp*"] + resources = ["arn:aws:es:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:domain/${"spack${var.deployment_name == "prod" ? "" : "-${var.deployment_name}"}"}/*"] + } +} + + # Configure custom domain name resource "aws_acm_certificate" "opensearch" { - count = var.provision_opensearch_cluster ? 1 : 0 - domain_name = local.domain_endpoint_name validation_method = "DNS" @@ -103,13 +128,13 @@ resource "aws_acm_certificate" "opensearch" { } resource "aws_route53_record" "opensearch_validation" { - for_each = var.provision_opensearch_cluster ? { - for dvo in aws_acm_certificate.opensearch[0].domain_validation_options : dvo.domain_name => { + for_each = { + for dvo in aws_acm_certificate.opensearch.domain_validation_options : dvo.domain_name => { name = dvo.resource_record_name record = dvo.resource_record_value type = dvo.resource_record_type } - } : {} + } allow_overwrite = false name = each.value.name @@ -120,17 +145,13 @@ resource "aws_route53_record" "opensearch_validation" { } resource "aws_acm_certificate_validation" "opensearch" { - count = var.provision_opensearch_cluster ? 1 : 0 - - certificate_arn = aws_acm_certificate.opensearch[0].arn + certificate_arn = aws_acm_certificate.opensearch.arn validation_record_fqdns = [for record in aws_route53_record.opensearch_validation : record.fqdn] } resource "aws_route53_record" "opensearch" { - count = var.provision_opensearch_cluster ? 1 : 0 - name = local.domain_endpoint_name - records = [aws_opensearch_domain.spack[0].endpoint] + records = [aws_opensearch_domain.spack.endpoint] ttl = 300 type = "CNAME" zone_id = data.aws_route53_zone.spack_io.zone_id @@ -142,8 +163,6 @@ data "aws_iam_policy" "amazon_opensearch_service_cognito_access" { } resource "aws_iam_role" "opensearch_cognito_role" { - count = var.provision_opensearch_cluster ? 1 : 0 - name = "OpenSearchCognitoAccessRole-${var.deployment_name}" description = "IAM role that gives OpenSearch permissions to configure the Amazon Cognito user and identity pools and use them for authentication." assume_role_policy = jsonencode({ @@ -161,16 +180,12 @@ resource "aws_iam_role" "opensearch_cognito_role" { } resource "aws_iam_role_policy_attachment" "opensearch_congito_role_policy_attach" { - count = var.provision_opensearch_cluster ? 1 : 0 - - role = aws_iam_role.opensearch_cognito_role[0].name + role = aws_iam_role.opensearch_cognito_role.name policy_arn = data.aws_iam_policy.amazon_opensearch_service_cognito_access.arn } # Configure role needed for fluent-bit to post data to OpenSearch resource "aws_iam_role" "fluent_bit_role" { - count = var.provision_opensearch_cluster ? 1 : 0 - name = "FluentBitRole-${var.deployment_name}" description = "IAM role that, when associated with a k8s service account, allows a fluent-bit pod to post logs to OpenSearch." @@ -195,10 +210,8 @@ resource "aws_iam_role" "fluent_bit_role" { } resource "aws_iam_role_policy" "fluent_bit_policy" { - count = var.provision_opensearch_cluster ? 1 : 0 - name = "FluentBitPolicy-${var.deployment_name}" - role = aws_iam_role.fluent_bit_role[0].id + role = aws_iam_role.fluent_bit_role.id policy = jsonencode({ "Version" : "2012-10-17", "Statement" : [ @@ -206,7 +219,7 @@ resource "aws_iam_role_policy" "fluent_bit_policy" { "Action" : [ "es:ESHttp*" ], - "Resource" : aws_opensearch_domain.spack[0].arn, + "Resource" : aws_opensearch_domain.spack.arn, "Effect" : "Allow" } ] @@ -214,8 +227,6 @@ resource "aws_iam_role_policy" "fluent_bit_policy" { } resource "kubectl_manifest" "fluent_bit_service_account" { - count = var.provision_opensearch_cluster ? 1 : 0 - yaml_body = <<-YAML apiVersion: v1 kind: ServiceAccount @@ -223,6 +234,6 @@ resource "kubectl_manifest" "fluent_bit_service_account" { name: fluent-bit namespace: fluent-bit annotations: - eks.amazonaws.com/role-arn: ${aws_iam_role.fluent_bit_role[0].arn} + eks.amazonaws.com/role-arn: ${aws_iam_role.fluent_bit_role.arn} YAML } diff --git a/terraform/modules/spack/variables.tf b/terraform/modules/spack/variables.tf index a19b3b125..35be5654a 100644 --- a/terraform/modules/spack/variables.tf +++ b/terraform/modules/spack/variables.tf @@ -68,10 +68,14 @@ variable "analytics_db_credentials_secret" { type = string } +variable "opensearch_instance_type" { + description = "AWS OpenSearch instance type for the Spack OpenSearch cluster." + type = string +} -variable "provision_opensearch_cluster" { - description = "Whether or not to provision an OpenSearch cluster for this deployment." - type = bool +variable "opensearch_volume_size" { + description = "AWS OpenSearch volume size for the Spack OpenSearch cluster." + type = string } variable "ses_email_domain" { diff --git a/terraform/production/main.tf b/terraform/production/main.tf index 9b5828e0b..3d36128a2 100644 --- a/terraform/production/main.tf +++ b/terraform/production/main.tf @@ -156,7 +156,9 @@ module "production_cluster" { # https://docs.gitlab.com/ee/administration/reference_architectures/3k_users.html#cluster-topology elasticache_instance_class = "cache.m6g.xlarge" - provision_opensearch_cluster = true + opensearch_instance_type = "r6g.xlarge.search" + opensearch_volume_size = 500 + ses_email_domain = "spack.io" } diff --git a/terraform/staging/main.tf b/terraform/staging/main.tf index 44f8a3a1d..d7b63c8f4 100644 --- a/terraform/staging/main.tf +++ b/terraform/staging/main.tf @@ -155,7 +155,9 @@ module "staging_cluster" { elasticache_instance_class = "cache.t4g.small" - provision_opensearch_cluster = false + # Use a cheap OpenSearch instance for staging deployment + opensearch_instance_type = "t3.small.search" + opensearch_volume_size = 100 ses_email_domain = "staging.spack.io" }