diff --git a/aws-redis.yml b/aws-redis.yml index 39eeaa337..846e6aeae 100644 --- a/aws-redis.yml +++ b/aws-redis.yml @@ -227,6 +227,52 @@ provision: List of EC2 availability zones in which the nodes will be created. The first item in the list will be the primary node. Number of entries must equal to node_count. + - field_name: logs_slow_log_enabled + type: boolean + default: false + details: | + Enable the streaming of Redis Slow Log to CloudWatch. Slow Log is supported for Redis replication groups using version 6.0 onward. + - field_name: logs_slow_log_loggroup_retention_in_days + type: number + default: 0 + details: | + Specifies the number of days you want to retain log events in the specified log group. + If 0 is specified, the events in the log group are always retained and never expire. + When specifying `logs_slow_log_loggroup_retention_in_days`, `logs_slow_log_enabled` needs to be set to true. + For more information, see + https://docs.aws.amazon.com/AmazonCloudWatchLogs/latest/APIReference/API_PutRetentionPolicy.html#API_PutRetentionPolicy_RequestSyntax + - field_name: logs_slow_log_loggroup_kms_key_id + type: string + default: "" + details: | + The ARN for the KMS key to encrypt Slow logs CloudWatch logs. + When specifying `logs_slow_log_loggroup_kms_key_id`, `logs_slow_log_enabled` needs to be set to true. + If omitted, CloudWatch default encryption will apply. + For information on CloudWatch log data encryption and how to configure a KMS key, see + https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/encrypt-log-data-kms.html + - field_name: logs_engine_log_enabled + type: boolean + default: false + details: | + Enable the streaming of Redis Engine logs to CloudWatch. Engine Log is supported for Redis replication groups using version 6.2 onward. + - field_name: logs_engine_log_loggroup_retention_in_days + type: number + default: 0 + details: | + Specifies the number of days you want to retain log events in the specified log group. + If 0 is specified, the events in the log group are always retained and never expire. + When specifying `logs_engine_log_loggroup_retention_in_days`, `logs_engine_log_enabled` needs to be set to true. + For more information, see + https://docs.aws.amazon.com/AmazonCloudWatchLogs/latest/APIReference/API_PutRetentionPolicy.html#API_PutRetentionPolicy_RequestSyntax + - field_name: logs_engine_log_loggroup_kms_key_id + type: string + default: "" + details: | + The ARN for the KMS key to encrypt Engine Log CloudWatch logs. + When specifying `logs_engine_log_loggroup_kms_key_id`, `logs_engine_log_enabled` needs to be set to true. + If omitted, CloudWatch default encryption will apply. + For information on CloudWatch log data encryption and how to configure a KMS key, see + https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/encrypt-log-data-kms.html computed_inputs: - name: labels default: ${json.marshal(request.default_labels)} @@ -273,7 +319,7 @@ examples: provision_params: {"region": "us-east-1", "multi_az_enabled": false} bind_params: {} - name: with-custom-node_type - description: Create a Redis instance with a custom node_type + description: Create a Redis instance with a custom node_type and slow logs enabled plan_id: 2deb6c13-7ea1-4bad-a519-0ac9600e9a29 - provision_params: {"node_type": "cache.t2.micro", "multi_az_enabled": false} + provision_params: {"node_type": "cache.t2.micro", "multi_az_enabled": false, "logs_slow_log_enabled": true} bind_params: {} diff --git a/integration-tests/redis_test.go b/integration-tests/redis_test.go index 530e76141..e171a92c7 100644 --- a/integration-tests/redis_test.go +++ b/integration-tests/redis_test.go @@ -275,37 +275,49 @@ var _ = Describe("Redis", Label("Redis"), func() { HaveKeyWithValue("backup_end_min", BeNil()), HaveKeyWithValue("parameter_group_name", ""), HaveKeyWithValue("preferred_azs", BeEmpty()), + HaveKeyWithValue("logs_slow_log_enabled", BeFalse()), + HaveKeyWithValue("logs_slow_log_loggroup_retention_in_days", BeNumerically("==", 0)), + HaveKeyWithValue("logs_slow_log_loggroup_kms_key_id", BeEmpty()), + HaveKeyWithValue("logs_engine_log_enabled", BeFalse()), + HaveKeyWithValue("logs_engine_log_loggroup_retention_in_days", BeNumerically("==", 0)), + HaveKeyWithValue("logs_engine_log_loggroup_kms_key_id", BeEmpty()), )) }) It("should allow properties to be set on provision", func() { _, err := broker.Provision(redisServiceName, redisCustomPlanName, map[string]any{ - "redis_version": "6.x", - "instance_name": "some-valid-instance-name", - "region": "some-valid-region", - "aws_vpc_id": "some-valid-aws-vpc-id", - "elasticache_subnet_group": "some-valid-elasticache-subnet-group", - "elasticache_vpc_security_group_ids": "some-valid-elasticache-vpc-security-group-ids", - "aws_access_key_id": "some-valid-aws-access-key-id", - "aws_secret_access_key": "some-valid-aws-secret-access-key", - "at_rest_encryption_enabled": false, - "kms_key_id": "fake-encryption-at-rest-key", - "maintenance_day": "Mon", - "maintenance_start_hour": "03", - "maintenance_start_min": "45", - "maintenance_end_hour": "10", - "maintenance_end_min": "15", - "automatic_failover_enabled": false, - "multi_az_enabled": false, - "backup_retention_limit": 32, - "final_backup_identifier": "tortoise", - "backup_name": "turtle", - "backup_start_hour": "04", - "backup_start_min": "15", - "backup_end_hour": "11", - "backup_end_min": "30", - "parameter_group_name": "fake-param-group-name", - "preferred_azs": []string{"fake-az1", "fake-az2"}, + "redis_version": "6.x", + "instance_name": "some-valid-instance-name", + "region": "some-valid-region", + "aws_vpc_id": "some-valid-aws-vpc-id", + "elasticache_subnet_group": "some-valid-elasticache-subnet-group", + "elasticache_vpc_security_group_ids": "some-valid-elasticache-vpc-security-group-ids", + "aws_access_key_id": "some-valid-aws-access-key-id", + "aws_secret_access_key": "some-valid-aws-secret-access-key", + "at_rest_encryption_enabled": false, + "kms_key_id": "fake-encryption-at-rest-key", + "maintenance_day": "Mon", + "maintenance_start_hour": "03", + "maintenance_start_min": "45", + "maintenance_end_hour": "10", + "maintenance_end_min": "15", + "automatic_failover_enabled": false, + "multi_az_enabled": false, + "backup_retention_limit": 32, + "final_backup_identifier": "tortoise", + "backup_name": "turtle", + "backup_start_hour": "04", + "backup_start_min": "15", + "backup_end_hour": "11", + "backup_end_min": "30", + "parameter_group_name": "fake-param-group-name", + "preferred_azs": []string{"fake-az1", "fake-az2"}, + "logs_slow_log_enabled": true, + "logs_slow_log_loggroup_retention_in_days": 1, + "logs_slow_log_loggroup_kms_key_id": "slow-log-key", + "logs_engine_log_enabled": true, + "logs_engine_log_loggroup_retention_in_days": 2, + "logs_engine_log_loggroup_kms_key_id": "engine-log-key", }) Expect(err).NotTo(HaveOccurred()) @@ -338,6 +350,12 @@ var _ = Describe("Redis", Label("Redis"), func() { HaveKeyWithValue("backup_end_min", "30"), HaveKeyWithValue("parameter_group_name", "fake-param-group-name"), HaveKeyWithValue("preferred_azs", ConsistOf("fake-az1", "fake-az2")), + HaveKeyWithValue("logs_slow_log_enabled", BeTrue()), + HaveKeyWithValue("logs_slow_log_loggroup_retention_in_days", BeNumerically("==", 1)), + HaveKeyWithValue("logs_slow_log_loggroup_kms_key_id", "slow-log-key"), + HaveKeyWithValue("logs_engine_log_enabled", BeTrue()), + HaveKeyWithValue("logs_engine_log_loggroup_retention_in_days", BeNumerically("==", 2)), + HaveKeyWithValue("logs_engine_log_loggroup_kms_key_id", "engine-log-key"), ), ) }) @@ -429,6 +447,12 @@ var _ = Describe("Redis", Label("Redis"), func() { Entry("backup_end_hour", "backup_end_hour", "05"), Entry("backup_end_min", "backup_end_min", "30"), Entry("parameter_group_name", "parameter_group_name", "fake-param-group-name"), + Entry("logs_slow_log_enabled", "logs_slow_log_enabled", true), + Entry("logs_slow_log_loggroup_retention_in_days", "logs_slow_log_loggroup_retention_in_days", 4), + Entry("logs_slow_log_loggroup_kms_key_id", "logs_slow_log_loggroup_kms_key_id", "slow-log-key-2"), + Entry("logs_engine_log_enabled", "logs_engine_log_enabled", true), + Entry("logs_engine_log_loggroup_retention_in_days", "logs_engine_log_loggroup_retention_in_days", 5), + Entry("logs_engine_log_loggroup_kms_key_id", "logs_engine_log_loggroup_kms_key_id", "engine-log-key-2"), ) }) diff --git a/terraform-tests/redis_test.go b/terraform-tests/redis_test.go index 27d70d46e..f1d30d973 100644 --- a/terraform-tests/redis_test.go +++ b/terraform-tests/redis_test.go @@ -20,37 +20,43 @@ var _ = Describe("Redis", Label("redis-terraform"), Ordered, func() { ) defaultVars := map[string]any{ - "cache_size": nil, - "redis_version": "6.0", - "instance_name": "csb-redis-test", - "labels": map[string]any{"key1": "some-redis-value"}, - "node_type": "cache.t3.medium", - "node_count": 2, - "elasticache_subnet_group": "", - "elasticache_vpc_security_group_ids": "", - "region": "us-west-2", - "aws_access_key_id": awsAccessKeyID, - "aws_secret_access_key": awsSecretAccessKey, - "aws_vpc_id": awsVPCID, - "at_rest_encryption_enabled": true, - "kms_key_id": "fake-encryption-at-rest-key", - "maintenance_end_hour": nil, - "maintenance_start_hour": nil, - "maintenance_end_min": nil, - "maintenance_start_min": nil, - "maintenance_day": nil, - "data_tiering_enabled": false, - "automatic_failover_enabled": true, - "multi_az_enabled": true, - "backup_retention_limit": 12, - "final_backup_identifier": "tortoise", - "backup_name": "turtle", - "backup_end_hour": nil, - "backup_start_hour": nil, - "backup_end_min": nil, - "backup_start_min": nil, - "parameter_group_name": "fake-param-group-name", - "preferred_azs": nil, + "cache_size": nil, + "redis_version": "6.0", + "instance_name": "csb-redis-test", + "labels": map[string]any{"key1": "some-redis-value"}, + "node_type": "cache.t3.medium", + "node_count": 2, + "elasticache_subnet_group": "", + "elasticache_vpc_security_group_ids": "", + "region": "us-west-2", + "aws_access_key_id": awsAccessKeyID, + "aws_secret_access_key": awsSecretAccessKey, + "aws_vpc_id": awsVPCID, + "at_rest_encryption_enabled": true, + "kms_key_id": "fake-encryption-at-rest-key", + "maintenance_end_hour": nil, + "maintenance_start_hour": nil, + "maintenance_end_min": nil, + "maintenance_start_min": nil, + "maintenance_day": nil, + "data_tiering_enabled": false, + "automatic_failover_enabled": true, + "multi_az_enabled": true, + "backup_retention_limit": 12, + "final_backup_identifier": "tortoise", + "backup_name": "turtle", + "backup_end_hour": nil, + "backup_start_hour": nil, + "backup_end_min": nil, + "backup_start_min": nil, + "parameter_group_name": "fake-param-group-name", + "preferred_azs": nil, + "logs_slow_log_loggroup_kms_key_id": "", + "logs_slow_log_loggroup_retention_in_days": 0, + "logs_slow_log_enabled": false, + "logs_engine_log_loggroup_kms_key_id": "", + "logs_engine_log_loggroup_retention_in_days": 0, + "logs_engine_log_enabled": false, } BeforeAll(func() { @@ -266,6 +272,143 @@ var _ = Describe("Redis", Label("redis-terraform"), Ordered, func() { }) }) + Context("slow_log is enabled", func() { + BeforeAll(func() { + plan = ShowPlan(terraformProvisionDir, buildVars(defaultVars, map[string]any{ + "logs_slow_log_loggroup_kms_key_id": "", + "logs_slow_log_loggroup_retention_in_days": 0, + "logs_slow_log_enabled": true, + })) + }) + + It("should create a log group for slow log", func() { + expectedResources := []string{"aws_cloudwatch_log_group"} + expectedResources = append(expectedResources, getExpectedResources()...) + Expect(ResourceChangesTypes(plan)).To(ConsistOf(expectedResources)) + }) + + It("should assign that log group to the replication group", func() { + Expect(AfterValuesForType(plan, resource)).To(MatchKeys(IgnoreExtras, Keys{ + "log_delivery_configuration": ConsistOf(MatchAllKeys(Keys{ + "destination": Equal("/aws/elasticache/cluster/csb-redis-test/slow-log"), + "destination_type": Equal("cloudwatch-logs"), + "log_format": Equal("json"), + "log_type": Equal("slow-log"), + })), + })) + }) + + Context("slow_log loggroup configuration", func() { + kmsKeyId := "" + retentionInDays := 0 + JustBeforeEach(func() { + plan = ShowPlan(terraformProvisionDir, buildVars(defaultVars, map[string]any{ + "logs_slow_log_loggroup_kms_key_id": kmsKeyId, + "logs_slow_log_loggroup_retention_in_days": retentionInDays, + "logs_slow_log_enabled": true, + })) + }) + + It("should configure the log group with default values", func() { + slowLogResource := "aws_cloudwatch_log_group" + Expect(AfterValuesForType(plan, slowLogResource)).To(MatchKeys(IgnoreExtras, Keys{ + "name": Equal("/aws/elasticache/cluster/csb-redis-test/slow-log"), + "retention_in_days": BeNumerically("==", retentionInDays), + "kms_key_id": BeNil(), + "skip_destroy": BeFalse(), + "tags": HaveKeyWithValue("key1", "some-redis-value"), + })) + }) + + Context("providing explicit values", func() { + BeforeEach(func() { + kmsKeyId = "test-kms-key" + retentionInDays = 180 + }) + + It("should configure the passed values", func() { + slowLogResource := "aws_cloudwatch_log_group" + Expect(AfterValuesForType(plan, slowLogResource)).To(MatchKeys(IgnoreExtras, Keys{ + "name": Equal("/aws/elasticache/cluster/csb-redis-test/slow-log"), + "retention_in_days": BeNumerically("==", retentionInDays), + "kms_key_id": Equal(kmsKeyId), + "skip_destroy": BeFalse(), + "tags": HaveKeyWithValue("key1", "some-redis-value"), + })) + }) + }) + }) + + }) + + Context("engine_log is enabled", func() { + BeforeAll(func() { + plan = ShowPlan(terraformProvisionDir, buildVars(defaultVars, map[string]any{ + "logs_engine_log_loggroup_kms_key_id": "", + "logs_engine_log_loggroup_retention_in_days": 0, + "logs_engine_log_enabled": true, + })) + }) + + It("should create a log group for engine log", func() { + expectedResources := []string{"aws_cloudwatch_log_group"} + expectedResources = append(expectedResources, getExpectedResources()...) + Expect(ResourceChangesTypes(plan)).To(ConsistOf(expectedResources)) + }) + + It("should assign that log group to the replication group", func() { + Expect(AfterValuesForType(plan, resource)).To(MatchKeys(IgnoreExtras, Keys{ + "log_delivery_configuration": ConsistOf(MatchAllKeys(Keys{ + "destination": Equal("/aws/elasticache/cluster/csb-redis-test/engine-log"), + "destination_type": Equal("cloudwatch-logs"), + "log_format": Equal("json"), + "log_type": Equal("engine-log"), + })), + })) + }) + + Context("engine_log loggroup configuration", func() { + kmsKeyId := "" + retentionInDays := 0 + JustBeforeEach(func() { + plan = ShowPlan(terraformProvisionDir, buildVars(defaultVars, map[string]any{ + "logs_engine_log_loggroup_kms_key_id": kmsKeyId, + "logs_engine_log_loggroup_retention_in_days": retentionInDays, + "logs_engine_log_enabled": true, + })) + }) + + It("should configure the log group with default values", func() { + engineLogResource := "aws_cloudwatch_log_group" + Expect(AfterValuesForType(plan, engineLogResource)).To(MatchKeys(IgnoreExtras, Keys{ + "name": Equal("/aws/elasticache/cluster/csb-redis-test/engine-log"), + "retention_in_days": BeNumerically("==", retentionInDays), + "kms_key_id": BeNil(), + "skip_destroy": BeFalse(), + "tags": HaveKeyWithValue("key1", "some-redis-value"), + })) + }) + + Context("providing explicit values", func() { + BeforeEach(func() { + kmsKeyId = "test-kms-key" + retentionInDays = 180 + }) + + It("should configure the passed values", func() { + engineLogResource := "aws_cloudwatch_log_group" + Expect(AfterValuesForType(plan, engineLogResource)).To(MatchKeys(IgnoreExtras, Keys{ + "name": Equal("/aws/elasticache/cluster/csb-redis-test/engine-log"), + "retention_in_days": BeNumerically("==", retentionInDays), + "kms_key_id": Equal(kmsKeyId), + "skip_destroy": BeFalse(), + "tags": HaveKeyWithValue("key1", "some-redis-value"), + })) + }) + }) + }) + + }) }) func getExpectedResources() []string { diff --git a/terraform/redis/cluster/provision/main.tf b/terraform/redis/cluster/provision/main.tf index 97d2dbe3d..358732677 100644 --- a/terraform/redis/cluster/provision/main.tf +++ b/terraform/redis/cluster/provision/main.tf @@ -73,7 +73,51 @@ resource "aws_elasticache_replication_group" "redis" { // Terraform detects engine_version difference attempts to re-create auto_minor_version_upgrade = false + dynamic "log_delivery_configuration" { + for_each = var.logs_slow_log_enabled ? [null] : [] + content { + destination = aws_cloudwatch_log_group.slow_log[0].name + destination_type = "cloudwatch-logs" + log_format = "json" + log_type = "slow-log" + } + } + + dynamic "log_delivery_configuration" { + for_each = var.logs_engine_log_enabled ? [null] : [] + content { + destination = aws_cloudwatch_log_group.engine_log[0].name + destination_type = "cloudwatch-logs" + log_format = "json" + log_type = "engine-log" + } + } + lifecycle { prevent_destroy = true } } + +resource "aws_cloudwatch_log_group" "engine_log" { + count = var.logs_engine_log_enabled ? 1 : 0 + lifecycle { + create_before_destroy = true + } + name = "/aws/elasticache/cluster/${var.instance_name}/engine-log" + retention_in_days = var.logs_engine_log_loggroup_retention_in_days + kms_key_id = var.logs_engine_log_loggroup_kms_key_id == "" ? null : var.logs_engine_log_loggroup_kms_key_id + + tags = var.labels +} + +resource "aws_cloudwatch_log_group" "slow_log" { + count = var.logs_slow_log_enabled ? 1 : 0 + lifecycle { + create_before_destroy = true + } + name = "/aws/elasticache/cluster/${var.instance_name}/slow-log" + retention_in_days = var.logs_slow_log_loggroup_retention_in_days + kms_key_id = var.logs_slow_log_loggroup_kms_key_id == "" ? null : var.logs_slow_log_loggroup_kms_key_id + + tags = var.labels +} diff --git a/terraform/redis/cluster/provision/variables.tf b/terraform/redis/cluster/provision/variables.tf index 93d15e2cb..052b0b768 100644 --- a/terraform/redis/cluster/provision/variables.tf +++ b/terraform/redis/cluster/provision/variables.tf @@ -39,4 +39,10 @@ variable "backup_start_min" { type = string } variable "backup_end_hour" { type = string } variable "backup_end_min" { type = string } variable "parameter_group_name" { type = string } -variable "preferred_azs" { type = list(string) } \ No newline at end of file +variable "preferred_azs" { type = list(string) } +variable "logs_slow_log_enabled" { type = bool } +variable "logs_slow_log_loggroup_retention_in_days" { type = number } +variable "logs_slow_log_loggroup_kms_key_id" { type = string } +variable "logs_engine_log_enabled" { type = bool } +variable "logs_engine_log_loggroup_retention_in_days" { type = number } +variable "logs_engine_log_loggroup_kms_key_id" { type = string } \ No newline at end of file