Skip to content

Commit

Permalink
feat(redis): expose logging configuration
Browse files Browse the repository at this point in the history 
slow logs and engine logs can be sent to
cloudwatch

[#184471699](https://www.pivotaltracker.com/story/show/184471699)
  • Loading branch information
@pivotal-marcela-campo
pivotal-marcela-campo committed Mar 29, 2023
1 parent 6182263 commit 50e8844
Show file tree
Hide file tree
Showing 5 changed files with 323 additions and 60 deletions.
50 changes: 48 additions & 2 deletions aws-redis.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,52 @@ provision:
List of EC2 availability zones in which the nodes will be created.
The first item in the list will be the primary node.
Number of entries must equal to node_count.
- field_name: logs_slow_log_enabled
type: boolean
default: false
details: |
Enable the streaming of Redis Slow Log to CloudWatch. Slow Log is supported for Redis replication groups using version 6.0 onward.
- field_name: logs_slow_log_loggroup_retention_in_days
type: number
default: 0
details: |
Specifies the number of days you want to retain log events in the specified log group.
If 0 is specified, the events in the log group are always retained and never expire.
When specifying `logs_slow_log_loggroup_retention_in_days`, `logs_slow_log_enabled` needs to be set to true.
For more information, see
https://docs.aws.amazon.com/AmazonCloudWatchLogs/latest/APIReference/API_PutRetentionPolicy.html#API_PutRetentionPolicy_RequestSyntax
- field_name: logs_slow_log_loggroup_kms_key_id
type: string
default: ""
details: |
The ARN for the KMS key to encrypt Slow logs CloudWatch logs.
When specifying `logs_slow_log_loggroup_kms_key_id`, `logs_slow_log_enabled` needs to be set to true.
If omitted, CloudWatch default encryption will apply.
For information on CloudWatch log data encryption and how to configure a KMS key, see
https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/encrypt-log-data-kms.html
- field_name: logs_engine_log_enabled
type: boolean
default: false
details: |
Enable the streaming of Redis Engine logs to CloudWatch. Engine Log is supported for Redis replication groups using version 6.2 onward.
- field_name: logs_engine_log_loggroup_retention_in_days
type: number
default: 0
details: |
Specifies the number of days you want to retain log events in the specified log group.
If 0 is specified, the events in the log group are always retained and never expire.
When specifying `logs_engine_log_loggroup_retention_in_days`, `logs_engine_log_enabled` needs to be set to true.
For more information, see
https://docs.aws.amazon.com/AmazonCloudWatchLogs/latest/APIReference/API_PutRetentionPolicy.html#API_PutRetentionPolicy_RequestSyntax
- field_name: logs_engine_log_loggroup_kms_key_id
type: string
default: ""
details: |
The ARN for the KMS key to encrypt Engine Log CloudWatch logs.
When specifying `logs_engine_log_loggroup_kms_key_id`, `logs_engine_log_enabled` needs to be set to true.
If omitted, CloudWatch default encryption will apply.
For information on CloudWatch log data encryption and how to configure a KMS key, see
https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/encrypt-log-data-kms.html
computed_inputs:
- name: labels
default: ${json.marshal(request.default_labels)}
Expand Down Expand Up @@ -273,7 +319,7 @@ examples:
provision_params: {"region": "us-east-1", "multi_az_enabled": false}
bind_params: {}
- name: with-custom-node_type
description: Create a Redis instance with a custom node_type
description: Create a Redis instance with a custom node_type and slow logs enabled
plan_id: 2deb6c13-7ea1-4bad-a519-0ac9600e9a29
provision_params: {"node_type": "cache.t2.micro", "multi_az_enabled": false}
provision_params: {"node_type": "cache.t2.micro", "multi_az_enabled": false, "logs_slow_log_enabled": true}
bind_params: {}
76 changes: 50 additions & 26 deletions integration-tests/redis_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -275,37 +275,49 @@ var _ = Describe("Redis", Label("Redis"), func() {
HaveKeyWithValue("backup_end_min", BeNil()),
HaveKeyWithValue("parameter_group_name", ""),
HaveKeyWithValue("preferred_azs", BeEmpty()),
HaveKeyWithValue("logs_slow_log_enabled", BeFalse()),
HaveKeyWithValue("logs_slow_log_loggroup_retention_in_days", BeNumerically("==", 0)),
HaveKeyWithValue("logs_slow_log_loggroup_kms_key_id", BeEmpty()),
HaveKeyWithValue("logs_engine_log_enabled", BeFalse()),
HaveKeyWithValue("logs_engine_log_loggroup_retention_in_days", BeNumerically("==", 0)),
HaveKeyWithValue("logs_engine_log_loggroup_kms_key_id", BeEmpty()),
))
})

It("should allow properties to be set on provision", func() {
_, err := broker.Provision(redisServiceName, redisCustomPlanName, map[string]any{
"redis_version": "6.x",
"instance_name": "some-valid-instance-name",
"region": "some-valid-region",
"aws_vpc_id": "some-valid-aws-vpc-id",
"elasticache_subnet_group": "some-valid-elasticache-subnet-group",
"elasticache_vpc_security_group_ids": "some-valid-elasticache-vpc-security-group-ids",
"aws_access_key_id": "some-valid-aws-access-key-id",
"aws_secret_access_key": "some-valid-aws-secret-access-key",
"at_rest_encryption_enabled": false,
"kms_key_id": "fake-encryption-at-rest-key",
"maintenance_day": "Mon",
"maintenance_start_hour": "03",
"maintenance_start_min": "45",
"maintenance_end_hour": "10",
"maintenance_end_min": "15",
"automatic_failover_enabled": false,
"multi_az_enabled": false,
"backup_retention_limit": 32,
"final_backup_identifier": "tortoise",
"backup_name": "turtle",
"backup_start_hour": "04",
"backup_start_min": "15",
"backup_end_hour": "11",
"backup_end_min": "30",
"parameter_group_name": "fake-param-group-name",
"preferred_azs": []string{"fake-az1", "fake-az2"},
"redis_version": "6.x",
"instance_name": "some-valid-instance-name",
"region": "some-valid-region",
"aws_vpc_id": "some-valid-aws-vpc-id",
"elasticache_subnet_group": "some-valid-elasticache-subnet-group",
"elasticache_vpc_security_group_ids": "some-valid-elasticache-vpc-security-group-ids",
"aws_access_key_id": "some-valid-aws-access-key-id",
"aws_secret_access_key": "some-valid-aws-secret-access-key",
"at_rest_encryption_enabled": false,
"kms_key_id": "fake-encryption-at-rest-key",
"maintenance_day": "Mon",
"maintenance_start_hour": "03",
"maintenance_start_min": "45",
"maintenance_end_hour": "10",
"maintenance_end_min": "15",
"automatic_failover_enabled": false,
"multi_az_enabled": false,
"backup_retention_limit": 32,
"final_backup_identifier": "tortoise",
"backup_name": "turtle",
"backup_start_hour": "04",
"backup_start_min": "15",
"backup_end_hour": "11",
"backup_end_min": "30",
"parameter_group_name": "fake-param-group-name",
"preferred_azs": []string{"fake-az1", "fake-az2"},
"logs_slow_log_enabled": true,
"logs_slow_log_loggroup_retention_in_days": 1,
"logs_slow_log_loggroup_kms_key_id": "slow-log-key",
"logs_engine_log_enabled": true,
"logs_engine_log_loggroup_retention_in_days": 2,
"logs_engine_log_loggroup_kms_key_id": "engine-log-key",
})
Expect(err).NotTo(HaveOccurred())

Expand Down Expand Up @@ -338,6 +350,12 @@ var _ = Describe("Redis", Label("Redis"), func() {
HaveKeyWithValue("backup_end_min", "30"),
HaveKeyWithValue("parameter_group_name", "fake-param-group-name"),
HaveKeyWithValue("preferred_azs", ConsistOf("fake-az1", "fake-az2")),
HaveKeyWithValue("logs_slow_log_enabled", BeTrue()),
HaveKeyWithValue("logs_slow_log_loggroup_retention_in_days", BeNumerically("==", 1)),
HaveKeyWithValue("logs_slow_log_loggroup_kms_key_id", "slow-log-key"),
HaveKeyWithValue("logs_engine_log_enabled", BeTrue()),
HaveKeyWithValue("logs_engine_log_loggroup_retention_in_days", BeNumerically("==", 2)),
HaveKeyWithValue("logs_engine_log_loggroup_kms_key_id", "engine-log-key"),
),
)
})
Expand Down Expand Up @@ -429,6 +447,12 @@ var _ = Describe("Redis", Label("Redis"), func() {
Entry("backup_end_hour", "backup_end_hour", "05"),
Entry("backup_end_min", "backup_end_min", "30"),
Entry("parameter_group_name", "parameter_group_name", "fake-param-group-name"),
Entry("logs_slow_log_enabled", "logs_slow_log_enabled", true),
Entry("logs_slow_log_loggroup_retention_in_days", "logs_slow_log_loggroup_retention_in_days", 4),
Entry("logs_slow_log_loggroup_kms_key_id", "logs_slow_log_loggroup_kms_key_id", "slow-log-key-2"),
Entry("logs_engine_log_enabled", "logs_engine_log_enabled", true),
Entry("logs_engine_log_loggroup_retention_in_days", "logs_engine_log_loggroup_retention_in_days", 5),
Entry("logs_engine_log_loggroup_kms_key_id", "logs_engine_log_loggroup_kms_key_id", "engine-log-key-2"),
)
})

Expand Down
205 changes: 174 additions & 31 deletions terraform-tests/redis_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,37 +20,43 @@ var _ = Describe("Redis", Label("redis-terraform"), Ordered, func() {
)

defaultVars := map[string]any{
"cache_size": nil,
"redis_version": "6.0",
"instance_name": "csb-redis-test",
"labels": map[string]any{"key1": "some-redis-value"},
"node_type": "cache.t3.medium",
"node_count": 2,
"elasticache_subnet_group": "",
"elasticache_vpc_security_group_ids": "",
"region": "us-west-2",
"aws_access_key_id": awsAccessKeyID,
"aws_secret_access_key": awsSecretAccessKey,
"aws_vpc_id": awsVPCID,
"at_rest_encryption_enabled": true,
"kms_key_id": "fake-encryption-at-rest-key",
"maintenance_end_hour": nil,
"maintenance_start_hour": nil,
"maintenance_end_min": nil,
"maintenance_start_min": nil,
"maintenance_day": nil,
"data_tiering_enabled": false,
"automatic_failover_enabled": true,
"multi_az_enabled": true,
"backup_retention_limit": 12,
"final_backup_identifier": "tortoise",
"backup_name": "turtle",
"backup_end_hour": nil,
"backup_start_hour": nil,
"backup_end_min": nil,
"backup_start_min": nil,
"parameter_group_name": "fake-param-group-name",
"preferred_azs": nil,
"cache_size": nil,
"redis_version": "6.0",
"instance_name": "csb-redis-test",
"labels": map[string]any{"key1": "some-redis-value"},
"node_type": "cache.t3.medium",
"node_count": 2,
"elasticache_subnet_group": "",
"elasticache_vpc_security_group_ids": "",
"region": "us-west-2",
"aws_access_key_id": awsAccessKeyID,
"aws_secret_access_key": awsSecretAccessKey,
"aws_vpc_id": awsVPCID,
"at_rest_encryption_enabled": true,
"kms_key_id": "fake-encryption-at-rest-key",
"maintenance_end_hour": nil,
"maintenance_start_hour": nil,
"maintenance_end_min": nil,
"maintenance_start_min": nil,
"maintenance_day": nil,
"data_tiering_enabled": false,
"automatic_failover_enabled": true,
"multi_az_enabled": true,
"backup_retention_limit": 12,
"final_backup_identifier": "tortoise",
"backup_name": "turtle",
"backup_end_hour": nil,
"backup_start_hour": nil,
"backup_end_min": nil,
"backup_start_min": nil,
"parameter_group_name": "fake-param-group-name",
"preferred_azs": nil,
"logs_slow_log_loggroup_kms_key_id": "",
"logs_slow_log_loggroup_retention_in_days": 0,
"logs_slow_log_enabled": false,
"logs_engine_log_loggroup_kms_key_id": "",
"logs_engine_log_loggroup_retention_in_days": 0,
"logs_engine_log_enabled": false,
}

BeforeAll(func() {
Expand Down Expand Up @@ -266,6 +272,143 @@ var _ = Describe("Redis", Label("redis-terraform"), Ordered, func() {
})
})

Context("slow_log is enabled", func() {
BeforeAll(func() {
plan = ShowPlan(terraformProvisionDir, buildVars(defaultVars, map[string]any{
"logs_slow_log_loggroup_kms_key_id": "",
"logs_slow_log_loggroup_retention_in_days": 0,
"logs_slow_log_enabled": true,
}))
})

It("should create a log group for slow log", func() {
expectedResources := []string{"aws_cloudwatch_log_group"}
expectedResources = append(expectedResources, getExpectedResources()...)
Expect(ResourceChangesTypes(plan)).To(ConsistOf(expectedResources))
})

It("should assign that log group to the replication group", func() {
Expect(AfterValuesForType(plan, resource)).To(MatchKeys(IgnoreExtras, Keys{
"log_delivery_configuration": ConsistOf(MatchAllKeys(Keys{
"destination": Equal("/aws/elasticache/cluster/csb-redis-test/slow-log"),
"destination_type": Equal("cloudwatch-logs"),
"log_format": Equal("json"),
"log_type": Equal("slow-log"),
})),
}))
})

Context("slow_log loggroup configuration", func() {
kmsKeyId := ""
retentionInDays := 0
JustBeforeEach(func() {
plan = ShowPlan(terraformProvisionDir, buildVars(defaultVars, map[string]any{
"logs_slow_log_loggroup_kms_key_id": kmsKeyId,
"logs_slow_log_loggroup_retention_in_days": retentionInDays,
"logs_slow_log_enabled": true,
}))
})

It("should configure the log group with default values", func() {
slowLogResource := "aws_cloudwatch_log_group"
Expect(AfterValuesForType(plan, slowLogResource)).To(MatchKeys(IgnoreExtras, Keys{
"name": Equal("/aws/elasticache/cluster/csb-redis-test/slow-log"),
"retention_in_days": BeNumerically("==", retentionInDays),
"kms_key_id": BeNil(),
"skip_destroy": BeFalse(),
"tags": HaveKeyWithValue("key1", "some-redis-value"),
}))
})

Context("providing explicit values", func() {
BeforeEach(func() {
kmsKeyId = "test-kms-key"
retentionInDays = 180
})

It("should configure the passed values", func() {
slowLogResource := "aws_cloudwatch_log_group"
Expect(AfterValuesForType(plan, slowLogResource)).To(MatchKeys(IgnoreExtras, Keys{
"name": Equal("/aws/elasticache/cluster/csb-redis-test/slow-log"),
"retention_in_days": BeNumerically("==", retentionInDays),
"kms_key_id": Equal(kmsKeyId),
"skip_destroy": BeFalse(),
"tags": HaveKeyWithValue("key1", "some-redis-value"),
}))
})
})
})

})

Context("engine_log is enabled", func() {
BeforeAll(func() {
plan = ShowPlan(terraformProvisionDir, buildVars(defaultVars, map[string]any{
"logs_engine_log_loggroup_kms_key_id": "",
"logs_engine_log_loggroup_retention_in_days": 0,
"logs_engine_log_enabled": true,
}))
})

It("should create a log group for engine log", func() {
expectedResources := []string{"aws_cloudwatch_log_group"}
expectedResources = append(expectedResources, getExpectedResources()...)
Expect(ResourceChangesTypes(plan)).To(ConsistOf(expectedResources))
})

It("should assign that log group to the replication group", func() {
Expect(AfterValuesForType(plan, resource)).To(MatchKeys(IgnoreExtras, Keys{
"log_delivery_configuration": ConsistOf(MatchAllKeys(Keys{
"destination": Equal("/aws/elasticache/cluster/csb-redis-test/engine-log"),
"destination_type": Equal("cloudwatch-logs"),
"log_format": Equal("json"),
"log_type": Equal("engine-log"),
})),
}))
})

Context("engine_log loggroup configuration", func() {
kmsKeyId := ""
retentionInDays := 0
JustBeforeEach(func() {
plan = ShowPlan(terraformProvisionDir, buildVars(defaultVars, map[string]any{
"logs_engine_log_loggroup_kms_key_id": kmsKeyId,
"logs_engine_log_loggroup_retention_in_days": retentionInDays,
"logs_engine_log_enabled": true,
}))
})

It("should configure the log group with default values", func() {
engineLogResource := "aws_cloudwatch_log_group"
Expect(AfterValuesForType(plan, engineLogResource)).To(MatchKeys(IgnoreExtras, Keys{
"name": Equal("/aws/elasticache/cluster/csb-redis-test/engine-log"),
"retention_in_days": BeNumerically("==", retentionInDays),
"kms_key_id": BeNil(),
"skip_destroy": BeFalse(),
"tags": HaveKeyWithValue("key1", "some-redis-value"),
}))
})

Context("providing explicit values", func() {
BeforeEach(func() {
kmsKeyId = "test-kms-key"
retentionInDays = 180
})

It("should configure the passed values", func() {
engineLogResource := "aws_cloudwatch_log_group"
Expect(AfterValuesForType(plan, engineLogResource)).To(MatchKeys(IgnoreExtras, Keys{
"name": Equal("/aws/elasticache/cluster/csb-redis-test/engine-log"),
"retention_in_days": BeNumerically("==", retentionInDays),
"kms_key_id": Equal(kmsKeyId),
"skip_destroy": BeFalse(),
"tags": HaveKeyWithValue("key1", "some-redis-value"),
}))
})
})
})

})
})

func getExpectedResources() []string {
Expand Down
Loading

0 comments on commit 50e8844

Please sign in to comment.