From 4534b7151425626452ec670fc01b0b4da3772de0 Mon Sep 17 00:00:00 2001 From: nouseforaname Date: Mon, 17 Oct 2022 09:41:03 +0200 Subject: [PATCH] feat:Add option to enable audit log export to cloudwatch [#183457241] We want to enable users to be able to export their audit logs for a service instance. This adds the required resources and input variables to enable exporting audit logs to cloudwatch. This setting requires providing a properly configured option_group identified by `option_group_name` (enabling the audit log plugin, (see)[https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Appendix.MySQL.Options.AuditPlugin.html] ) Known Issues: IF an instance is created without audit log enabled AND then the Instance get's updated to enable audit logging AND then the Instance get's updated to disable audit logging THEN the log group created in step 2 will NOT be deleted. This will create an issue if you try to set audit logging to true again because terraform will error trying to create an existing log group. To avoid this, the log groups use name_prefix rather than name. --- aws-mysql.yml | 39 ++++++++++---- integration-tests/mysql_test.go | 75 ++++++++++++++------------ terraform/mysql/provision/data.tf | 3 +- terraform/mysql/provision/main.tf | 23 +++++++- terraform/mysql/provision/variables.tf | 8 ++- 5 files changed, 101 insertions(+), 47 deletions(-) diff --git a/aws-mysql.yml b/aws-mysql.yml index 905e4c222..c136b53c1 100644 --- a/aws-mysql.yml +++ b/aws-mysql.yml @@ -77,14 +77,6 @@ provision: maximum: 4096 minimum: 5 user_inputs: - - field_name: option_group_name - type: string - default: null - nullable: true - details: | - Name of the DB option group to associate. - MySQL offers additional features such as the audit plugin or Memcached to manage data and database or to provide additional security for the database. - RDS uses option groups to enable and configure these features. - field_name: storage_type type: string details: Type of storage to be used. One of "standard" (magnetic), "gp2" (general purpose SSD), or "io1" (provisioned IOPS SSD). @@ -117,6 +109,14 @@ provision: The ARN for the KMS encryption key (see https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Overview.Encryption.html) The `storage_encrypted` property must be enabled if the key is specified. prohibit_update: true + - field_name: option_group_name + type: string + default: "" + nullable: true + details: | + Name of the DB option group to associate. If left empty, defaults to `default:mysql--` + MySQL offers additional features such as the audit plugin or Memcached to manage data and database or to provide additional security for the database. + RDS uses option groups to enable and configure these features. - field_name: parameter_group_name type: string default: "" @@ -314,6 +314,25 @@ provision: default: 7 constraints: minimum: 7 + - field_name: enable_audit_logging + type: boolean + default: false + details: | + Requires setting option_group_name with a pre-created option_group that fullfils requirements for audit log exports. + See AWS Docs for config options: https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Appendix.MySQL.Options.AuditPlugin.html#Appendix.MySQL.Options.AuditPlugin.Add + If set will enable the `audit` cloud_watch_log_export on the rds instance. + - field_name: cloudwatch_log_group_retention_in_days + type: number + details: | + Used in conjunction with `enable_audit_logging`. If provided will set the retention days for the log group containing the rds audit logs. Defaults to 30 Days. + default: 30 + constraints: + minimum: 1 + maximum: 3653 #10y + - field_name: cloudwatch_log_group_kms_key_id + type: string + default: "" + details: Used in conjunction with `enable_audit_logging`. If provided will set the kms_key to use for encrypting the cloudwatch log group create for the rds audit logs. computed_inputs: - name: labels default: ${json.marshal(request.default_labels)} @@ -349,7 +368,7 @@ provision: details: The password to authenticate to the database instance. - field_name: use_tls type: boolean - details: Using TLS for connection + details: Using TLS for connection bind: plan_inputs: [] user_inputs: [] @@ -373,7 +392,7 @@ bind: - name: use_tls type: boolean default: ${instance.details["use_tls"]} - overwrite: true + overwrite: true template_refs: outputs: terraform/mysql/bind/outputs.tf provider: terraform/mysql/bind/provider.tf diff --git a/integration-tests/mysql_test.go b/integration-tests/mysql_test.go index b7817313b..e690e20d6 100644 --- a/integration-tests/mysql_test.go +++ b/integration-tests/mysql_test.go @@ -155,45 +155,51 @@ var _ = Describe("MySQL", Label("MySQL"), func() { HaveKeyWithValue("performance_insights_enabled", false), HaveKeyWithValue("performance_insights_kms_key_id", ""), HaveKeyWithValue("performance_insights_retention_period", float64(7)), + HaveKeyWithValue("enable_audit_logging", false), + HaveKeyWithValue("cloudwatch_log_group_kms_key_id", ""), + HaveKeyWithValue("cloudwatch_log_group_retention_in_days", float64(30)), ), ) }) It("should allow properties to be set on provision", func() { _, err := broker.Provision(serviceName, customMySQLPlan["name"].(string), map[string]any{ - "use_tls": false, - "storage_type": "gp2", - "storage_autoscale": true, - "storage_autoscale_limit_gb": float64(150), - "storage_encrypted": true, - "kms_key_id": "arn:aws:xxxx", - "parameter_group_name": "fake-parameter-group", - "instance_name": "csb-mysql-fake-name", - "db_name": "fake-db-name", - "publicly_accessible": true, - "region": "africa-north-4", - "multi_az": true, - "instance_class": "", - "rds_subnet_group": "", - "rds_vpc_security_group_ids": "", - "allow_major_version_upgrade": false, - "auto_minor_version_upgrade": false, - "maintenance_day": "Mon", - "maintenance_start_hour": "03", - "maintenance_start_min": "45", - "maintenance_end_hour": "10", - "maintenance_end_min": "15", - "deletion_protection": true, - "backup_retention_period": float64(2), - "backup_window": "01:02-03:04", - "copy_tags_to_snapshot": false, - "delete_automated_backups": false, - "option_group_name": "option-group-name", - "monitoring_interval": 30, - "monitoring_role_arn": "arn:aws:iam::xxxxxxxxxxxx:role/enhanced_monitoring_access", - "performance_insights_enabled": true, - "performance_insights_kms_key_id": "arn:aws:kms:us-west-2:649758297924:key/ebbb4ecc-ddfb-4e2f-8e93-c96d7bc43daa", - "performance_insights_retention_period": 93, + "use_tls": false, + "storage_type": "gp2", + "storage_autoscale": true, + "storage_autoscale_limit_gb": float64(150), + "storage_encrypted": true, + "kms_key_id": "arn:aws:xxxx", + "parameter_group_name": "fake-parameter-group", + "instance_name": "csb-mysql-fake-name", + "db_name": "fake-db-name", + "publicly_accessible": true, + "region": "africa-north-4", + "multi_az": true, + "instance_class": "", + "rds_subnet_group": "", + "rds_vpc_security_group_ids": "", + "allow_major_version_upgrade": false, + "auto_minor_version_upgrade": false, + "maintenance_day": "Mon", + "maintenance_start_hour": "03", + "maintenance_start_min": "45", + "maintenance_end_hour": "10", + "maintenance_end_min": "15", + "deletion_protection": true, + "backup_retention_period": float64(2), + "backup_window": "01:02-03:04", + "copy_tags_to_snapshot": false, + "delete_automated_backups": false, + "option_group_name": "option-group-name", + "monitoring_interval": 30, + "monitoring_role_arn": "arn:aws:iam::xxxxxxxxxxxx:role/enhanced_monitoring_access", + "performance_insights_enabled": true, + "performance_insights_kms_key_id": "arn:aws:kms:us-west-2:649758297924:key/ebbb4ecc-ddfb-4e2f-8e93-c96d7bc43daa", + "performance_insights_retention_period": 93, + "enable_audit_logging": true, + "cloudwatch_log_group_kms_key_id": "arn:aws:kms:us-west-2:649758297924:key/ebbb4ecc-ddfb-4e2f-8e93-c96d7bc43daa", + "cloudwatch_log_group_retention_in_days": 33, }) Expect(err).NotTo(HaveOccurred()) @@ -236,6 +242,9 @@ var _ = Describe("MySQL", Label("MySQL"), func() { HaveKeyWithValue("performance_insights_enabled", true), HaveKeyWithValue("performance_insights_kms_key_id", "arn:aws:kms:us-west-2:649758297924:key/ebbb4ecc-ddfb-4e2f-8e93-c96d7bc43daa"), HaveKeyWithValue("performance_insights_retention_period", float64(93)), + HaveKeyWithValue("enable_audit_logging", true), + HaveKeyWithValue("cloudwatch_log_group_kms_key_id", "arn:aws:kms:us-west-2:649758297924:key/ebbb4ecc-ddfb-4e2f-8e93-c96d7bc43daa"), + HaveKeyWithValue("cloudwatch_log_group_retention_in_days", float64(33)), ), ) }) diff --git a/terraform/mysql/provision/data.tf b/terraform/mysql/provision/data.tf index ca197d193..3a5d264ee 100644 --- a/terraform/mysql/provision/data.tf +++ b/terraform/mysql/provision/data.tf @@ -36,6 +36,7 @@ locals { subnet_group = length(var.rds_subnet_group) > 0 ? var.rds_subnet_group : aws_db_subnet_group.rds-private-subnet[0].name parameter_group_name = length(var.parameter_group_name) == 0 ? format("default.%s%s", var.engine, var.engine_version) : var.parameter_group_name + option_group_name = length(var.option_group_name) == 0 ? format("default:%s-%s", var.engine, replace(var.engine_version,".","-")) : var.option_group_name max_allocated_storage = (var.storage_autoscale && var.storage_autoscale_limit_gb > var.storage_gb) ? var.storage_autoscale_limit_gb : null @@ -66,4 +67,4 @@ data "aws_subnets" "all" { name = "vpc-id" values = [data.aws_vpc.vpc.id] } -} \ No newline at end of file +} diff --git a/terraform/mysql/provision/main.tf b/terraform/mysql/provision/main.tf index ca88d75ce..5f5833543 100644 --- a/terraform/mysql/provision/main.tf +++ b/terraform/mysql/provision/main.tf @@ -77,14 +77,33 @@ resource "aws_db_instance" "db_instance" { backup_window = var.backup_window copy_tags_to_snapshot = var.copy_tags_to_snapshot delete_automated_backups = var.delete_automated_backups - option_group_name = var.option_group_name + option_group_name = local.option_group_name monitoring_interval = var.monitoring_interval monitoring_role_arn = var.monitoring_role_arn performance_insights_enabled = var.performance_insights_enabled performance_insights_kms_key_id = var.performance_insights_kms_key_id == "" ? null : var.performance_insights_kms_key_id performance_insights_retention_period = var.performance_insights_enabled ? var.performance_insights_retention_period : null + # Audit Logging + enabled_cloudwatch_logs_exports = var.enable_audit_logging == true ? ["audit"] : [] + lifecycle { prevent_destroy = true } -} \ No newline at end of file + +} +locals { + log_groups = var.enable_audit_logging == true ? { "audit" : true } : {} +} +resource "aws_cloudwatch_log_group" "this" { + for_each = local.log_groups + lifecycle { + create_before_destroy = true + } + name_prefix = "/aws/rds/instance/${var.instance_name}/${each.key}/" + retention_in_days = var.cloudwatch_log_group_retention_in_days + kms_key_id = var.cloudwatch_log_group_kms_key_id == "" ? null : var.cloudwatch_log_group_kms_key_id + + tags = var.labels + +} diff --git a/terraform/mysql/provision/variables.tf b/terraform/mysql/provision/variables.tf index 3e17a5408..fa00c9582 100644 --- a/terraform/mysql/provision/variables.tf +++ b/terraform/mysql/provision/variables.tf @@ -50,4 +50,10 @@ variable "monitoring_interval" { type = number } variable "monitoring_role_arn" { type = string } variable "performance_insights_enabled" { type = bool } variable "performance_insights_kms_key_id" { type = string } -variable "performance_insights_retention_period" { type = number } \ No newline at end of file +variable "performance_insights_retention_period" { type = number } +variable "enable_audit_logging" { type = bool } +variable "cloudwatch_log_group_retention_in_days" { + type = number + default = 30 +} +variable "cloudwatch_log_group_kms_key_id" { type = string }