merge changes from outerbounds-master

README.md

@@ -108,7 +108,11 @@ resource "local_file" "metaflow_config" {
 | <a name="input_compute_environment_instance_types"></a> [compute\_environment\_instance\_types](#input\_compute\_environment\_instance\_types) | The instance types for the compute environment | `list(string)` | <pre>[<br>  "c4.large",<br>  "c4.xlarge",<br>  "c4.2xlarge",<br>  "c4.4xlarge",<br>  "c4.8xlarge"<br>]</pre> | no |
 | <a name="input_compute_environment_max_vcpus"></a> [compute\_environment\_max\_vcpus](#input\_compute\_environment\_max\_vcpus) | Maximum VCPUs for Batch Compute Environment [16-96] | `number` | `64` | no |
 | <a name="input_compute_environment_min_vcpus"></a> [compute\_environment\_min\_vcpus](#input\_compute\_environment\_min\_vcpus) | Minimum VCPUs for Batch Compute Environment [0-16] for EC2 Batch Compute Environment (ignored for Fargate) | `number` | `8` | no |
+| <a name="input_db_engine_version"></a> [db\_engine\_version](#input\_db\_engine\_version) | n/a | `string` | `"11"` | no |
+| <a name="input_db_instance_type"></a> [db\_instance\_type](#input\_db\_instance\_type) | RDS instance type to launch for PostgresQL database. | `string` | `"db.t2.small"` | no |
+| <a name="input_db_migrate_lambda_zip_file"></a> [db\_migrate\_lambda\_zip\_file](#input\_db\_migrate\_lambda\_zip\_file) | Output path for the zip file containing the DB migrate lambda | `string` | `null` | no |
 | <a name="input_enable_custom_batch_container_registry"></a> [enable\_custom\_batch\_container\_registry](#input\_enable\_custom\_batch\_container\_registry) | Provisions infrastructure for custom Amazon ECR container registry if enabled | `bool` | `false` | no |
+| <a name="input_enable_key_rotation"></a> [enable\_key\_rotation](#input\_enable\_key\_rotation) | Enable key rotation for KMS keys | `bool` | `false` | no |
 | <a name="input_enable_step_functions"></a> [enable\_step\_functions](#input\_enable\_step\_functions) | Provisions infrastructure for step functions if enabled | `bool` | n/a | yes |
 | <a name="input_extra_ui_backend_env_vars"></a> [extra\_ui\_backend\_env\_vars](#input\_extra\_ui\_backend\_env\_vars) | Additional environment variables for UI backend container | `map(string)` | `{}` | no |
 | <a name="input_extra_ui_static_env_vars"></a> [extra\_ui\_static\_env\_vars](#input\_extra\_ui\_static\_env\_vars) | Additional environment variables for UI static app | `map(string)` | `{}` | no |

examples/eks_argo/metaflow.tf

@@ -20,7 +20,7 @@ data "aws_availability_zones" "available" {
 
 module "metaflow-datastore" {
   source  = "outerbounds/metaflow/aws//modules/datastore"
-  version = "0.8.0"
+  version = "0.10.0"
 
   force_destroy_s3_bucket = true
 
@@ -37,18 +37,18 @@ module "metaflow-datastore" {
 
 module "metaflow-common" {
   source  = "outerbounds/metaflow/aws//modules/common"
-  version = "0.8.0"
+  version = "0.10.0"
 }
 
 module "metaflow-metadata-service" {
   source  = "outerbounds/metaflow/aws//modules/metadata-service"
-  version = "0.8.0"
+  version = "0.10.0"
 
   resource_prefix = local.resource_prefix
   resource_suffix = local.resource_suffix
 
   access_list_cidr_blocks          = []
-  api_basic_auth                   = true
+  enable_api_basic_auth            = true
   database_name                    = module.metaflow-datastore.database_name
   database_password                = module.metaflow-datastore.database_password
   database_username                = module.metaflow-datastore.database_username

examples/eks_argo/vpc.tf

@@ -2,7 +2,7 @@
 # VPC infra using https://github.com/terraform-aws-modules/terraform-aws-vpc
 module "vpc" {
   source  = "terraform-aws-modules/vpc/aws"
-  version = "3.13.0"
+  version = "5.1.2"
 
   name = "${local.resource_prefix}-${local.resource_suffix}"
   cidr = "10.10.0.0/16"

main.tf

@@ -1,9 +1,8 @@
 module "metaflow-datastore" {
   source = "./modules/datastore"
 
-  db_instance_type = var.db_instance_type
-
   force_destroy_s3_bucket = var.force_destroy_s3_bucket
+  enable_key_rotation     = var.enable_key_rotation
 
   resource_prefix = local.resource_prefix
   resource_suffix = local.resource_suffix
@@ -13,6 +12,9 @@ module "metaflow-datastore" {
   subnet1_id                         = var.subnet1_id
   subnet2_id                         = var.subnet2_id
 
+  db_instance_type  = var.db_instance_type
+  db_engine_version = var.db_engine_version
+
   standard_tags = var.tags
 }
 
@@ -26,6 +28,7 @@ module "metaflow-metadata-service" {
   database_name                    = module.metaflow-datastore.database_name
   database_password                = module.metaflow-datastore.database_password
   database_username                = module.metaflow-datastore.database_username
+  db_migrate_lambda_zip_file       = var.db_migrate_lambda_zip_file
   datastore_s3_bucket_kms_key_arn  = module.metaflow-datastore.datastore_s3_bucket_kms_key_arn
   enable_api_basic_auth            = var.metadata_service_enable_api_basic_auth
   enable_api_gateway               = var.metadata_service_enable_api_gateway

modules/datastore/README.md

@@ -21,9 +21,10 @@ To read more, see [the Metaflow docs](https://docs.metaflow.org/metaflow-on-aws/
 |------|-------------|------|---------|:--------:|
 | <a name="input_db_engine"></a> [db\_engine](#input\_db\_engine) | n/a | `string` | `"postgres"` | no |
 | <a name="input_db_engine_version"></a> [db\_engine\_version](#input\_db\_engine\_version) | n/a | `string` | `"11"` | no |
-| <a name="input_db_instance_type"></a> [db\_instance\_type](#input\_db\_instance\_type) | RDS instance type to launch for PostgresQL database. | `string` | `"db.t2.small"` | no |
+| <a name="input_db_instance_type"></a> [db\_instance\_type](#input\_db\_instance\_type) | RDS instance type to launch for PostgresQL database. | `string` | `"db.t3.small"` | no |
 | <a name="input_db_name"></a> [db\_name](#input\_db\_name) | Name of PostgresQL database for Metaflow service. | `string` | `"metaflow"` | no |
 | <a name="input_db_username"></a> [db\_username](#input\_db\_username) | PostgresQL username; defaults to 'metaflow' | `string` | `"metaflow"` | no |
+| <a name="input_enable_key_rotation"></a> [enable\_key\_rotation](#input\_enable\_key\_rotation) | Enable key rotation for KMS keys | `bool` | `false` | no |
 | <a name="input_force_destroy_s3_bucket"></a> [force\_destroy\_s3\_bucket](#input\_force\_destroy\_s3\_bucket) | Empty S3 bucket before destroying via terraform destroy | `bool` | `false` | no |
 | <a name="input_metadata_service_security_group_id"></a> [metadata\_service\_security\_group\_id](#input\_metadata\_service\_security\_group\_id) | The security group ID used by the MetaData service. We'll grant this access to our DB. | `string` | n/a | yes |
 | <a name="input_metaflow_vpc_id"></a> [metaflow\_vpc\_id](#input\_metaflow\_vpc\_id) | ID of the Metaflow VPC this SageMaker notebook instance is to be deployed in | `string` | n/a | yes |

modules/datastore/kms.tf

@@ -1,11 +1,13 @@
 resource "aws_kms_key" "s3" {
-  description = "This key is used to encrypt and decrypt the S3 bucket used to store blobs."
+  description         = "This key is used to encrypt and decrypt the S3 bucket used to store blobs."
+  enable_key_rotation = var.enable_key_rotation
 
   tags = var.standard_tags
 }
 
 resource "aws_kms_key" "rds" {
-  description = "This key is used to encrypt and decrypt the RDS database used to store flow execution data."
+  description         = "This key is used to encrypt and decrypt the RDS database used to store flow execution data."
+  enable_key_rotation = var.enable_key_rotation
 
   tags = var.standard_tags
 }

modules/datastore/variables.tf

@@ -1,7 +1,7 @@
 variable "db_instance_type" {
   type        = string
   description = "RDS instance type to launch for PostgresQL database."
-  default     = "db.t2.small"
+  default     = "db.t3.small"
 }
 
 variable "db_engine" {
@@ -65,3 +65,9 @@ variable "subnet2_id" {
   type        = string
   description = "Second subnet used for availability zone redundancy"
 }
+
+variable "enable_key_rotation" {
+  type        = bool
+  description = "Enable key rotation for KMS keys"
+  default     = false
+}

modules/metadata-service/README.md

@@ -20,6 +20,7 @@ If the `access_list_cidr_blocks` variable is set, only traffic originating from
 | <a name="input_database_password"></a> [database\_password](#input\_database\_password) | The database password | `string` | n/a | yes |
 | <a name="input_database_username"></a> [database\_username](#input\_database\_username) | The database username | `string` | n/a | yes |
 | <a name="input_datastore_s3_bucket_kms_key_arn"></a> [datastore\_s3\_bucket\_kms\_key\_arn](#input\_datastore\_s3\_bucket\_kms\_key\_arn) | The ARN of the KMS key used to encrypt the Metaflow datastore S3 bucket | `string` | n/a | yes |
+| <a name="input_db_migrate_lambda_zip_file"></a> [db\_migrate\_lambda\_zip\_file](#input\_db\_migrate\_lambda\_zip\_file) | Output path for the zip file containing the DB migrate lambda | `string` | `null` | no |
 | <a name="input_enable_api_basic_auth"></a> [enable\_api\_basic\_auth](#input\_enable\_api\_basic\_auth) | Enable basic auth for API Gateway? (requires key export) | `bool` | `true` | no |
 | <a name="input_enable_api_gateway"></a> [enable\_api\_gateway](#input\_enable\_api\_gateway) | Enable API Gateway for public metadata service endpoint | `bool` | `true` | no |
 | <a name="input_fargate_execution_role_arn"></a> [fargate\_execution\_role\_arn](#input\_fargate\_execution\_role\_arn) | The IAM role that grants access to ECS and Batch services which we'll use as our Metadata Service API's execution\_role for our Fargate instance | `string` | n/a | yes |

modules/metadata-service/lambda.tf

@@ -78,8 +78,13 @@ resource "aws_iam_role_policy" "grant_lambda_ecs_vpc" {
   policy = data.aws_iam_policy_document.lambda_ecs_task_execute_policy_vpc.json
 }
 
-resource "local_file" "db_migrate_lambda" {
-  content  = <<EOF
+data "archive_file" "db_migrate_lambda" {
+  type             = "zip"
+  output_file_mode = "0666"
+  output_path      = local.db_migrate_lambda_zip_file
+
+  source {
+    content  = <<EOF
 import os, json
 from urllib import request
 
@@ -101,21 +106,14 @@ def handler(event, context):
   print(response)
   return(response)
 EOF
-  filename = local.db_migrate_lambda_source_file
-}
-
-data "archive_file" "db_migrate_lambda" {
-  type             = "zip"
-  source_file      = local.db_migrate_lambda_source_file
-  output_file_mode = "0666"
-  output_path      = local.db_migrate_lambda_zip_file
-  depends_on       = [local_file.db_migrate_lambda]
+    filename = "index.py"
+  }
 }
 
 resource "aws_lambda_function" "db_migrate_lambda" {
   function_name    = local.db_migrate_lambda_name
   handler          = "index.handler"
-  runtime          = "python3.7"
+  runtime          = "python3.12"
   memory_size      = 128
   timeout          = 900
   description      = "Trigger DB Migration"

modules/metadata-service/locals.tf

@@ -22,10 +22,9 @@ locals {
   api_gateway_stage_name                  = "api"
   api_gateway_usage_plan_name             = "${var.resource_prefix}usage-plan${var.resource_suffix}"
 
-  db_migrate_lambda_source_file = "${path.module}/index.py"
-  db_migrate_lambda_zip_file    = "${path.module}/db_migrate_lambda.zip"
-  db_migrate_lambda_name        = "${var.resource_prefix}db_migrate${var.resource_suffix}"
-  lambda_ecs_execute_role_name  = "${var.resource_prefix}lambda_ecs_execute${var.resource_suffix}"
+  db_migrate_lambda_zip_file   = coalesce(var.db_migrate_lambda_zip_file, "${path.module}/db_migrate_lambda.zip")
+  db_migrate_lambda_name       = "${var.resource_prefix}db_migrate${var.resource_suffix}"
+  lambda_ecs_execute_role_name = "${var.resource_prefix}lambda_ecs_execute${var.resource_suffix}"
 
   cloudwatch_logs_arn_prefix = "arn:${var.iam_partition}:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}"
 }

modules/metadata-service/variables.tf

@@ -37,6 +37,12 @@ variable "enable_api_gateway" {
   description = "Enable API Gateway for public metadata service endpoint"
 }
 
+variable "db_migrate_lambda_zip_file" {
+  type        = string
+  description = "Output path for the zip file containing the DB migrate lambda"
+  default     = null
+}
+
 variable "fargate_execution_role_arn" {
   type        = string
   description = "The IAM role that grants access to ECS and Batch services which we'll use as our Metadata Service API's execution_role for our Fargate instance"

variables.tf

@@ -10,6 +10,12 @@ variable "batch_type" {
   default     = "ec2"
 }
 
+variable "db_migrate_lambda_zip_file" {
+  type        = string
+  description = "Output path for the zip file containing the DB migrate lambda"
+  default     = null
+}
+
 variable "enable_custom_batch_container_registry" {
   type        = bool
   default     = false
@@ -61,6 +67,17 @@ variable "compute_environment_egress_cidr_blocks" {
   description = "CIDR blocks to which egress is allowed from the Batch Compute environment's security group"
 }
 
+variable "db_instance_type" {
+  type        = string
+  description = "RDS instance type to launch for PostgresQL database."
+  default     = "db.t3.small"
+}
+
+variable "db_engine_version" {
+  type    = string
+  default = "13"
+}
+
 variable "launch_template_http_endpoint" {
   type        = string
   description = "Whether the metadata service is available. Can be 'enabled' or 'disabled'"
@@ -177,10 +194,10 @@ variable "force_destroy_s3_bucket" {
   default     = false
 }
 
-variable "db_instance_type" {
-  type        = string
-  description = "RDS instance type to launch for PostgresQL database."
-  default     = "db.t2.small"
+variable "enable_key_rotation" {
+  type        = bool
+  description = "Enable key rotation for KMS keys"
+  default     = false
 }
 
 variable "nlb_arn" {