Refactor: Move sensitive config to private file, improve ECS deployment & healthchecks

terraform_templates/AWS/.gitignore

@@ -0,0 +1,4 @@
+# Ignore sensitive and private config files
+config/environments.private.json
+config/credentials.json
+config/private.key

terraform_templates/AWS/alb.tf

@@ -27,6 +27,9 @@ resource "aws_security_group" "http" {
     to_port     = 0
     cidr_blocks = ["0.0.0.0/0"]
   }
+  tags                   = {
+    Name = "RiskTool - LB - Prod"
+  }
 }
 
 resource "aws_lb" "main" {
@@ -47,10 +50,10 @@ resource "aws_lb_target_group" "app" {
     enabled             = true
     path                = "/healthcheck"
     matcher             = 200
-    interval            = 30
-    timeout             = 5
+    interval            = 60
+    timeout             = 30
     healthy_threshold   = 2
-    unhealthy_threshold = 3
+    unhealthy_threshold = 5
   }
 }
 
@@ -65,10 +68,10 @@ resource "aws_lb_target_group" "api" {
     enabled             = true
     path                = "/liveness"
     matcher             = 200
-    interval            = 10
-    timeout             = 5
+    interval            = 60  # Augmente l'intervalle à 60s
+    timeout             = 30  # Augmente le timeout à 30s
     healthy_threshold   = 2
-    unhealthy_threshold = 3
+    unhealthy_threshold = 5  # Plus tolérant aux échecs
   }
   lifecycle {
     create_before_destroy = true

terraform_templates/AWS/ecs_launch_template.tf

@@ -7,7 +7,7 @@ data "aws_ssm_parameter" "ecs_node_ami" {
 resource "aws_launch_template" "ecs_ec2" {
   name_prefix            = "marble-ecs-ec2-"
   image_id               = data.aws_ssm_parameter.ecs_node_ami.value
-  instance_type          = "t3.small"
+  instance_type          = "t3.medium"
   vpc_security_group_ids = [aws_security_group.ecs_node_sg.id]
   key_name =  var.aws_key_pair
 

terraform_templates/AWS/ecs_node_sg.tf

@@ -15,4 +15,7 @@ resource "aws_security_group" "ecs_node_sg" {
     protocol    = "tcp"
     cidr_blocks = ["0.0.0.0/0"]
   }
+  tags                   = {
+    Name = "RiskTool - Node - Prod"
+  }
 }

terraform_templates/AWS/ecs_service.tf

@@ -18,6 +18,9 @@ resource "aws_security_group" "ecs_task" {
     protocol    = "tcp"
     cidr_blocks = ["0.0.0.0/0"]
   }
+  tags                   = {
+    Name = "RiskTool - Task - Prod"
+  }
 }
 
 resource "aws_ecs_service" "app" {

terraform_templates/AWS/esc_tasks_definition.tf

@@ -5,7 +5,7 @@ resource "aws_ecs_task_definition" "app" {
   task_role_arn      = aws_iam_role.ecs_task_role.arn
   execution_role_arn = aws_iam_role.ecs_exec_role.arn
   network_mode       = "bridge"
-  memory             = 256
+  memory             = 1024
 
   container_definitions = jsonencode([{
     name         = "app",
@@ -59,20 +59,23 @@ resource "aws_ecs_task_definition" "app" {
         { name = "PG_HOSTNAME", value = "${element(split(":", aws_db_instance.rds-marble.endpoint), 0)}" },
         { name = "PG_PORT", value = "${element(split(":", aws_db_instance.rds-marble.endpoint), 1)}" },
         { name = "PG_USER", value = "postgres" },
+        { name = "INGESTION_BUCKET_URL", value = "-" },
         { name = "PG_PASSWORD", value = "${random_string.rds-db-password.result}" },
         { name = "GOOGLE_APPLICATION_CREDENTIALS", value = "/config/credentials.json" },
         { name = "GOOGLE_CLOUD_PROJECT", value = local.environment.firebase.projectId },
         { name = "CREATE_GLOBAL_ADMIN_EMAIL", value = local.environment.org.global },
         { name = "CREATE_ORG_NAME", value = local.environment.org.name },
         { name = "CREATE_ORG_ADMIN_EMAIL", value = local.environment.org.admin },
-        { name = "MARBLE_APP_URL", value = local.environment.frontend.domain },
-        { name = "MARBLE_BACKOFFICE_HOST", value = local.environment.backend.domain },
+        { name = "MARBLE_APP_URL", value = local.environment.frontend.url },
         { name = "SESSION_SECRET", value = local.environment.session.secret },
         { name = "SESSION_MAX_AGE", value = local.environment.session.max_age },
         { name = "LICENSE_KEY", value = local.environment.licence_key },
         { name = "SENTRY_ENVIRONMENT", value = local.environment.sentry.backend.env },
         { name = "SENTRY_DSN", value = local.environment.sentry.backend.dsn },
         { name = "SEGMENT_WRITE_KEY", value = local.environment.segment_write_key.backend },
+        { name = "CONVOY_API_URL", value = local.environment.convoy.url },
+        { name = "CONVOY_API_KEY", value = local.environment.convoy.key },
+        { name = "CONVOY_PROJECT_ID", value = local.environment.convoy.project_id },
         { name = "AUTHENTICATION_JWT_SIGNING_KEY", value = "${file("config/private.key")}" }
       ]
 
@@ -95,7 +98,7 @@ resource "aws_ecs_task_definition" "app" {
     {
       name      = "cron",
       image     = local.environment.backend.image,
-      essential = true,
+      essential = false,
 
       entryPoint : ["./app", "--worker"],
 
@@ -106,7 +109,7 @@ resource "aws_ecs_task_definition" "app" {
         { name = "PG_PORT", value = "${element(split(":", aws_db_instance.rds-marble.endpoint), 1)}" },
         { name = "PG_USER", value = "postgres" },
         { name = "PG_PASSWORD", value = "${random_string.rds-db-password.result}" },
-        # { name = "INGESTION_BUCKET_URL", value = "data-ingestion-bucket" },
+        { name = "INGESTION_BUCKET_URL", value = "-" },
         { name = "AWS_REGION", value = var.aws_region },
         { name = "AWS_ACCESS_KEY", value = var.aws_access_key_id },
         { name = "AWS_SECRET_KEY", value = var.aws_secret_access_key },

terraform_templates/AWS/locals_environments.tf

@@ -1,65 +1,3 @@
 locals {
-
-  environments = {
-
-    production = {
-      # TO CONFIGURE
-      firebase : {
-        // Copy the JSON Configuration File from Firebase Console
-        apiKey : "...",
-        authDomain : "...",
-        projectId : "...",
-        storageBucket : "...",
-        messagingSenderId : "...",
-        appId : "..."
-      }
-
-      licence_key = ""
-
-      session : {
-        secret  = "...", // Change It
-        max_age = "43200"
-      }
-
-      org : {
-        global = "...@...", // Gloabl Admin Email Address
-        name   = "...", // Organization name
-        admin  = "...@..." // Organization Admin Email Address
-      }
-
-      segment_write_key = {
-        frontend = "bEDdodQ5CBrUFeaHvVClSf0BfuWYyzeN",
-        backend  = "JeAT8VCKjBs7gVrFY23PG7aSMPqcvNFE"
-      }
-
-      sentry = {
-        frontend = {
-          dsn = "...",
-          env = "prod"
-        }
-        backend  = {
-          dsn  =  "...",
-          env   =  "prod"
-        }
-      }
-
-      frontend = {
-        image  = "europe-west1-docker.pkg.dev/marble-infra/marble/marble-frontend:latest"
-        domain = "..." // Your Application Domain (ex. app.xxx.xxx)
-        url    = "..." // Your Application URL (ex. https://marble-app.xxx.xxx)
-      }
-
-      backend = {
-        image              = "europe-west1-docker.pkg.dev/marble-infra/marble/marble-backend:latest"
-        domain             = "..." // Your API Domain (ex. api.xxx.xxx)
-        url                = "..." // Your API URL (ex. https://marble-api.xxx.xxx)
-        max_instance_count = 3
-      }
-
-      cron = {
-        s3 = "" // S3 for file ingestion ??
-      }
-    }
-
-  }
+  environments = jsondecode(file("${path.module}/config/environments.private.json")).environments
 }

terraform_templates/AWS/rds.tf

@@ -54,6 +54,9 @@ resource "aws_security_group" "rds" {
     protocol        = "tcp"
     security_groups = [aws_security_group.ecs_node_sg.id]
   }
+  tags                   = {
+    Name = "RiskTool - DB - Prod"
+  }
 }
 
 resource "aws_db_subnet_group" "marble_rds_subnet_group" {
@@ -68,13 +71,14 @@ resource "aws_db_subnet_group" "marble_rds_subnet_group" {
 resource "aws_db_instance" "rds-marble" {
 
   identifier                  = "rds-marble-${terraform.workspace}"
-  instance_class              = "db.t4g.small"
-  allocated_storage           = 10
+  instance_class              = "db.t4g.large"
+  allocated_storage           = 150
   engine                      = "postgres"
   engine_version              = "15"
   publicly_accessible         = true
   allow_major_version_upgrade = true
-
+  max_allocated_storage  = 3000 
+
   vpc_security_group_ids = [aws_security_group.rds.id]
   db_subnet_group_name   = aws_db_subnet_group.marble_rds_subnet_group.name
   parameter_group_name   = aws_db_parameter_group.pg-marble.name
@@ -96,6 +100,7 @@ resource "aws_db_instance" "rds-marble" {
 
   # Enable performance insights
   performance_insights_enabled = true
+  apply_immediately = true
 }
 
 

terraform_templates/AWS/vpc.tf

@@ -11,7 +11,7 @@ resource "aws_vpc" "main" {
   cidr_block           = "10.10.0.0/16"
   enable_dns_hostnames = true
   enable_dns_support   = true
-  tags                 = { Name = "marble-vpc" }
+  tags                 = { Name = "VPC - Marble" }
 }
 
 # Create Public Subnet 

terraform_templates/README.md

@@ -42,7 +42,83 @@ The GCP terraform files are largely inspired from Marble's internal cloud deploy
   - Create a folder named config (gitignored) under `terraform_templates/AWS`
   - Copy in this folder the file credentials.json (Downloaded from Google console and associated to your service account)
   - Genrate a RSA Private key (2048 Bits) and copy its content into a file named private.key
-- Run `terraform apply` in `terraform_templates/AWS`
+  - Copy all your environment variables and secrets in a file named `environments.private.json` in the `config` folder (see example structure below)
+
+#### Example: `config/environments.private.json`
+```json
+locals {
+
+  environments = {
+
+    production = {
+      # TO CONFIGURE
+      firebase : {
+        // Copy the JSON Configuration File from Firebase Console
+        apiKey : "...",
+        authDomain : "...",
+        projectId : "...",
+        storageBucket : "...",
+        messagingSenderId : "...",
+        appId : "..."
+      }
+
+      licence_key = ""
+
+      session : {
+        secret  = "...", // Change It
+        max_age = "43200"
+      }
+
+      org : {
+        global = "...@...", // Gloabl Admin Email Address
+        name   = "...", // Organization name
+        admin  = "...@..." // Organization Admin Email Address
+      }
+
+      segment_write_key = {
+        frontend = "bEDdodQ5CBrUFeaHvVClSf0BfuWYyzeN",
+        backend  = "JeAT8VCKjBs7gVrFY23PG7aSMPqcvNFE"
+      }
+
+      sentry = {
+        frontend = {
+          dsn = "...",
+          env = "prod"
+        }
+        backend  = {
+          dsn  =  "...",
+          env   =  "prod"
+        }
+      }
+
+      frontend = {
+        image  = "europe-west1-docker.pkg.dev/marble-infra/marble/marble-frontend:latest"
+        domain = "..." // Your Application Domain (ex. app.xxx.xxx)
+        url    = "..." // Your Application URL (ex. https://marble-app.xxx.xxx)
+      }
+
+      backend = {
+        image              = "europe-west1-docker.pkg.dev/marble-infra/marble/marble-backend:latest"
+        domain             = "..." // Your API Domain (ex. api.xxx.xxx)
+        url                = "..." // Your API URL (ex. https://marble-api.xxx.xxx)
+        max_instance_count = 3
+      }
+
+      cron = {
+        s3 = "" // S3 for file ingestion ??
+      }
+    }
+
+  }
+}
+
+```
+- Le fichier `locals_environments.tf` charge désormais ces données via :
+  ```hcl
+  locals {
+    environments = jsondecode(file("${path.module}/config/environments.private.json")).environments
+  }
+  ```
 
 At the end of the deployment, you should be able to access the application at the URL provided by terraform and set the domain as CNAME record both for your application domain and you api domain