thunderbird · ryanjjung · Apr 3, 2026 · Apr 6, 2026 · Apr 6, 2026 · Apr 7, 2026
diff --git a/fluent-bit/filesystem/etc/fluent-bit/fluent-bit.yaml b/fluent-bit/filesystem/etc/fluent-bit/fluent-bit.yaml
@@ -18,6 +18,11 @@ pipeline:
       tag: untagged
       port: 1337
       threaded: true
+
+    # Log shippers should override the tag with the name of the application
+    - name: forward
+      tag: cloudwatch.untagged
+      port: 24224
 
   filters:
     # Run a Lua script that breaks all events in a single Stalwart telemetry payload into multiple
@@ -45,3 +50,26 @@ pipeline:
       tls: on
       uri: /batch
       format: json_lines
+
+    # Send logs onward to CloudWatch. Log groups by the derived name must pre-exist, and this
+    # service must have sufficient IAM permissions to create log streams and post events to them.
+    - name: cloudwatch_logs
+      match: cloudwatch.stalwart.mail
+      log_group_name: /tb/${ENV}/stalwart
+      log_stream_name: mail
+      region: eu-central-1
+      # log_key: systemd message field?
+
+    - name: cloudwatch_logs
+      match: cloudwatch.stalwart.api
+      log_group_name: /tb/${ENV}/stalwart
+      log_stream_name: api
+      region: eu-central-1
+      # log_key: systemd message field?
+
+    - name: cloudwatch_logs
+      match: cloudwatch.untagged
+      log_group_name: /tb/${ENV}/observability
+      log_stream_name: untagged
+      region: eu-central-1
+      # log_key: systemd message field?
diff --git a/pulumi/Pulumi.dev.yaml b/pulumi/Pulumi.dev.yaml
@@ -1,3 +1,7 @@
 config:
   observability:posthog_api_key:
     secure: AAABAACLeD5lasJAmY66NyJXtacSmTSMj/PiXtmBNIHeBfLx2HA3mhTzyWkPZnD9j8MCYPbtnjJiWeZBzOROWVKEcKpuysV/FV5CDoHCJg==
+  observability:cloudflare_zone_id:
+    secure: AAABAKragv0vFq2i/lBhwJRTkD/wjW8jefzGy6Mq5A4eZubZLEeh4cSFESB+M3Fv34TvYNxJpFlT208EMqUQLw==
+  cloudflare:apiToken:
+    secure: AAABAJfLM7HTgF++SR/ps+pkQQFMNxc0XyRidPcCJKD2nzpc9mRnqdEguDnJlKKwtStygHkRT95D/6n568y+TUf/hkGut6P5
diff --git a/pulumi/__main__.py b/pulumi/__main__.py
@@ -9,7 +9,11 @@
 of any of those larger infrastructure patterns.
 """
 
+import pulumi_aws as aws
+import pulumi_cloudflare as cloudflare
 import tb_pulumi
+import tb_pulumi.cloudwatch
+import tb_pulumi.iam
 import tb_pulumi.fargate
 import tb_pulumi.network
 import tb_pulumi.secrets
@@ -36,6 +40,34 @@
         **psm_opts,
     )
 
+    logdest_opts = resources.get('tb:cloudwatch:LogDestination', {})
+    logdests = {
+        logdest_name: tb_pulumi.cloudwatch.LogDestination(
+            f'{project.name_prefix}-logdest-{logdest_name}',
+            project=project,
+            **logdest_config,
+        )
+        for logdest_name, logdest_config in logdest_opts.items()
+    }
+
+    fluentbit_user = aws.iam.User(
+        f'{project.name_prefix}-user-fluentbit',
+        name=f'observability-{project.stack}-fluentbit',
+        tags=project.common_tags,
+    )
+
+    aws.iam.UserPolicyAttachment(
+        f'{project.name_prefix}-upa-obsv-logwrite',
+        policy_arn=logdests['observability'].resources['iam_policies']['write'],
+        user=fluentbit_user.id,
+    )
+
+    aws.iam.UserPolicyAttachment(
+        f'{project.name_prefix}-upa-mailstrom-logwrite',
+        policy_arn=f'arn:aws:iam::768512802988:policy/mailstrom-{project.stack}-stalwart-logs-write-access',
+        user=fluentbit_user.id,
+    )
+
     vpc_config = resources.get('tb:network:MultiCidrVpc', {}).get('fluentbit', {})
     vpc_fluentbit = tb_pulumi.network.MultiCidrVpc(
         f'{project.name_prefix}-vpc-fluentbit',
@@ -54,3 +86,13 @@
             'tb:fargate:AutoscalingFargateCluster'
         ).items()
     }
+
+    cloudflare_zone_id = project.pulumi_config.require_secret('cloudflare_zone_id')
+    fluent_bit_dns = cloudflare.DnsRecord(
+        f'{project.name_prefix}-dns-fluentbit',
+        name='fluentbit' if project.stack == 'prod' else f'fluentbit-{project.stack}',
+        content=ecs_clusters['fluentbit'].resources['load_balancers']['fluentbit'].dns_name,
+        ttl=60,
+        type='CNAME',
+        zone_id=cloudflare_zone_id,
+    )
diff --git a/pulumi/config.dev.yaml b/pulumi/config.dev.yaml
@@ -10,6 +10,14 @@ resources:
       secret_names:
         - posthog_api_key
 
+  tb:cloudwatch:LogDestination:
+    observability:
+      log_group:
+        retention_in_days: 7
+      log_streams:
+        untagged: untagged
+      org_name: tb
+
   tb:network:MultiCidrVpc:
     fluentbit:
       # The observability project has all of 10.202.0.0/16 assigned to it, but let's not soak all
@@ -30,22 +38,28 @@ resources:
         - secretsmanager
       additional_routes:
         - destination_cidr_block: 10.2.0.0/16  # mailstrom-dev
-          vpc_peering_connection_id: pcx-0d2027442f0e54ca4
-  
+          vpc_peering_connection_id: pcx-04d7e54008cd9326c
+
   tb:fargate:AutoscalingFargateCluster:
     fluentbit:
       cluster: {}
 
       container_security_groups:
-        fluentbit:
-          fluentbit-http:
+        fluentbit: # Service
+          fluentbit: # Load Balancer
             rules:
               ingress:
-                - description: Allow traffic from the load balancer to the container
+                # Maddeningly, apostrophes aren't allowed in descriptions
+                - description: Allow traffic from the load balancer to the containers HTTP service
                   # Sources are set in code
                   protocol: tcp
                   from_port: 1337
                   to_port: 1337
+                - description: Allow traffic from the load balancer to the containers log forwarding service
+                  # Sources are set in code
+                  protocol: tcp
+                  from_port: 24224
+                  to_port: 24224
               egress:
                 - description: Allow traffic from the container out to the Internet
                   protocol: tcp
@@ -61,7 +75,7 @@ resources:
       ssm_params: {}
 
       task_definitions:
-        fluentbit:
+        fluentbit: # Service
           container_definitions:
             - name: fluentbit
               environment:
@@ -78,16 +92,18 @@ resources:
               secrets:
                 - name: POSTHOG_API_KEY
                   valueFrom: arn:aws:secretsmanager:eu-central-1:768512802988:secret:observability/dev/posthog_api_key-e3UEK4
-              image: 768512802988.dkr.ecr.eu-central-1.amazonaws.com/thunderbird/fluent-bit:fdd1b4748cfaee29553ee2c83fcaa428b68ba8e88c2791e1626e282b48127b9d
+              image: 768512802988.dkr.ecr.eu-central-1.amazonaws.com/thunderbird/fluent-bit:0bc8e6cdee4226f9c090ca22b08b3f6c61a5456a2090669414a43c52dd2580a8
               logConfiguration:
                 logDriver: awslogs
                 options:
-                  awslogs-group: observability-dev-fargate-fluentbit-loggroup-fluentbit
+                  awslogs-group: /tb/dev/observability
                   awslogs-region: eu-central-1
-                  awslogs-stream-prefix: observability/dev/fluentbit/
+                  awslogs-stream-prefix: 'ecs'
               portMappings:
                 - containerPort: 1337
                   protocol: tcp
+                - containerPort: 24224
+                  protocol: tcp
               restartPolicy:
                 enabled: yes
                 restartAttemptPeriod: 300
@@ -99,16 +115,22 @@ resources:
             - FARGATE
 
       load_balancer_security_groups:
-        fluentbit-http:
-          description: Governs access to the fluent-bit-http load balancer in dev
+        fluentbit: # Load Balancer
+          description: Governs access to the fluentbit-http load balancer in dev
           rules:
             ingress:
               - from_port: 443
                 to_port: 443
                 protocol: tcp
                 cidr_blocks:
                   - 10.2.0.0/16  # stalwart-dev
-                description: Allow access from stalwart-dev
+                description: Allow access to telemetry forwarding service from stalwart-dev
+              - from_port: 24224
+                to_port: 24224
+                protocol: tcp
+                cidr_blocks:
+                  - 10.2.0.0/16  # stalwart-dev
+                description: Allow access to log forwarding service from stalwart-dev
             egress:
               - from_port: 0
                 to_port: 65535
@@ -117,49 +139,64 @@ resources:
                   - 0.0.0.0/0
 
       load_balancers:
-        fluentbit-http:
+        fluentbit:
           enable_cross_zone_load_balancing: yes
           internal: yes
           ip_address_type: ipv4
-          load_balancer_type: application
-          name: fluentbit-http-dev
+          load_balancer_type: network
+          name: obsv-dev-fluentbit
           preserve_host_header: yes
 
       targets:
+        cwlogs:
+          name: obsv-dev-fluent-logs
+          health_check:
+            port: 24224
+            protocol: TCP
+          port: 24224
+          protocol: TCP
+          target_type: ip
+          ip_address_type: ipv4
         stalwart-metrics:
-          name: dev-telemetry-http
+          name: obsv-dev-fluent-telemetry
           health_check:
-            protocol: HTTP
-            # fluentbit is set up to throw away records submitted to this endpoint
-            path: /health/check
             port: 1337
-            # I wish it wasn't like this. fluentbit will not accept non-POST methods, but these health checks cannot be
-            # made with custom methods. Therefore, we expect a 400 Bad Request as a sign of health. *shrug*
-            matcher: "400"
+            protocol: TCP
           port: 1337
-          protocol: HTTP
-          # Next two options are required for ECS services; ref:
-          # https://docs.aws.amazon.com/AmazonECS/latest/developerguide/alb.html
+          protocol: TCP
           target_type: ip
           ip_address_type: ipv4
 
       listeners:
-        fluentbit-http:
-          stalwart-metrics:
+        fluentbit: # Load Balancer
+          cwlogs: # Target
+            port: 24224
+            protocol: TCP
+          stalwart-metrics: # Target
+            port: 443
+            protocol: TLS
             # This cert is for fluentbit-dev.tb.pro
             certificate_arn: arn:aws:acm:eu-central-1:768512802988:certificate/04dd0573-a3cc-4c19-b483-a868876c63b0
-            port: 443
-            protocol: HTTPS
 
       services:
-        fluentbit:
+        fluentbit: # Service
           assign_public_ip: yes
-          container_name: fluentbit
+          container_name: fluentbit  # Name from container definition
           container_port: 1337
-          load_balancer: fluentbit-http
+          load_balancer: fluentbit
           service:
             desired_count: 2
-          target: stalwart-metrics
+          targets:
+            - container_name: fluentbit
+              container_port: 24224
+              target_name: cwlogs
+            - container_name: fluentbit
+              container_port: 1337
+              target_name: stalwart-metrics
+
+      extra_policies:
+        fluentbit:
+          - arn:aws:iam::768512802988:policy/observability-dev-observability-logs-write-access
 
       autoscalers:
         fluentbit:

diff --git a/pulumi/requirements.txt b/pulumi/requirements.txt
@@ -1,3 +1,5 @@
 requests>=2.32.5
-tb_pulumi @ git+https://github.com/thunderbird/pulumi.git@main
+pulumi_cloudflare>=6.14.0,<7
+# tb_pulumi @ git+https://github.com/thunderbird/pulumi.git@main
+-e /home/rjung/workspace/thunderbird/pulumi
 sdks/site24x7