Stage dedicated Agent Mail infrastructure

.github/workflows/apply.yml

@@ -41,10 +41,11 @@ on:
           - rtr_routing
           - networkd_resolved
           - mail_openbsd
+          - agent_mail
           - freebsd_resolv
           - noc_mcp_key
       limit:
-        description: Ansible --limit pattern (host or group)
+        description: Ansible --limit pattern; empty safely defaults to all:!ci-pr:!staged
         type: string
         required: true
         default: ""
@@ -117,6 +118,8 @@ jobs:
       # New inventory hosts become reachable without a manual `apply.yml ci`
       # known_hosts reseed (network-operations#404).
       - name: Seed missing SSH host keys
+        env:
+          SEED_HOST_KEYS_LIMIT: ${{ inputs.limit != '' && inputs.limit || 'all:!ci-pr:!staged' }}
         run: scripts/ci/seed-missing-host-keys.sh
 
       - name: Source Vault-rendered secrets
@@ -141,6 +144,21 @@ jobs:
             echo "::warning::/etc/github-runner/secrets.env not found — apply may fail on env-var lookups"
           fi
 
+      - name: Validate Agent Mail runner secrets
+        if: ${{ !inputs.dry_run && inputs.playbook == 'agent_mail' }}
+        run: |
+          set -euo pipefail
+          missing=()
+          for key in AGENT_MAIL_DNS_TSIG_SECRET AGENT_MAIL_WEBHOOK_SECRET; do
+            if [ -z "${!key-}" ]; then
+              missing+=("$key")
+            fi
+          done
+          if [ "${#missing[@]}" -ne 0 ]; then
+            echo "::error::Agent Mail apply is missing Vault-rendered runner values: ${missing[*]}"
+            exit 1
+          fi
+
       - name: Summarize app version pins
         if: ${{ contains(fromJSON('["noc","engineering-loop","cloud","web","network-proxy"]'), inputs.playbook) }}
         env:
@@ -461,14 +479,11 @@ jobs:
           PLAYBOOK: ${{ inputs.playbook }}
           LIMIT: ${{ inputs.limit }}
         run: |
-          limit_args=()
-          if [ -n "$LIMIT" ]; then
-            limit_args=(--limit "$LIMIT")
-          fi
+          effective_limit="${LIMIT:-all:!ci-pr:!staged}"
           ansible-playbook "playbooks/${PLAYBOOK}.yml" \
             --tags validate \
             --connection=local \
-            "${limit_args[@]}"
+            --limit "$effective_limit"
 
       # -e ansible_user=ci: the runner connects as the dedicated `ci` deploy
       # user on every host (created + NOPASSWD-root by the ci_runner_key role)
@@ -484,10 +499,7 @@ jobs:
         run: |
           playbook="$PLAYBOOK"
           user_args=(-e ansible_user=ci)
-          limit_args=()
-          if [ -n "$LIMIT" ]; then
-            limit_args=(--limit "$LIMIT")
-          fi
+          effective_limit="${LIMIT:-all:!ci-pr:!staged}"
           if [ "$playbook" = "ci-runner-key" ] && [ "$BOOTSTRAP_CI_RUNNER_KEY" = "true" ]; then
             user_args=()
           fi
@@ -500,7 +512,7 @@ jobs:
             "${user_args[@]}" \
             -e "${APPLY_VAR}" \
             "${extra_var_args[@]}" \
-            "${limit_args[@]}"
+            --limit "$effective_limit"
 
       - name: Post-deploy Goss validation
         if: ${{ !inputs.dry_run }}

ansible/generated/agentmail/agent-mail-backup

@@ -0,0 +1,123 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+umask 077
+readonly compose_file="/opt/agent-mail/docker-compose.yml"
+readonly backup_dir="/mnt/agent-mail-backup"
+readonly lock_file="/run/lock/agent-mail-backup.lock"
+readonly min_capacity_bytes="107374182400"
+readonly min_free_bytes="34359738368"
+readonly metrics_file="/var/lib/node_exporter/textfile_collector/agent-mail-backup.prom"
+
+exec 9>"$lock_file"
+flock -n 9 || exit 0
+
+if ! mountpoint --quiet "$backup_dir"; then
+  echo "Agent Mail backup directory is not a dedicated mount: $backup_dir" >&2
+  exit 1
+fi
+
+if ! backup_device="$(stat --file-system --format=%i "$backup_dir")" ||
+   ! root_device="$(stat --file-system --format=%i /)" ||
+   ! data_device="$(stat --file-system --format=%i /var/lib/stalwart)"; then
+  echo "Agent Mail backup filesystem identity could not be verified" >&2
+  exit 1
+fi
+if [[ "$backup_device" == "$root_device" ]] ||
+   [[ "$backup_device" == "$data_device" ]]; then
+  echo "Agent Mail backup directory must use a filesystem distinct from root and Stalwart data" >&2
+  exit 1
+fi
+
+# The mount is sized for at most three full snapshots during the two-day
+# retention window (two retained archives plus the one being created). Clean
+# expired archives and abandoned partials before checking capacity.
+find "$backup_dir" -type f \
+  \( -name 'stalwart-*.tar.zst' -o -name 'stalwart-*.tar.zst.sha256' \) \
+  -mtime +2 -delete
+find "$backup_dir" -type f -name 'stalwart-*.tar.zst.partial' -delete
+
+read -r capacity_bytes available_bytes < <(
+  LC_ALL=C df --output=size,avail --block-size=1 "$backup_dir" |
+    awk 'NR == 2 { print $1, $2 }'
+)
+if [[ ! "$capacity_bytes" =~ ^[0-9]+$ ]] ||
+   [[ ! "$available_bytes" =~ ^[0-9]+$ ]] ||
+   (( capacity_bytes < min_capacity_bytes )) ||
+   (( available_bytes < min_free_bytes )); then
+  echo "Agent Mail backup volume lacks required capacity/free space" >&2
+  exit 1
+fi
+
+timestamp="$(date -u +%Y%m%dT%H%M%SZ)"
+partial="$backup_dir/stalwart-$timestamp.tar.zst.partial"
+archive="$backup_dir/stalwart-$timestamp.tar.zst"
+was_active=0
+
+container_ids=""
+if ! container_ids="$(docker compose --file "$compose_file" ps --all --quiet stalwart)"; then
+  echo "Agent Mail could not enumerate Stalwart containers; refusing an unsafe backup" >&2
+  exit 1
+fi
+while IFS= read -r container_id; do
+  [[ -n "$container_id" ]] || continue
+  if ! container_state="$(docker inspect --format '{{.State.Status}}' "$container_id")"; then
+    echo "Agent Mail could not inspect Stalwart container state; refusing an unsafe backup" >&2
+    exit 1
+  fi
+  case "$container_state" in
+    created|exited|dead) ;;
+    *) was_active=1 ;;
+  esac
+done <<<"$container_ids"
+if [[ "$was_active" -eq 1 ]]; then
+  docker compose --file "$compose_file" stop --timeout 120 stalwart
+fi
+
+write_success_metric() {
+  local completed_at metric_tmp
+  completed_at="$(date +%s)"
+  metric_tmp="${metrics_file}.tmp.$$"
+  {
+    echo '# HELP agent_mail_backup_last_success_timestamp_seconds Unix timestamp of the last completed quiesced Agent Mail backup.'
+    echo '# TYPE agent_mail_backup_last_success_timestamp_seconds gauge'
+    printf 'agent_mail_backup_last_success_timestamp_seconds %s\n' "$completed_at"
+  } >"$metric_tmp"
+  chmod 0644 "$metric_tmp"
+  mv -f -- "$metric_tmp" "$metrics_file"
+}
+
+finish_backup() {
+  status=$?
+  if [[ "$status" -ne 0 ]]; then
+    rm -f -- "$partial"
+  fi
+  if [[ "$was_active" -eq 1 ]]; then
+    if ! docker compose --file "$compose_file" up --detach stalwart; then
+      echo "Agent Mail backup could not restart Stalwart" >&2
+      status=1
+    fi
+  fi
+  if [[ "$status" -eq 0 ]] && ! write_success_metric; then
+    echo "Agent Mail backup completed but its success metric could not be recorded" >&2
+    status=1
+  fi
+  trap - EXIT
+  exit "$status"
+}
+trap finish_backup EXIT
+
+# RocksDB must be quiescent. Preserve numeric UID/GID, xattrs, and ACLs so a
+# restore can recreate both the Stalwart configuration and data directories.
+tar --acls --xattrs --numeric-owner --zstd -C / \
+  -cf "$partial" \
+  etc/stalwart var/lib/stalwart
+mv "$partial" "$archive"
+archive_name="${archive##*/}"
+(
+  cd "$backup_dir"
+  sha256sum "$archive_name" > "$archive_name.sha256"
+)
+
+# A dedicated-volume snapshot is not launch-ready by itself. Public readiness
+# separately requires an off-host copy and a successfully rehearsed restore.

ansible/generated/agentmail/agent-mail-backup.service

@@ -0,0 +1,15 @@
+[Unit]
+Description=Quiesced Stalwart Agent Mail backup
+Requires=docker.service
+RequiresMountsFor=/mnt/agent-mail-backup
+After=docker.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/sbin/agent-mail-backup
+TimeoutStartSec=infinity
+Nice=10
+IOSchedulingClass=best-effort
+IOSchedulingPriority=7
+PrivateTmp=true
+ProtectHome=true

ansible/generated/agentmail/agent-mail-backup.timer

@@ -0,0 +1,11 @@
+[Unit]
+Description=Daily Stalwart Agent Mail backup
+
+[Timer]
+OnCalendar=*-*-* 02:30:00 UTC
+RandomizedDelaySec=20m
+Persistent=true
+Unit=agent-mail-backup.service
+
+[Install]
+WantedBy=timers.target

ansible/generated/agentmail/agent-mail.env.example

@@ -0,0 +1,4 @@
+# Review artifact only — never accepted as a runtime secrets file.
+STALWART_PUBLIC_URL=https://mx1.agentmail.hyrule.host
+STALWART_DNS_TSIG_SECRET='<from-vault>'
+STALWART_WEBHOOK_SECRET='<from-vault>'

ansible/generated/agentmail/bootstrap.json

@@ -0,0 +1 @@
+{"using":["urn:ietf:params:jmap:core","urn:stalwart:jmap"],"methodCalls":[["x:Bootstrap/set",{"update":{"singleton":{"serverHostname":"mx1.agentmail.hyrule.host","defaultDomain":"agentmail.hyrule.host","requestTlsCertificate":true,"generateDkimKeys":true,"dataStore":{"@type":"RocksDb","path":"/var/lib/stalwart/"},"blobStore":{"@type":"Default"},"searchStore":{"@type":"Default"},"inMemoryStore":{"@type":"Default"},"directory":{"@type":"Internal"},"tracer":{"@type":"Stdout","ansi":false,"multiline":false,"buffered":true,"enable":true,"level":"info","lossy":false,"events":[],"eventsPolicy":"exclude"},"dnsServer":{"@type":"Tsig","host":"2a0c:b641:b50:2::10","port":53,"keyName":"hyrule-dns","key":{"@type":"EnvironmentVariable","variableName":"STALWART_DNS_TSIG_SECRET"},"protocol":"tcp","tsigAlgorithm":"hmac-sha256","description":"Hyrule Knot RFC2136","timeout":"30s","ttl":"5m","pollingInterval":"15s","propagationTimeout":"2m"}}}},"bootstrap"]]}

ansible/generated/agentmail/desired-state.ndjson

@@ -0,0 +1,3 @@
+{"@type":"update","object":"DataRetention","value":{"expungeTrashAfter":"30d","holdMtaReportsFor":"30d","archiveDeletedItemsFor":null,"archiveDeletedAccountsFor":null}}
+{"@type":"update","object":"Metrics","value":{"openTelemetry":{"@type":"Disabled"},"prometheus":{"@type":"Enabled","authSecret":{"@type":"None"},"authUsername":null},"metrics":[],"metricsPolicy":"exclude"}}
+{"@type":"upsert","object":"WebHook","matchOn":["url"],"value":{"agent-mail-events":{"url":"https://cloud.hyrule.host/v1/internal/mail/events","signatureKey":{"@type":"EnvironmentVariable","variableName":"STALWART_WEBHOOK_SECRET"},"httpAuth":{"@type":"Unauthenticated"},"httpHeaders":{},"events":["message-ingest.ham", "message-ingest.spam", "message-ingest.imap-append", "message-ingest.jmap-append", "message-ingest.duplicate", "message-ingest.error", "delivery.delivered", "delivery.failed", "delivery.completed", "delivery.rcpt-to-rejected", "delivery.message-rejected", "delivery.dsn-temp-fail", "delivery.dsn-perm-fail", "incoming-report.abuse-report", "incoming-report.fraud-report", "incoming-report.virus-report"],"eventsPolicy":"include","timeout":"10s","throttle":"1s","discardAfter":"1h","allowInvalidCerts":false,"enable":true,"level":"info","lossy":false}}}

ansible/generated/agentmail/docker-compose.yml

@@ -0,0 +1,29 @@
+# Managed by Ansible (agent_mail). Public listeners are inventory-gated.
+services:
+  stalwart:
+    image: "stalwartlabs/stalwart:v0.16.4@sha256:c8aee803933a643558a9afaa3c208d4175a4ac09884f555b821aa5df1e89230c"
+    container_name: agent-mail-stalwart
+    restart: unless-stopped
+    stop_grace_period: 2m
+    env_file:
+      - "/etc/agent-mail/agent-mail.env"
+    volumes:
+      - "/etc/stalwart:/etc/stalwart"
+      - "/var/lib/stalwart:/var/lib/stalwart"
+    ports:
+      - "[2a0c:b641:b50:2::110]:443:443/tcp"
+    logging:
+      driver: journald
+      options:
+        tag: agent-mail-stalwart
+
+networks:
+  default:
+    name: agent-mail
+    driver: bridge
+    enable_ipv6: true
+    driver_opts:
+      com.docker.network.bridge.name: "br-agentmail"
+    ipam:
+      config:
+        - subnet: "fd21:5932:110::/64"

ansible/generated/agentmail/icinga_host.conf

@@ -0,0 +1,71 @@
+// /etc/icinga2/conf.d/hosts/ansible/agentmail.conf
+// Managed by ansible/roles/monitoring — do not edit by hand.
+// Source of truth: ansible/inventory/host_vars/agentmail.yml
+
+object Host "agentmail" {
+  address6 = "2a0c:b641:b50:2::110"
+  display_name = "agentmail (Stalwart, API-only)"
+
+  check_command = "ping6"
+
+  vars.os = "Debian"
+  vars.role = "agent-mail"
+  vars.ssh_port = 22
+
+  vars.prom_instance_node = "[2a0c:b641:b50:2::110]:9100"
+
+  vars.disks["disk /"] = { mountpoint = "/" }
+  vars.disks["disk /mnt/agent-mail-backup"] = { mountpoint = "/mnt/agent-mail-backup" }
+}
+
+object Service "stalwart-ready" {
+  host_name = "agentmail"
+  check_command = "http"
+  check_interval = 1m
+  retry_interval = 30s
+  max_check_attempts = 5
+  notes = "Stalwart readiness over the private HTTPS/JMAP listener"
+  vars.http_address = "2a0c:b641:b50:2::110"
+  vars.http_port = 443
+  vars.http_uri = "/healthz/ready"
+  vars.http_vhost = "mx1.agentmail.hyrule.host"
+  vars.http_ssl = true
+  vars.http_sni = true
+  vars.http_ipv6 = true
+  vars.http_timeout = 10
+}
+
+object Service "agent-mail-backup-timer" {
+  host_name = "agentmail"
+  check_command = "prom_systemd_unit"
+  check_interval = 5m
+  retry_interval = 1m
+  max_check_attempts = 3
+  notes = "Agent Mail quiesced-backup timer is active"
+  vars.prom_instance = "[2a0c:b641:b50:2::110]:9100"
+  vars.systemd_unit = "agent-mail-backup.timer"
+}
+
+object Service "agent-mail-backup-service" {
+  host_name = "agentmail"
+  check_command = "prom_systemd_not_failed"
+  check_interval = 5m
+  retry_interval = 1m
+  max_check_attempts = 3
+  notes = "Agent Mail backup oneshot has no failed systemd state"
+  vars.prom_instance = "[2a0c:b641:b50:2::110]:9100"
+  vars.systemd_unit = "agent-mail-backup.service"
+}
+
+object Service "agent-mail-backup-freshness" {
+  host_name = "agentmail"
+  check_command = "prom_agent_mail_backup_freshness"
+  check_interval = 5m
+  retry_interval = 1m
+  max_check_attempts = 3
+  notes = "Latest successful Agent Mail backup is recent"
+  vars.prom_instance = "[2a0c:b641:b50:2::110]:9100"
+  vars.backup_warn_seconds = 93600
+  vars.backup_crit_seconds = 129600
+}
+