feat: add AS215932 extmon and diagnostics ops support

.gitignore

@@ -15,3 +15,6 @@ ansible/generated/**/knot.conf
 # Deploy-bracket Icinga snapshots are runtime state captured on the runner during
 # an apply (network-operations#149), not desired state — never commit them.
 ansible/generated/snapshots/
+
+# extmon agent env files carry bearer tokens; deployed to /etc/extmon by install.yml, never tracked
+ansible/generated/extmon/*.env

ansible/generated/extmon/alerts.yml

@@ -43,6 +43,81 @@ groups:
         annotations:
           summary: 'TLS cert expires in <7d: {{ $labels.instance }}'
 
+  - name: extmon-bgp
+    interval: 60s
+    rules:
+      - alert: AS215932PrefixNotVisible
+        expr: bgp_prefix_visible{prefix="2a0c:b641:b50::/44"} == 0
+        for: 5m
+        labels:
+          severity: critical
+          source: extmon
+        annotations:
+          summary: 'AS215932 prefix not visible from {{ $labels.source }}'
+          description: 'External BGP monitoring does not currently see 2a0c:b641:b50::/44.'
+
+      - alert: AS215932UnexpectedOrigin
+        expr: bgp_prefix_origin_visible{prefix="2a0c:b641:b50::/44",origin!="215932"} > 0
+        for: 2m
+        labels:
+          severity: critical
+          source: extmon
+        annotations:
+          summary: 'Unexpected origin AS for AS215932 prefix: {{ $labels.origin }}'
+
+      - alert: AS215932RPKIInvalid
+        expr: bgp_rpki_valid{prefix="2a0c:b641:b50::/44",origin="215932"} == 0
+        for: 10m
+        labels:
+          severity: critical
+          source: extmon
+        annotations:
+          summary: 'RPKI invalid/not-valid for 2a0c:b641:b50::/44 origin AS215932'
+
+      - alert: BGPSourceDown
+        expr: bgp_source_up == 0
+        for: 15m
+        labels:
+          severity: warning
+          source: extmon
+        annotations:
+          summary: 'BGP data source down/degraded: {{ $labels.source }}'
+
+      - alert: BGPSourceStale
+        expr: time() - bgp_source_last_success_timestamp > 1800
+        for: 10m
+        labels:
+          severity: warning
+          source: extmon
+        annotations:
+          summary: 'BGP data source stale: {{ $labels.source }}'
+
+      - alert: BGPalerterCriticalEvent
+        # Fire from the last-event timestamp gauge, not increase() on the
+        # counter: a counter's first sample for a new labelset has no prior
+        # point, so increase() would miss the very first hijack/withdrawal.
+        expr: time() - max(bgpalerter_last_event_timestamp{severity="critical"}) < 600
+        for: 0m
+        labels:
+          severity: critical
+          source: extmon
+        annotations:
+          summary: 'BGPalerter critical BGP event for AS215932'
+
+      - alert: ExtmonBGPAgentDown
+        expr: up{job="extmon-bgp-agent"} == 0
+        for: 5m
+        labels:
+          severity: warning
+          source: extmon
+        annotations:
+          summary: 'extmon BGP agent not serving /metrics'
+          description: |
+            up{job="extmon-bgp-agent"}=0 — the BGP monitoring agent is down, so
+            every bgp_prefix_*/bgp_source_*/BGPalerter series goes stale and the
+            BGP alerts above can no longer fire. This job is outside the
+            ExtmonComponentDown (extmon-self) matcher, hence its own rule.
+
   - name: extmon-self
     interval: 30s
     rules:
@@ -100,3 +175,21 @@ groups:
           source: extmon
         annotations:
           summary: 'OVH expiry collector has not run in over an hour'
+
+      - alert: OVHCollectorErrors
+        # The collector always writes its textfile (a fresh timestamp) even when
+        # OVH API calls fail, so OVHCollectorStale alone can't see a partial
+        # failure. This consumes the error gauge so missing expiry/billing
+        # coverage pages — e.g. the rtr failover IP whose expiry breaks NAT64.
+        expr: ovh_collector_errors_total > 0
+        for: 30m
+        labels:
+          severity: warning
+          source: extmon
+        annotations:
+          summary: 'OVH expiry collector reported {{ $value }} error(s)'
+          description: |
+            One or more OVH API calls (/dedicated, /ip, /me/bill) failed on the
+            last runs, so ovh_service_expires_seconds / ovh_unpaid_bill_count may
+            be missing or stale despite a fresh collector timestamp. Check OVH
+            credentials / API status before an expiry slips through unmonitored.

ansible/generated/extmon/bgpalerter-config.yml

@@ -0,0 +1,125 @@
+connectors:
+  - file: connectorRIS
+    name: ris
+    params:
+      carefulSubscription: true
+      url: ws://ris-live.ripe.net/v1/ws/
+      perMessageDeflate: true
+      authorizationHeader: null
+      subscription:
+        moreSpecific: true
+        type: UPDATE
+        socketOptions:
+          includeRaw: false
+
+monitors:
+  - file: monitorHijack
+    channel: hijack
+    name: basic-hijack-detection
+    params:
+      thresholdMinPeers: 3
+
+  - file: monitorPath
+    channel: path
+    name: path-matching
+    params:
+      thresholdMinPeers: 1
+
+  - file: monitorVisibility
+    channel: visibility
+    name: withdrawal-detection
+    params:
+      thresholdMinPeers: 40
+      notificationIntervalSeconds: 3600
+
+  - file: monitorAS
+    channel: misconfiguration
+    name: asn-monitor
+    params:
+      skipPrefixMatch: false
+      thresholdMinPeers: 3
+
+  - file: monitorRPKI
+    channel: rpki
+    name: rpki-monitor
+    params:
+      thresholdMinPeers: 3
+      checkUncovered: false
+      checkDisappearing: false
+
+  - file: monitorROAS
+    channel: roa
+    name: rpki-diff
+    params:
+      enableDiffAlerts: true
+      # ROA and TA *expiry* checks need a per-VRP `expires` field that the rpki
+      # provider below (Routinator's /json VRP export) does not supply — only an
+      # expiry-capable provider such as rpkiclient does. Leaving them on would
+      # advertise ROA-expiry coverage that can never fire, so they stay off until
+      # extmon has an expiry-capable VRP source. Diff + TA-deletion do not need
+      # expiry data and remain enabled.
+      enableExpirationAlerts: false
+      enableExpirationCheckTA: false
+      enableDeletedCheckTA: true
+      checkOnlyASns: true
+
+reports:
+  - file: reportFile
+    channels: [hijack, newprefix, visibility, path, misconfiguration, rpki, roa]
+    params:
+      persistAlertData: false
+      alertDataDirectory: alertdata/
+
+  - file: reportHTTP
+    channels: [hijack, newprefix, visibility, path, misconfiguration, rpki, roa]
+    params:
+      isTemplateJSON: true
+      templates:
+        default: '{"source":"bgpalerter","channel":"${channel}","type":"${type}","summary":"${summary}","prefix":"${prefix}","asn":"${asn}","newprefix":"${newprefix}","neworigin":"${neworigin}","earliest":"${earliest}","latest":"${latest}"}'
+      # Must stay 0: with isTemplateJSON, BGPalerter appends the top paths into
+      # ${summary} when showPaths > 0, then JSON.parse()s the interpolated body.
+      # The path blob is not JSON-escaped, so a path-bearing hijack/visibility/
+      # RPKI alert would fail to parse and be dropped before reaching the agent —
+      # silently losing exactly the critical alerts. The agent only needs
+      # channel/type/severity anyway; paths are not consumed.
+      showPaths: 0
+      hooks:
+        default: http://127.0.0.1:9188/bgpalerter
+
+processMonitors:
+  - file: uptimeApi
+    params:
+      useStatusCodes: true
+
+rest:
+  host: "127.0.0.1"
+  port: 8011
+
+monitoredPrefixesFiles:
+  - prefixes.yml
+
+logging:
+  directory: logs
+  logRotatePattern: YYYY-MM-DD
+  compressOnRotation: true
+  maxFileSizeMB: 15
+  maxRetainedFiles: 10
+  useUTC: true
+
+rpki:
+  vrpProvider: api
+  url: http://127.0.0.1:8323/json
+  refreshVrpListMinutes: 15
+  markDataAsStaleAfterMinutes: 120
+
+notificationIntervalSeconds: 1800
+checkForUpdatesAtBoot: false
+persistStatus: true
+alertOnlyOnce: false
+fadeOffSeconds: 360
+checkFadeOffGroupsSeconds: 30
+pidFile: bgpalerter.pid
+maxMessagesPerSecond: 6000
+multiProcess: false
+environment: production
+configVersion: 3

ansible/generated/extmon/bgpalerter-prefixes.yml

@@ -0,0 +1,10 @@
+2a0c:b641:b50::/44:
+  description: "AS215932 aggregate"
+  asn: 215932
+  ignoreMorespecifics: false
+  group: default
+
+options:
+  monitorASns:
+    215932:
+      group: default

ansible/generated/extmon/bgpalerter.service

@@ -0,0 +1,21 @@
+[Unit]
+Description=BGPalerter external BGP monitoring
+After=network-online.target extmon-bgp-agent.service
+Wants=network-online.target extmon-bgp-agent.service
+
+[Service]
+Type=simple
+User=prometheus
+Group=prometheus
+WorkingDirectory=/etc/bgpalerter
+ExecStart=/opt/bgpalerter/bgpalerter-linux-x64 -M -d /etc/bgpalerter/
+Restart=always
+RestartSec=20s
+NoNewPrivileges=true
+PrivateTmp=true
+ProtectSystem=strict
+ProtectHome=true
+ReadWritePaths=/etc/bgpalerter /var/lib/bgpalerter /tmp
+
+[Install]
+WantedBy=multi-user.target

ansible/generated/extmon/blackbox.yml

@@ -21,6 +21,17 @@ modules:
       tls_config:
         insecure_skip_verify: false
 
+  # Plain-HTTP loopback check for local REST endpoints (e.g. BGPalerter's
+  # /status on 127.0.0.1). Must NOT set fail_if_not_ssl — these are http://.
+  http_2xx_local:
+    prober: http
+    timeout: 10s
+    http:
+      preferred_ip_protocol: ip4
+      ip_protocol_fallback: false
+      valid_status_codes: [200, 204]
+      fail_if_not_ssl: false
+
   icmp_v4:
     prober: icmp
     timeout: 5s