Report external logging queue statistics to the container log

ascender/main/conf.py

@@ -689,6 +689,23 @@
     category=_('Logging'),
     category_slug='logging',
 )
+register(
+    'LOG_AGGREGATOR_ACTION_QUEUE_STATS_INTERVAL',
+    field_class=fields.IntegerField,
+    default=0,
+    min_value=0,
+    label=_('Interval (in seconds) between rsyslogd queue statistics'),
+    help_text=_(
+        'When greater than 0, rsyslogd writes its counters to its standard output, '
+        'which is the container log, once per interval: one JSON object per queue '
+        'and action, including the external logging action queue\'s current size, '
+        'peak size, and cumulative discarded.nf and discarded.full counts. '
+        'Equivalent to loading the rsyslogd impstats module. '
+        'The statistics are never sent to the log aggregator. 0 disables them (the default).'
+    ),
+    category=_('Logging'),
+    category_slug='logging',
+)
 register(
     'LOG_AGGREGATOR_MAX_DISK_USAGE_PATH',
     field_class=fields.CharField,

ascender/main/tests/unit/api/test_logger.py

@@ -202,6 +202,37 @@ def test_rsyslog_conf_template(enabled, log_type, host, port, protocol, errorfil
     assert expected_config in tmpl
 
 
+STATS_MODULE = 'module(load="impstats" interval="60" format="json" resetCounters="off" ruleset="ascender_queue_stats")'
+STATS_RULESET = 'ruleset(name="ascender_queue_stats") { action(type="omfile" file="/dev/stdout") }'
+
+
+@pytest.mark.parametrize(
+    'enabled, stats_interval, expect_stats',
+    [
+        (True, None, False),  # registered default (0): no impstats, configuration unchanged
+        (True, 0, False),
+        (True, 60, True),
+        (False, 60, False),  # no external action, so no queue to report on
+    ],
+)
+def test_rsyslog_conf_queue_stats(enabled, stats_interval, expect_stats):
+    mock_settings, _ = _mock_logging_defaults()
+
+    logging_defaults = getattr(settings, 'LOGGING')
+    setattr(mock_settings, 'LOGGING', logging_defaults)
+    setattr(mock_settings, 'LOG_AGGREGATOR_ENABLED', enabled)
+    setattr(mock_settings, 'LOG_AGGREGATOR_TYPE', 'other')
+    setattr(mock_settings, 'LOG_AGGREGATOR_HOST', 'localhost')
+    setattr(mock_settings, 'LOG_AGGREGATOR_PORT', 9000)
+    setattr(mock_settings, 'LOG_AGGREGATOR_PROTOCOL', 'tcp')
+    if stats_interval is not None:
+        setattr(mock_settings, 'LOG_AGGREGATOR_ACTION_QUEUE_STATS_INTERVAL', stats_interval)
+
+    lines = construct_rsyslog_conf_template(mock_settings).split('\n')
+    assert (STATS_MODULE in lines) is expect_stats
+    assert (STATS_RULESET in lines) is expect_stats
+
+
 def test_splunk_auth():
     mock_settings, _ = _mock_logging_defaults()
     # Set test settings

ascender/main/utils/external_logging.py

@@ -87,6 +87,7 @@ def construct_rsyslog_conf_template(settings=settings):
     max_disk_space_action_queue = getattr(settings, 'LOG_AGGREGATOR_ACTION_MAX_DISK_USAGE_GB', 1)
     spool_directory = getattr(settings, 'LOG_AGGREGATOR_MAX_DISK_USAGE_PATH', DEFAULT_SPOOL_DIRECTORY).rstrip('/')
     error_log_file = getattr(settings, 'LOG_AGGREGATOR_RSYSLOGD_ERROR_LOG_FILE', '')
+    stats_interval = getattr(settings, 'LOG_AGGREGATOR_ACTION_QUEUE_STATS_INTERVAL', 0)
 
     # Has to happen before queue.spoolDirectory is written below. The fallback used
     # to sit after it, where it changed a variable nothing read again, so a setting
@@ -130,6 +131,16 @@ def construct_rsyslog_conf_template(settings=settings):
             'template(name="ascender" type="string" string="%rawmsg-after-pri%")',
         ]
     )
+    # rsyslog already counts what the action queue discards (discarded.nf, discarded.full);
+    # impstats reports it, to rsyslogd's stdout, which is the container log. Its own ruleset
+    # keeps the statistics out of the external action, which sits in the default ruleset.
+    if enabled and stats_interval > 0:
+        parts.extend(
+            [
+                f'module(load="impstats" interval="{stats_interval}" format="json" resetCounters="off" ruleset="ascender_queue_stats")',
+                'ruleset(name="ascender_queue_stats") { action(type="omfile" file="/dev/stdout") }',
+            ]
+        )
 
     def escape_quotes(x):
         return x.replace('"', '\\"')