fix(hosting): disable ClickHouse system-log telemetry tables and apply profile settings via users.d

ClickHouse's system log tables ship unbounded (no TTL); on the recommended
webapp machine size their background merges eventually exceed the memory cap
and are retried forever, pinning the CPU and failing the webapp's own inserts.
Port the dev stack's disable list (PR #3565) to hosting/docker, keep
query_log/error_log with a config-level TTL, and move the profile settings to
users.d where they actually take effect.

fixes #4343

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Author: Leafgard

.server-changes/hosting-clickhouse-system-logs.md

@@ -0,0 +1,6 @@
+---
+area: hosting
+type: fix
+---
+
+Self-hosted ClickHouse no longer accumulates unbounded system-log telemetry (metric_log, text_log, asynchronous_metric_log, ...) that eventually pins the CPU in a background merge-retry loop on the recommended machine size; query_log and error_log are kept with a TTL, and the low-memory profile settings now actually apply (moved from config.d to users.d).

hosting/docker/clickhouse/override.xml

@@ -9,12 +9,42 @@
     <!-- Official recommendations for systems with <16GB RAM -->
     <mark_cache_size>524288000</mark_cache_size> <!-- 500MB -->
     <concurrent_threads_soft_limit_num>1</concurrent_threads_soft_limit_num>
-    <profiles>
-        <default>
-        <max_block_size>8192</max_block_size>
-        <max_download_threads>1</max_download_threads>
-        <input_format_parallel_parsing>0</input_format_parallel_parsing>
-        <output_format_parallel_formatting>0</output_format_parallel_formatting>
-        </default>
-    </profiles>
-</clickhouse>
+
+    <!-- ClickHouse's own telemetry tables have no TTL and grow without bound; on the
+         recommended webapp machine size their background merges eventually stop fitting
+         in memory and are retried forever (there is no backoff for failed merges),
+         pinning the CPU and ultimately failing the webapp's own inserts (#4343).
+         The ClickHouse low-RAM guide recommends disabling exactly these tables:
+         https://clickhouse.com/docs/operations/tips
+         Same list as the dev stack (docker/config/clickhouse-disable-system-logs.xml),
+         extended with the newer log tables. -->
+    <metric_log remove="1"/>
+    <asynchronous_metric_log remove="1"/>
+    <part_log remove="1"/>
+    <trace_log remove="1"/>
+    <query_thread_log remove="1"/>
+    <session_log remove="1"/>
+    <text_log remove="1"/>
+    <zookeeper_log remove="1"/>
+    <processors_profile_log remove="1"/>
+    <asynchronous_insert_log remove="1"/>
+    <backup_log remove="1"/>
+    <latency_log remove="1"/>
+    <query_metric_log remove="1"/>
+    <opentelemetry_span_log remove="1"/>
+    <query_views_log remove="1"/>
+
+    <!-- Keep query_log and error_log (small and useful for debugging), but bounded.
+         A config-level TTL survives log-table recreation, unlike ALTER ... MODIFY TTL.
+         Note: ClickHouse renames the existing table to *_log_<N> when this changes;
+         drop those leftovers to reclaim disk. -->
+    <query_log>
+        <ttl>event_date + INTERVAL 7 DAY DELETE</ttl>
+    </query_log>
+    <error_log>
+        <ttl>event_date + INTERVAL 30 DAY DELETE</ttl>
+    </error_log>
+
+    <!-- Profile settings do NOT live here: config.d is silently ignored for them.
+         See users-override.xml, mounted under users.d. -->
+</clickhouse>

hosting/docker/clickhouse/users-override.xml

@@ -0,0 +1,21 @@
+<clickhouse>
+    <profiles>
+        <default>
+            <!-- Low-memory settings for the recommended webapp machine size. These were
+                 previously in config.d/override.xml where profile settings are silently
+                 ignored — they only take effect from the users config tree (#4343). -->
+            <max_block_size>8192</max_block_size>
+            <max_download_threads>1</max_download_threads>
+            <input_format_parallel_parsing>0</input_format_parallel_parsing>
+            <output_format_parallel_formatting>0</output_format_parallel_formatting>
+            <!-- The memory/query profilers sample stacks into system.trace_log
+                 (a Memory/MemoryPeak sample every 4 MB allocated, by default) — the main
+                 firehose that fills it. trace_log is disabled in override.xml; turn the
+                 samplers off too so they don't burn CPU producing discarded rows. -->
+            <memory_profiler_step>0</memory_profiler_step>
+            <memory_profiler_sample_probability>0</memory_profiler_sample_probability>
+            <query_profiler_real_time_period_ns>0</query_profiler_real_time_period_ns>
+            <query_profiler_cpu_time_period_ns>0</query_profiler_cpu_time_period_ns>
+        </default>
+    </profiles>
+</clickhouse>

hosting/docker/webapp/docker-compose.yml

@@ -177,6 +177,7 @@ services:
       - clickhouse:/var/lib/clickhouse
       - ../clickhouse/data-paths.xml:/etc/clickhouse-server/config.d/data-paths.xml:ro
       - ../clickhouse/override.xml:/etc/clickhouse-server/config.d/override.xml:ro
+      - ../clickhouse/users-override.xml:/etc/clickhouse-server/users.d/override.xml:ro
     networks:
       - webapp
     healthcheck: