From 1b7d797087e7c64492dbf84b907406e3ac58cee1 Mon Sep 17 00:00:00 2001
From: Yann SEGET <yann.seget@actemium.ch>
Date: Mon, 10 Aug 2026 08:17:45 +0200
Subject: [PATCH 1/4] fix(hosting): disable ClickHouse system-log telemetry
 tables and apply profile settings via users.d

ClickHouse's system log tables ship unbounded (no TTL); on the recommended
webapp machine size their background merges eventually exceed the memory cap
and are retried forever, pinning the CPU and failing the webapp's own inserts.
Port the dev stack's disable list (PR #3565) to hosting/docker, keep
query_log/error_log with a config-level TTL, and move the profile settings to
users.d where they actually take effect.

fixes #4343

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 .../hosting-clickhouse-system-logs.md         |  6 +++
 hosting/docker/clickhouse/override.xml        | 48 +++++++++++++++----
 hosting/docker/clickhouse/users-override.xml  | 21 ++++++++
 hosting/docker/webapp/docker-compose.yml      |  1 +
 4 files changed, 67 insertions(+), 9 deletions(-)
 create mode 100644 .server-changes/hosting-clickhouse-system-logs.md
 create mode 100644 hosting/docker/clickhouse/users-override.xml

diff --git a/.server-changes/hosting-clickhouse-system-logs.md b/.server-changes/hosting-clickhouse-system-logs.md
new file mode 100644
index 00000000000..efd4166ae51
--- /dev/null
+++ b/.server-changes/hosting-clickhouse-system-logs.md
@@ -0,0 +1,6 @@
+---
+area: hosting
+type: fix
+---
+
+Self-hosted ClickHouse no longer accumulates unbounded system-log telemetry (metric_log, text_log, asynchronous_metric_log, ...) that eventually pins the CPU in a background merge-retry loop on the recommended machine size; query_log and error_log are kept with a TTL, and the low-memory profile settings now actually apply (moved from config.d to users.d).
diff --git a/hosting/docker/clickhouse/override.xml b/hosting/docker/clickhouse/override.xml
index 41897c29849..f9088054a7e 100644
--- a/hosting/docker/clickhouse/override.xml
+++ b/hosting/docker/clickhouse/override.xml
@@ -9,12 +9,42 @@
     <!-- Official recommendations for systems with <16GB RAM -->
     <mark_cache_size>524288000</mark_cache_size> <!-- 500MB -->
     <concurrent_threads_soft_limit_num>1</concurrent_threads_soft_limit_num>
-    <profiles>
-        <default>
-        <max_block_size>8192</max_block_size>
-        <max_download_threads>1</max_download_threads>
-        <input_format_parallel_parsing>0</input_format_parallel_parsing>
-        <output_format_parallel_formatting>0</output_format_parallel_formatting>
-        </default>
-    </profiles>
-</clickhouse>
\ No newline at end of file
+
+    <!-- ClickHouse's own telemetry tables have no TTL and grow without bound; on the
+         recommended webapp machine size their background merges eventually stop fitting
+         in memory and are retried forever (there is no backoff for failed merges),
+         pinning the CPU and ultimately failing the webapp's own inserts (#4343).
+         The ClickHouse low-RAM guide recommends disabling exactly these tables:
+         https://clickhouse.com/docs/operations/tips
+         Same list as the dev stack (docker/config/clickhouse-disable-system-logs.xml),
+         extended with the newer log tables. -->
+    <metric_log remove="1"/>
+    <asynchronous_metric_log remove="1"/>
+    <part_log remove="1"/>
+    <trace_log remove="1"/>
+    <query_thread_log remove="1"/>
+    <session_log remove="1"/>
+    <text_log remove="1"/>
+    <zookeeper_log remove="1"/>
+    <processors_profile_log remove="1"/>
+    <asynchronous_insert_log remove="1"/>
+    <backup_log remove="1"/>
+    <latency_log remove="1"/>
+    <query_metric_log remove="1"/>
+    <opentelemetry_span_log remove="1"/>
+    <query_views_log remove="1"/>
+
+    <!-- Keep query_log and error_log (small and useful for debugging), but bounded.
+         A config-level TTL survives log-table recreation, unlike ALTER ... MODIFY TTL.
+         Note: ClickHouse renames the existing table to *_log_<N> when this changes;
+         drop those leftovers to reclaim disk. -->
+    <query_log>
+        <ttl>event_date + INTERVAL 7 DAY DELETE</ttl>
+    </query_log>
+    <error_log>
+        <ttl>event_date + INTERVAL 30 DAY DELETE</ttl>
+    </error_log>
+
+    <!-- Profile settings do NOT live here: config.d is silently ignored for them.
+         See users-override.xml, mounted under users.d. -->
+</clickhouse>
diff --git a/hosting/docker/clickhouse/users-override.xml b/hosting/docker/clickhouse/users-override.xml
new file mode 100644
index 00000000000..a3964125b8c
--- /dev/null
+++ b/hosting/docker/clickhouse/users-override.xml
@@ -0,0 +1,21 @@
+<clickhouse>
+    <profiles>
+        <default>
+            <!-- Low-memory settings for the recommended webapp machine size. These were
+                 previously in config.d/override.xml where profile settings are silently
+                 ignored — they only take effect from the users config tree (#4343). -->
+            <max_block_size>8192</max_block_size>
+            <max_download_threads>1</max_download_threads>
+            <input_format_parallel_parsing>0</input_format_parallel_parsing>
+            <output_format_parallel_formatting>0</output_format_parallel_formatting>
+            <!-- The memory/query profilers sample stacks into system.trace_log
+                 (a Memory/MemoryPeak sample every 4 MB allocated, by default) — the main
+                 firehose that fills it. trace_log is disabled in override.xml; turn the
+                 samplers off too so they don't burn CPU producing discarded rows. -->
+            <memory_profiler_step>0</memory_profiler_step>
+            <memory_profiler_sample_probability>0</memory_profiler_sample_probability>
+            <query_profiler_real_time_period_ns>0</query_profiler_real_time_period_ns>
+            <query_profiler_cpu_time_period_ns>0</query_profiler_cpu_time_period_ns>
+        </default>
+    </profiles>
+</clickhouse>
diff --git a/hosting/docker/webapp/docker-compose.yml b/hosting/docker/webapp/docker-compose.yml
index a8bc1167b77..b893c17eca9 100644
--- a/hosting/docker/webapp/docker-compose.yml
+++ b/hosting/docker/webapp/docker-compose.yml
@@ -177,6 +177,7 @@ services:
       - clickhouse:/var/lib/clickhouse
       - ../clickhouse/data-paths.xml:/etc/clickhouse-server/config.d/data-paths.xml:ro
       - ../clickhouse/override.xml:/etc/clickhouse-server/config.d/override.xml:ro
+      - ../clickhouse/users-override.xml:/etc/clickhouse-server/users.d/override.xml:ro
     networks:
       - webapp
     healthcheck:

From 8833ebbe39b64afac125dd9d7a1f621bde01a6f6 Mon Sep 17 00:00:00 2001
From: Yann SEGET <yann.seget@actemium.ch>
Date: Mon, 10 Aug 2026 08:48:38 +0200
Subject: [PATCH 2/4] chore: reword .server-changes entry as a user-facing
 release note

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 .server-changes/hosting-clickhouse-system-logs.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.server-changes/hosting-clickhouse-system-logs.md b/.server-changes/hosting-clickhouse-system-logs.md
index efd4166ae51..dc6724232f9 100644
--- a/.server-changes/hosting-clickhouse-system-logs.md
+++ b/.server-changes/hosting-clickhouse-system-logs.md
@@ -3,4 +3,4 @@ area: hosting
 type: fix
 ---
 
-Self-hosted ClickHouse no longer accumulates unbounded system-log telemetry (metric_log, text_log, asynchronous_metric_log, ...) that eventually pins the CPU in a background merge-retry loop on the recommended machine size; query_log and error_log are kept with a TTL, and the low-memory profile settings now actually apply (moved from config.d to users.d).
+Self-hosted ClickHouse now disables its unbounded internal telemetry tables and keeps query/error logs on a bounded retention, fixing runaway CPU and memory usage on the recommended machine size. Existing self-hosted deployments must drop the previously written disabled log tables separately to reclaim disk.

From 371a0a82b669a8f4edf7449e1921a80ef2584a50 Mon Sep 17 00:00:00 2001
From: nicktrn <55853254+nicktrn@users.noreply.github.com>
Date: Mon, 24 Aug 2026 11:15:24 +0100
Subject: [PATCH 3/4] chore(hosting): trim clickhouse config comments

Rationale for the config belongs in the PR, not the artifact. The one claim kept was also inaccurate: processors_profile_log and asynchronous_insert_log already ship TTLs in the pinned image, so the tables are not all unbounded.

Drops the .server-changes entry: hosting-only changes are not covered by the webapp/supervisor areas, and a docker vs helm split is a separate discussion.
---
 .../hosting-clickhouse-system-logs.md           |  6 ------
 hosting/docker/clickhouse/override.xml          | 17 +++--------------
 hosting/docker/clickhouse/users-override.xml    |  8 +-------
 3 files changed, 4 insertions(+), 27 deletions(-)
 delete mode 100644 .server-changes/hosting-clickhouse-system-logs.md

diff --git a/.server-changes/hosting-clickhouse-system-logs.md b/.server-changes/hosting-clickhouse-system-logs.md
deleted file mode 100644
index dc6724232f9..00000000000
--- a/.server-changes/hosting-clickhouse-system-logs.md
+++ /dev/null
@@ -1,6 +0,0 @@
----
-area: hosting
-type: fix
----
-
-Self-hosted ClickHouse now disables its unbounded internal telemetry tables and keeps query/error logs on a bounded retention, fixing runaway CPU and memory usage on the recommended machine size. Existing self-hosted deployments must drop the previously written disabled log tables separately to reclaim disk.
diff --git a/hosting/docker/clickhouse/override.xml b/hosting/docker/clickhouse/override.xml
index f9088054a7e..967426d4751 100644
--- a/hosting/docker/clickhouse/override.xml
+++ b/hosting/docker/clickhouse/override.xml
@@ -10,14 +10,7 @@
     <mark_cache_size>524288000</mark_cache_size> <!-- 500MB -->
     <concurrent_threads_soft_limit_num>1</concurrent_threads_soft_limit_num>
 
-    <!-- ClickHouse's own telemetry tables have no TTL and grow without bound; on the
-         recommended webapp machine size their background merges eventually stop fitting
-         in memory and are retried forever (there is no backoff for failed merges),
-         pinning the CPU and ultimately failing the webapp's own inserts (#4343).
-         The ClickHouse low-RAM guide recommends disabling exactly these tables:
-         https://clickhouse.com/docs/operations/tips
-         Same list as the dev stack (docker/config/clickhouse-disable-system-logs.xml),
-         extended with the newer log tables. -->
+    <!-- ClickHouse's own telemetry, too heavy for the recommended machine size -->
     <metric_log remove="1"/>
     <asynchronous_metric_log remove="1"/>
     <part_log remove="1"/>
@@ -34,10 +27,7 @@
     <opentelemetry_span_log remove="1"/>
     <query_views_log remove="1"/>
 
-    <!-- Keep query_log and error_log (small and useful for debugging), but bounded.
-         A config-level TTL survives log-table recreation, unlike ALTER ... MODIFY TTL.
-         Note: ClickHouse renames the existing table to *_log_<N> when this changes;
-         drop those leftovers to reclaim disk. -->
+    <!-- A config-level ttl survives log-table recreation, unlike ALTER ... MODIFY TTL -->
     <query_log>
         <ttl>event_date + INTERVAL 7 DAY DELETE</ttl>
     </query_log>
@@ -45,6 +35,5 @@
         <ttl>event_date + INTERVAL 30 DAY DELETE</ttl>
     </error_log>
 
-    <!-- Profile settings do NOT live here: config.d is silently ignored for them.
-         See users-override.xml, mounted under users.d. -->
+    <!-- Profile settings only apply from users.d, see users-override.xml -->
 </clickhouse>
diff --git a/hosting/docker/clickhouse/users-override.xml b/hosting/docker/clickhouse/users-override.xml
index a3964125b8c..13bcb2aec6d 100644
--- a/hosting/docker/clickhouse/users-override.xml
+++ b/hosting/docker/clickhouse/users-override.xml
@@ -1,17 +1,11 @@
 <clickhouse>
     <profiles>
         <default>
-            <!-- Low-memory settings for the recommended webapp machine size. These were
-                 previously in config.d/override.xml where profile settings are silently
-                 ignored — they only take effect from the users config tree (#4343). -->
             <max_block_size>8192</max_block_size>
             <max_download_threads>1</max_download_threads>
             <input_format_parallel_parsing>0</input_format_parallel_parsing>
             <output_format_parallel_formatting>0</output_format_parallel_formatting>
-            <!-- The memory/query profilers sample stacks into system.trace_log
-                 (a Memory/MemoryPeak sample every 4 MB allocated, by default) — the main
-                 firehose that fills it. trace_log is disabled in override.xml; turn the
-                 samplers off too so they don't burn CPU producing discarded rows. -->
+            <!-- Samplers only feed trace_log, which is disabled -->
             <memory_profiler_step>0</memory_profiler_step>
             <memory_profiler_sample_probability>0</memory_profiler_sample_probability>
             <query_profiler_real_time_period_ns>0</query_profiler_real_time_period_ns>

From b707b75945b0bbcb79891b4d0d196e049e7b6aaa Mon Sep 17 00:00:00 2001
From: nicktrn <55853254+nicktrn@users.noreply.github.com>
Date: Mon, 24 Aug 2026 11:15:44 +0100
Subject: [PATCH 4/4] fix(hosting): bound part_log, cap query threads

part_log was not implicated in either report behind this change and is not on ClickHouse's sub-16GB disable list, but it is the merge history you need to diagnose the next occurrence. Measured at 0.18 KiB per part event under insert churn - 10x cheaper than text_log over the same window - so a TTL bounds it rather than removing it.

max_threads is the one sub-16GB recommendation the stack was still missing: it resolved to auto(4) and queries really did use 4 threads, so concurrent_threads_soft_limit_num was not covering it.
---
 hosting/docker/clickhouse/override.xml       | 4 +++-
 hosting/docker/clickhouse/users-override.xml | 1 +
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/hosting/docker/clickhouse/override.xml b/hosting/docker/clickhouse/override.xml
index 967426d4751..253669ab09d 100644
--- a/hosting/docker/clickhouse/override.xml
+++ b/hosting/docker/clickhouse/override.xml
@@ -13,7 +13,6 @@
     <!-- ClickHouse's own telemetry, too heavy for the recommended machine size -->
     <metric_log remove="1"/>
     <asynchronous_metric_log remove="1"/>
-    <part_log remove="1"/>
     <trace_log remove="1"/>
     <query_thread_log remove="1"/>
     <session_log remove="1"/>
@@ -28,6 +27,9 @@
     <query_views_log remove="1"/>
 
     <!-- A config-level ttl survives log-table recreation, unlike ALTER ... MODIFY TTL -->
+    <part_log>
+        <ttl>event_date + INTERVAL 7 DAY DELETE</ttl>
+    </part_log>
     <query_log>
         <ttl>event_date + INTERVAL 7 DAY DELETE</ttl>
     </query_log>
diff --git a/hosting/docker/clickhouse/users-override.xml b/hosting/docker/clickhouse/users-override.xml
index 13bcb2aec6d..46699505e34 100644
--- a/hosting/docker/clickhouse/users-override.xml
+++ b/hosting/docker/clickhouse/users-override.xml
@@ -1,6 +1,7 @@
 <clickhouse>
     <profiles>
         <default>
+            <max_threads>1</max_threads>
             <max_block_size>8192</max_block_size>
             <max_download_threads>1</max_download_threads>
             <input_format_parallel_parsing>0</input_format_parallel_parsing>