feat(clickhouse): optional retention TTL on the raw events table

src/Usage/Adapter/ClickHouse.php

@@ -135,6 +135,14 @@ class ClickHouse extends SQL
      */
     private readonly float $dualReadSampleRate;
 
+    /**
+     * Retention window in days. When set, setup() applies a TTL to the raw
+     * events table so rows older than the window are dropped by background
+     * merges. The aggregated events_daily table is left untouched. Null
+     * disables TTL (the default).
+     */
+    private readonly ?int $retention;
+
     /**
      * @param  string  $host  ClickHouse host
      * @param  string  $username  ClickHouse username (default: 'default')
@@ -156,6 +164,10 @@ class ClickHouse extends SQL
      *   re-executed against the raw events table and logs a `warning` route
      *   entry if the totals diverge by >1%. Use 0.01 for a production canary
      *   or 1.0 in CI.
+     * @param  int|null  $retention  Retention window in days for the raw events
+     *   table. When set, setup() applies a TTL that drops rows older than the
+     *   window; the aggregated events_daily table (long-term usage/billing
+     *   history) is left untouched. Null disables TTL (default). Must be positive.
      */
     public function __construct(
         string $host,
@@ -169,10 +181,14 @@ public function __construct(
         bool $sharedTables = false,
         bool $asyncInserts = false,
         bool $asyncInsertWait = true,
-        float $dualReadSampleRate = 0.0
+        float $dualReadSampleRate = 0.0,
+        ?int $retention = null
     ) {
         $this->validateHost($host);
         $this->validatePort($port);
+        if ($retention !== null && $retention < 1) {
+            throw new Exception('Retention must be a positive number of days');
+        }
         if (!empty($namespace)) {
             $this->validateIdentifier($namespace, 'Namespace');
         }
@@ -191,6 +207,7 @@ public function __construct(
         // Clamp to [0.0, 1.0] so out-of-range rates can't disable or
         // over-trigger the parity sampler.
         $this->dualReadSampleRate = max(0.0, min(1.0, $dualReadSampleRate));
+        $this->retention = $retention;
 
         // `withConnectionReuse()` keeps the underlying cURL handle alive across
         // requests so the TCP/TLS handshake is paid once. Auth and database are
@@ -729,6 +746,8 @@ public function setup(): void
 
         $this->ensureEventDimColumns();
 
+        $this->applyEventsRetention();
+
         // --- Events daily table (SummingMergeTree) ---
         $this->createDailyTable();
 
@@ -765,6 +784,48 @@ public function setup(): void
         }
     }
 
+    /**
+     * Apply (or strip) the retention TTL on the raw events table as a separate
+     * idempotent ALTER. CREATE TABLE IF NOT EXISTS won't add a TTL to an
+     * existing table, and MODIFY TTL is a no-op when unchanged, so setup()
+     * stays re-runnable. The aggregated events_daily table is intentionally
+     * left untouched — it backs long-term usage/billing history.
+     * materialize_ttl_after_modify = 0 defers the purge to background merges
+     * rather than an immediate mutation.
+     *
+     * @throws Exception
+     */
+    private function applyEventsRetention(): void
+    {
+        $escapedTable = $this->escapeIdentifier($this->database)
+            . '.' . $this->escapeIdentifier($this->getEventsTableName());
+
+        if ($this->retention !== null) {
+            $this->query(
+                "ALTER TABLE {$escapedTable} "
+                . "MODIFY TTL toDateTime(time) + INTERVAL {$this->retention} DAY "
+                . 'SETTINGS materialize_ttl_after_modify = 0'
+            );
+            return;
+        }
+
+        // Disabling retention must actively strip any TTL a previous run
+        // applied; otherwise rows keep being purged despite retention being
+        // null. REMOVE TTL on a table with no TTL raises BAD_ARGUMENTS
+        // (code 36) — a generic code, so anchor on both the stable code and
+        // a "TTL" mention rather than the full English phrase (which can
+        // drift by version). This keeps setup() idempotent without swallowing
+        // unrelated bad-argument errors.
+        try {
+            $this->query("ALTER TABLE {$escapedTable} REMOVE TTL");
+        } catch (Exception $e) {
+            $message = $e->getMessage();
+            if (!str_contains($message, 'Code: 36') || !str_contains($message, 'TTL')) {
+                throw $e;
+            }
+        }
+    }
+
     /**
      * Allow lightweight DELETE on tables that carry projections. ClickHouse
      * defaults to throwing because a delete can leave projection parts

tests/Usage/Adapter/ClickHouseSchemaTest.php

@@ -205,6 +205,50 @@ public function testSetupBackfillsIpOnLegacyEventsTable(): void
         }
     }
 
+    public function testRetentionAppliesTtlToEventsTableOnly(): void
+    {
+        $adapter = new ClickHouseAdapter(
+            getenv('CLICKHOUSE_HOST') ?: 'clickhouse',
+            getenv('CLICKHOUSE_USER') ?: 'default',
+            getenv('CLICKHOUSE_PASSWORD') ?: 'clickhouse',
+            (int) (getenv('CLICKHOUSE_PORT') ?: 8123),
+            (bool) (getenv('CLICKHOUSE_SECURE') ?: false),
+            namespace: 'utopia_usage_schema_retention',
+            database: getenv('CLICKHOUSE_DATABASE') ?: 'default',
+            sharedTables: true,
+            retention: 30,
+        );
+        $database = $this->databaseName($adapter);
+        $eventsTable = $this->resolveTableName($adapter, 'getEventsTableName');
+        $dailyTable = $this->resolveTableName($adapter, 'getEventsDailyTableName');
+        $dailyMv = $this->resolveTableName($adapter, 'getTableName') . '_events_daily_mv';
+
+        // Start clean so the TTL is applied by setup(), not left over.
+        $this->queryRaw($adapter, "DROP TABLE IF EXISTS `{$database}`.`{$dailyMv}`");
+        $this->queryRaw($adapter, "DROP TABLE IF EXISTS `{$database}`.`{$dailyTable}`");
+        $this->queryRaw($adapter, "DROP TABLE IF EXISTS `{$database}`.`{$eventsTable}`");
+
+        $usage = new Usage($adapter);
+        $usage->setup();
+
+        $eventsDdl = $this->showCreateFor($adapter, "`{$database}`.`{$eventsTable}`");
+        $this->assertStringContainsString('TTL toDateTime(time)', $eventsDdl);
+
+        // Aggregated billing history must never carry a TTL.
+        $dailyDdl = $this->showCreateFor($adapter, "`{$database}`.`{$dailyTable}`");
+        $this->assertStringNotContainsString('TTL', $dailyDdl);
+    }
+
+    public function testRetentionRejectsNonPositiveDays(): void
+    {
+        $this->expectException(\Exception::class);
+
+        new ClickHouseAdapter(
+            getenv('CLICKHOUSE_HOST') ?: 'clickhouse',
+            retention: 0,
+        );
+    }
+
     /**
      * @param  array<int, string>  $columns
      * @return array<int, string>