diff --git a/CHANGELOG.md b/CHANGELOG.md index 75b08dc..8978a2f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,25 @@ # Changelog -## Unreleased — resourceType rename + queued time + ip dim + gauge fill +## 0.10.0 — premium geo dimensions + +### Added + +- Nine event-only premium geo dimensions in `Metric::EVENT_COLUMNS` + and `Metric::getEventSchema()`: `city`, `continentCode`, + `subdivisions`, `isp`, `autonomousSystemNumber`, + `autonomousSystemOrganization`, `connectionType`, + `connectionUsageType`, `connectionOrganization`. All are optional + strings. In ClickHouse the lower-cardinality dims (`continentCode`, + `subdivisions`, `connectionType`, `connectionUsageType`, + `autonomousSystemNumber`) are `LowCardinality(Nullable(String))`; + the high-cardinality dims (`city`, `isp`, + `autonomousSystemOrganization`, `connectionOrganization`) are plain + `Nullable(String)`. Existing tables auto-materialize the columns on + `setup()` via the `ADD COLUMN IF NOT EXISTS` path. Gauges are + unchanged; these columns are not added to the primary key or + indexes. + +## 0.9.0 — resourceType rename + queued time + ip dim + gauge fill ### Breaking diff --git a/src/Usage/Adapter/ClickHouse.php b/src/Usage/Adapter/ClickHouse.php index 8a88899..0f3c3ba 100644 --- a/src/Usage/Adapter/ClickHouse.php +++ b/src/Usage/Adapter/ClickHouse.php @@ -1148,6 +1148,10 @@ private function getColumnType(string $id, string $type = 'event'): string 'clientEngine', 'clientEngineVersion', 'deviceName', 'deviceBrand', 'deviceModel', 'hostname', 'ip', + // premium geo (lower-cardinality only; city/isp/AS org/connection org + // are high-cardinality and intentionally fall through to Nullable(String)) + 'continentCode', 'subdivisions', 'connectionType', + 'connectionUsageType', 'autonomousSystemNumber', ]; if (in_array($id, $lowCardinality, true)) { @@ -1188,6 +1192,9 @@ private function getColumnCodec(string $id): string 'resourceId', 'resourceInternalId', 'teamId', 'teamInternalId', 'osVersion', 'clientVersion', 'clientEngineVersion', 'deviceModel', + 'city', 'continentCode', 'subdivisions', 'isp', + 'autonomousSystemNumber', 'autonomousSystemOrganization', + 'connectionType', 'connectionUsageType', 'connectionOrganization', ]; if (in_array($id, $zstdColumns, true)) { diff --git a/src/Usage/Metric.php b/src/Usage/Metric.php index c68b485..734a674 100644 --- a/src/Usage/Metric.php +++ b/src/Usage/Metric.php @@ -54,6 +54,10 @@ class Metric extends ArrayObject 'service', 'resourceType', 'resourceId', 'resourceInternalId', 'teamId', 'teamInternalId', 'country', 'region', 'hostname', 'ip', + // premium geo + 'city', 'continentCode', 'subdivisions', + 'isp', 'autonomousSystemNumber', 'autonomousSystemOrganization', + 'connectionType', 'connectionUsageType', 'connectionOrganization', 'osCode', 'osName', 'osVersion', 'clientType', 'clientCode', 'clientName', 'clientVersion', 'clientEngine', 'clientEngineVersion', @@ -82,6 +86,9 @@ class Metric extends ArrayObject * - resourceType / resourceId / resourceInternalId: resource identity * - teamId / teamInternalId: owning team identity * - country / region / hostname / ip: geographic + caller origin + * - city / continentCode / subdivisions: premium geo location fields + * - isp / autonomousSystemNumber / autonomousSystemOrganization: premium network origin + * - connectionType / connectionUsageType / connectionOrganization: premium connection intelligence * - osCode / osName / osVersion: parsed user-agent OS fields * - clientType / clientCode / clientName / clientVersion: parsed client * - clientEngine / clientEngineVersion: parsed client engine @@ -628,6 +635,16 @@ public static function getEventSchema(): array $stringColumn('region', 64), $stringColumn('hostname', 255), $stringColumn('ip', 45), + // premium geo + $stringColumn('city', 256), + $stringColumn('continentCode', 2), + $stringColumn('subdivisions', 256), + $stringColumn('isp', 256), + $stringColumn('autonomousSystemNumber', 255), + $stringColumn('autonomousSystemOrganization', 256), + $stringColumn('connectionType', 256), + $stringColumn('connectionUsageType', 256), + $stringColumn('connectionOrganization', 256), $stringColumn('osCode', 256), $stringColumn('osName', 256), $stringColumn('osVersion', 255), diff --git a/tests/Usage/Adapter/ClickHouseColumnTypeTest.php b/tests/Usage/Adapter/ClickHouseColumnTypeTest.php new file mode 100644 index 0000000..a56e7a6 --- /dev/null +++ b/tests/Usage/Adapter/ClickHouseColumnTypeTest.php @@ -0,0 +1,75 @@ +adapter = new ClickHouseAdapter( + 'clickhouse', + 'default', + '', + 8123, + false, + namespace: 'utopia_usage_coltype', + ); + + $this->getColumnType = new ReflectionMethod(ClickHouseAdapter::class, 'getColumnType'); + $this->getColumnType->setAccessible(true); + } + + private function columnType(string $id): string + { + /** @var string $type */ + $type = $this->getColumnType->invoke($this->adapter, $id, 'event'); + return $type; + } + + /** + * Lower-cardinality premium geo dims must map to LowCardinality(Nullable(String)). + */ + public function testLowCardinalityPremiumGeoColumns(): void + { + foreach ([ + 'continentCode', 'subdivisions', 'connectionType', + 'connectionUsageType', 'autonomousSystemNumber', + ] as $col) { + $this->assertSame( + 'LowCardinality(Nullable(String))', + $this->columnType($col), + "{$col} should be LowCardinality(Nullable(String))" + ); + } + } + + /** + * High-cardinality premium geo dims must fall through to Nullable(String). + */ + public function testHighCardinalityPremiumGeoColumns(): void + { + foreach ([ + 'city', 'isp', 'autonomousSystemOrganization', 'connectionOrganization', + ] as $col) { + $this->assertSame( + 'Nullable(String)', + $this->columnType($col), + "{$col} should be plain Nullable(String)" + ); + } + } +} diff --git a/tests/Usage/Adapter/ClickHouseSchemaTest.php b/tests/Usage/Adapter/ClickHouseSchemaTest.php index 281ef27..abe9d2d 100644 --- a/tests/Usage/Adapter/ClickHouseSchemaTest.php +++ b/tests/Usage/Adapter/ClickHouseSchemaTest.php @@ -44,6 +44,13 @@ public function testEventsTableCarriesCodecsAndLowCardinality(): void $this->assertStringContainsString('`teamId` Nullable(String) CODEC(ZSTD(3))', $ddl); $this->assertStringContainsString('`osVersion` LowCardinality(Nullable(String)) CODEC(ZSTD(3))', $ddl); $this->assertStringContainsString('`deviceModel` LowCardinality(Nullable(String)) CODEC(ZSTD(3))', $ddl); + + // premium geo: lower-cardinality dims are LowCardinality, high-cardinality + // dims fall through to plain Nullable(String). + $this->assertStringContainsString('`continentCode` LowCardinality(Nullable(String)) CODEC(ZSTD(3))', $ddl); + $this->assertStringContainsString('`connectionType` LowCardinality(Nullable(String)) CODEC(ZSTD(3))', $ddl); + $this->assertStringContainsString('`city` Nullable(String) CODEC(ZSTD(3))', $ddl); + $this->assertStringContainsString('`isp` Nullable(String) CODEC(ZSTD(3))', $ddl); } public function testEventsTableSwapsBloomForSetOnLowCardinality(): void @@ -211,6 +218,8 @@ private function expectedDimAssertions(array $columns, string $type): array 'clientEngine', 'clientEngineVersion', 'deviceName', 'deviceBrand', 'deviceModel', 'hostname', 'ip', + 'continentCode', 'subdivisions', 'connectionType', + 'connectionUsageType', 'autonomousSystemNumber', ]; $baseKey = ['id', 'metric', 'value', 'time', 'tenant']; diff --git a/tests/Usage/MetricTest.php b/tests/Usage/MetricTest.php index 90eba04..b27a2d2 100644 --- a/tests/Usage/MetricTest.php +++ b/tests/Usage/MetricTest.php @@ -609,6 +609,9 @@ public function testEventColumnsConstant(): void 'service', 'resourceType', 'resourceId', 'resourceInternalId', 'teamId', 'teamInternalId', 'country', 'region', 'hostname', 'ip', + 'city', 'continentCode', 'subdivisions', + 'isp', 'autonomousSystemNumber', 'autonomousSystemOrganization', + 'connectionType', 'connectionUsageType', 'connectionOrganization', 'osCode', 'osName', 'osVersion', 'clientType', 'clientCode', 'clientName', 'clientVersion', 'clientEngine', 'clientEngineVersion', @@ -645,6 +648,39 @@ public function testEventSchemaHasAllNewColumns(): void $this->assertNotContains('tags', $ids, 'tags must be removed'); } + /** + * The premium geo dimensions are event-only string columns that are + * present in EVENT_COLUMNS and the event schema (as non-required + * strings) but never leak into the gauge column set. + */ + public function testPremiumGeoColumnsArePresentAsEventStrings(): void + { + $geo = [ + 'city', 'continentCode', 'subdivisions', + 'isp', 'autonomousSystemNumber', 'autonomousSystemOrganization', + 'connectionType', 'connectionUsageType', 'connectionOrganization', + ]; + + $schema = Metric::getEventSchema(); + $eventIds = array_column($schema, '$id'); + $gaugeIds = array_column(Metric::getGaugeSchema(), '$id'); + + foreach ($geo as $col) { + $this->assertContains($col, Metric::EVENT_COLUMNS, "EVENT_COLUMNS missing {$col}"); + $this->assertContains($col, $eventIds, "Event schema missing {$col}"); + $this->assertNotContains($col, Metric::GAUGE_COLUMNS, "{$col} must be event-only"); + $this->assertNotContains($col, $gaugeIds, "{$col} must not be in gauge schema"); + + $matches = array_values(array_filter( + $schema, + static fn (array $attr): bool => $attr['$id'] === $col, + )); + $this->assertCount(1, $matches, "{$col} must appear exactly once"); + $this->assertSame('string', $matches[0]['type'], "{$col} must be a string column"); + $this->assertFalse($matches[0]['required'], "{$col} must be optional"); + } + } + /** * Test that the gauge schema contains the new team and resource id columns. */