use has()/hasAny() on KV token columns for SQL/facet map filters

.changeset/petite-clouds-decide.md

@@ -0,0 +1,5 @@
+---
+'@hyperdx/common-utils': patch
+---
+
+Rewrite SQL map filters to has()/hasAny() for KV text index direct read

packages/common-utils/src/__tests__/clickhouse.test.ts

@@ -203,6 +203,32 @@ describe('chSqlToAliasMap - alias unit test', () => {
     expect(res).toEqual(aliasMap);
   });
 
+  it('parses SELECT aliases when WHERE has hasAny(array(...)) from KV rewrite', () => {
+    const chSqlInput: ChSql = {
+      sql: "SELECT Timestamp as ts, Body as body FROM {HYPERDX_PARAM_1:Identifier}.{HYPERDX_PARAM_2:Identifier} WHERE (has(`ResourceAttributeTokens`, concat('facility', '=', 'local0')) OR hasAny(`ResourceAttributeTokens`, array('cloud.availability_zone=zone-a', 'cloud.availability_zone=zone-b'))) ORDER BY Timestamp DESC LIMIT {HYPERDX_PARAM_3:Int32}",
+      params: {
+        HYPERDX_PARAM_1: 'otel',
+        HYPERDX_PARAM_2: 'otel_logs',
+        HYPERDX_PARAM_3: 200,
+      },
+    };
+    expect(chSqlToAliasMap(chSqlInput)).toEqual({
+      ts: 'Timestamp',
+      body: 'Body',
+    });
+  });
+
+  it('parses SELECT aliases when WHERE has legacy hasAny bracket arrays', () => {
+    const chSqlInput: ChSql = {
+      sql: "SELECT Timestamp as ts FROM {HYPERDX_PARAM_1:Identifier}.{HYPERDX_PARAM_2:Identifier} WHERE hasAny(`ResourceAttributeTokens`, ['facility=local0', 'facility=local1']) ORDER BY Timestamp DESC",
+      params: {
+        HYPERDX_PARAM_1: 'otel',
+        HYPERDX_PARAM_2: 'otel_logs',
+      },
+    };
+    expect(chSqlToAliasMap(chSqlInput)).toEqual({ ts: 'Timestamp' });
+  });
+
   it('Alias, with JSON expressions', () => {
     const chSqlInput: ChSql = {
       sql: "SELECT Timestamp as ts,ResourceAttributes.service.name as service,toStartOfDay(LogAttributes.start.`time`) as start_time,Body,TimestampTime,ServiceName,TimestampTime FROM {HYPERDX_PARAM_1544803905:Identifier}.{HYPERDX_PARAM_129845054:Identifier} WHERE (TimestampTime >= fromUnixTimestamp64Milli({HYPERDX_PARAM_1456399765:Int64}) AND TimestampTime <= fromUnixTimestamp64Milli({HYPERDX_PARAM_1719057412:Int64})) AND (`ResourceAttributes`.`service`.`name` = 'serviceName') ORDER BY TimestampTime DESC LIMIT {HYPERDX_PARAM_49586:Int32} OFFSET {HYPERDX_PARAM_48:Int32}",

packages/common-utils/src/__tests__/metadata.test.ts

@@ -1,3 +1,4 @@
+import { ColumnMeta } from '../clickhouse';
 import { ClickhouseClient } from '../clickhouse/node';
 import { Metadata, MetadataCache, parseKeyPath } from '../core/metadata';
 import * as renderChartConfigModule from '../core/renderChartConfig';
@@ -416,6 +417,116 @@ describe('Metadata', () => {
     });
   });
 
+  describe('getKvItemsLookup', () => {
+    const otelKvExpression =
+      "arrayMap(arr -> concat(arr.1, '=', arr.2), CAST(ResourceAttributes, 'Array(Tuple(String, String))'))";
+
+    const otelColumns: ColumnMeta[] = [
+      {
+        name: 'ResourceAttributes',
+        type: 'Map(LowCardinality(String), String)',
+        default_type: '',
+        default_expression: '',
+        codec_expression: '',
+        comment: '',
+        ttl_expression: '',
+      },
+      {
+        name: 'ResourceAttributeTokens',
+        type: 'Array(String)',
+        default_type: 'MATERIALIZED',
+        default_expression: otelKvExpression,
+        codec_expression: '',
+        comment: '',
+        ttl_expression: '',
+      },
+    ];
+
+    const otelTokensTextIndex = {
+      name: 'idx_res_attr_tokens_text',
+      type: 'text',
+      typeFull: 'text(tokenizer = array)',
+      expression: 'ResourceAttributeTokens',
+      granularity: 1,
+    };
+
+    beforeEach(() => {
+      mockCache.getOrFetch.mockImplementation((_key, queryFn) => queryFn());
+      jest.spyOn(metadata, 'getColumns').mockResolvedValue(otelColumns);
+      jest
+        .spyOn(metadata, 'getSkipIndices')
+        .mockResolvedValue([otelTokensTextIndex]);
+    });
+
+    afterEach(() => {
+      jest.restoreAllMocks();
+    });
+
+    it('registers ResourceAttributes when KV column and array text index exist', async () => {
+      const lookup = await metadata.getKvItemsLookup({
+        databaseName: 'otel',
+        tableName: 'otel_logs',
+        connectionId: 'test_connection',
+      });
+
+      expect(lookup.get('ResourceAttributes')).toEqual({
+        kvItemsColumn: 'ResourceAttributeTokens',
+        separator: '=',
+      });
+    });
+
+    it('returns empty lookup when text index tokenizer is not array', async () => {
+      jest.spyOn(metadata, 'getSkipIndices').mockResolvedValue([
+        {
+          ...otelTokensTextIndex,
+          typeFull: "text(tokenizer = 'splitByNonAlpha')",
+        },
+      ]);
+
+      const lookup = await metadata.getKvItemsLookup({
+        databaseName: 'otel',
+        tableName: 'otel_logs',
+        connectionId: 'test_connection',
+      });
+
+      expect(lookup.size).toBe(0);
+    });
+
+    it('returns empty lookup when text index expression does not match KV column', async () => {
+      jest.spyOn(metadata, 'getSkipIndices').mockResolvedValue([
+        {
+          ...otelTokensTextIndex,
+          expression: 'mapKeys(ResourceAttributes)',
+        },
+      ]);
+
+      const lookup = await metadata.getKvItemsLookup({
+        databaseName: 'otel',
+        tableName: 'otel_logs',
+        connectionId: 'test_connection',
+      });
+
+      expect(lookup.size).toBe(0);
+    });
+
+    it('returns empty lookup when default_expression is not a KV items pattern', async () => {
+      jest.spyOn(metadata, 'getColumns').mockResolvedValue([
+        {
+          ...otelColumns[1],
+          default_expression: 'toString(ResourceAttributes)',
+        },
+      ]);
+
+      const lookup = await metadata.getKvItemsLookup({
+        databaseName: 'otel',
+        tableName: 'otel_logs',
+        connectionId: 'test_connection',
+      });
+
+      expect(lookup.size).toBe(0);
+    });
+  });
+
   describe('getKeyValues', () => {
     const mockChartConfig: BuilderChartConfigWithDateRange = {
       from: {

packages/common-utils/src/__tests__/queryParser.test.ts

@@ -1745,6 +1745,14 @@ describe('parseKvItemsCastExpression', () => {
     ).toEqual({ mapColumn: 'ResourceAttributes', separator: '=' });
   });
 
+  it('parses CAST KV items expression with bare lambda param (ClickHouse form)', () => {
+    expect(
+      parseKvItemsCastExpression(
+        "arrayMap(arr -> concat(arr.1, '=', arr.2), CAST(ResourceAttributes, 'Array(Tuple(String, String))'))",
+      ),
+    ).toEqual({ mapColumn: 'ResourceAttributes', separator: '=' });
+  });
+
   it('parses CAST form without spaces in type', () => {
     expect(
       parseKvItemsCastExpression(