From bacd1baab38d305d02f8cdf498c77a249bd7d1c5 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 29 May 2026 20:47:01 +0000 Subject: [PATCH] chore(release): bump HyperDX app/package versions --- .../add-to-filter-json-column-toString.md | 15 -- .changeset/clever-suns-nail.md | 9 - .changeset/cli-upload-sourcemaps-exit-code.md | 11 - .changeset/dashboard-toc-right-rail.md | 13 - .changeset/dedupe-map-subkey-filters.md | 11 - .changeset/describe-source-tool.md | 17 -- .changeset/direct-read-map-default.md | 44 ---- .changeset/event-deltas-mcp-tool.md | 14 -- .changeset/filters-direct-read.md | 9 - .changeset/five-frogs-dream.md | 5 - .changeset/great-papayas-obey.md | 5 - .changeset/hdx-4090-swap-memory-limiter.md | 73 ------ .../hdx-4371-histogram-bucket-datetime.md | 23 -- .../hdx-4373-row-panel-empty-sections.md | 26 -- .../hdx-4376-implicit-column-body-fallback.md | 18 -- ...383-skip-string-inference-on-json-level.md | 43 ---- .changeset/lower-body-text-index.md | 5 - .changeset/lucene-colon-in-map-key.md | 10 - .changeset/mcp-dashboard-prompts-overhaul.md | 15 -- .changeset/mcp-tool-quality-improvements.md | 12 - .changeset/mcp-trace-tools.md | 14 -- .changeset/neat-pianos-cheat.md | 6 - .changeset/number-tile-static-color.md | 5 - .changeset/onclick-hover-hint.md | 6 - .changeset/page-header-title-list-pages.md | 5 - .changeset/page-migrate-clickhouse.md | 5 - .changeset/page-migrate-kubernetes.md | 5 - .changeset/page-migrate-service-map.md | 5 - .changeset/perfect-ladybugs-scream.md | 5 - .changeset/purple-peaches-smash.md | 5 - .changeset/purple-pets-fail.md | 7 - .changeset/real-trainers-camp.md | 5 - .changeset/share-session-stale-url.md | 7 - .changeset/sixty-adults-raise.md | 8 - .changeset/wicked-rabbits-repeat.md | 7 - .env | 4 +- packages/api/CHANGELOG.md | 161 +++++++++++++ packages/api/package.json | 4 +- packages/app/CHANGELOG.md | 225 ++++++++++++++++++ packages/app/package.json | 4 +- packages/cli/CHANGELOG.md | 19 ++ packages/cli/package.json | 4 +- packages/common-utils/CHANGELOG.md | 108 +++++++++ packages/common-utils/package.json | 2 +- packages/otel-collector/CHANGELOG.md | 115 +++++++++ packages/otel-collector/package.json | 2 +- yarn.lock | 8 +- 47 files changed, 642 insertions(+), 487 deletions(-) delete mode 100644 .changeset/add-to-filter-json-column-toString.md delete mode 100644 .changeset/clever-suns-nail.md delete mode 100644 .changeset/cli-upload-sourcemaps-exit-code.md delete mode 100644 .changeset/dashboard-toc-right-rail.md delete mode 100644 .changeset/dedupe-map-subkey-filters.md delete mode 100644 .changeset/describe-source-tool.md delete mode 100644 .changeset/direct-read-map-default.md delete mode 100644 .changeset/event-deltas-mcp-tool.md delete mode 100644 .changeset/filters-direct-read.md delete mode 100644 .changeset/five-frogs-dream.md delete mode 100644 .changeset/great-papayas-obey.md delete mode 100644 .changeset/hdx-4090-swap-memory-limiter.md delete mode 100644 .changeset/hdx-4371-histogram-bucket-datetime.md delete mode 100644 .changeset/hdx-4373-row-panel-empty-sections.md delete mode 100644 .changeset/hdx-4376-implicit-column-body-fallback.md delete mode 100644 .changeset/hdx-4383-skip-string-inference-on-json-level.md delete mode 100644 .changeset/lower-body-text-index.md delete mode 100644 .changeset/lucene-colon-in-map-key.md delete mode 100644 .changeset/mcp-dashboard-prompts-overhaul.md delete mode 100644 .changeset/mcp-tool-quality-improvements.md delete mode 100644 .changeset/mcp-trace-tools.md delete mode 100644 .changeset/neat-pianos-cheat.md delete mode 100644 .changeset/number-tile-static-color.md delete mode 100644 .changeset/onclick-hover-hint.md delete mode 100644 .changeset/page-header-title-list-pages.md delete mode 100644 .changeset/page-migrate-clickhouse.md delete mode 100644 .changeset/page-migrate-kubernetes.md delete mode 100644 .changeset/page-migrate-service-map.md delete mode 100644 .changeset/perfect-ladybugs-scream.md delete mode 100644 .changeset/purple-peaches-smash.md delete mode 100644 .changeset/purple-pets-fail.md delete mode 100644 .changeset/real-trainers-camp.md delete mode 100644 .changeset/share-session-stale-url.md delete mode 100644 .changeset/sixty-adults-raise.md delete mode 100644 .changeset/wicked-rabbits-repeat.md diff --git a/.changeset/add-to-filter-json-column-toString.md b/.changeset/add-to-filter-json-column-toString.md deleted file mode 100644 index 4b0b247962..0000000000 --- a/.changeset/add-to-filter-json-column-toString.md +++ /dev/null @@ -1,15 +0,0 @@ ---- -"@hyperdx/app": patch ---- - -fix: "Add to Filters" on a JSON-typed ClickHouse column no longer produces an -unparseable Lucene query - -Previously, clicking "Add to Filters" on a field under a JSON column wrapped -the field path with `toString(...)` before handing it off as a Lucene filter -key. Lucene's grammar forbids parentheses inside field names, so the resulting -condition like `toString(JSONColumn.\`foo\`):"…"` failed to parse with -`Expected … but ":" found.` - -The handler now passes the clean dot-notation path (e.g. `JSONColumn.foo`) -to the filter setter. diff --git a/.changeset/clever-suns-nail.md b/.changeset/clever-suns-nail.md deleted file mode 100644 index af19fd8b47..0000000000 --- a/.changeset/clever-suns-nail.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -"@hyperdx/otel-collector": minor -"@hyperdx/common-utils": minor -"@hyperdx/api": minor -"@hyperdx/app": minor -"@hyperdx/cli": minor ---- - -feat: experimental promql support diff --git a/.changeset/cli-upload-sourcemaps-exit-code.md b/.changeset/cli-upload-sourcemaps-exit-code.md deleted file mode 100644 index 073f5487b5..0000000000 --- a/.changeset/cli-upload-sourcemaps-exit-code.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -"@hyperdx/cli": patch ---- - -fix(cli): exit with non-zero code when `upload-sourcemaps` fails - -The `upload-sourcemaps` command now exits with code 1 when uploads fail -(missing source maps, pre-signed URL request failure, authentication failure, -or any per-file upload failure after retries). Previously these failures were -logged to stderr but the process exited cleanly with code 0, causing CI -pipelines to treat failed uploads as successes. diff --git a/.changeset/dashboard-toc-right-rail.md b/.changeset/dashboard-toc-right-rail.md deleted file mode 100644 index 89e8450bb8..0000000000 --- a/.changeset/dashboard-toc-right-rail.md +++ /dev/null @@ -1,13 +0,0 @@ ---- -'@hyperdx/app': patch ---- - -feat(dashboard): add Table of Contents right rail with bulk collapse/expand - -Adds a toggleable right-rail Table of Contents to the dashboard page, plus -"Collapse all sections" and "Expand all sections" actions. All three live -under a new "View" section in the dashboard's existing menu. TOC visibility -is persisted per-user via localStorage; bulk collapse uses the same -per-viewer URL state as single-section toggling, so it's shareable via link -and does not change the dashboard's stored defaults. Clicking a TOC entry -scrolls the section into view, auto-expanding it first if collapsed. diff --git a/.changeset/dedupe-map-subkey-filters.md b/.changeset/dedupe-map-subkey-filters.md deleted file mode 100644 index 7c28285295..0000000000 --- a/.changeset/dedupe-map-subkey-filters.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -'@hyperdx/app': patch ---- - -fix: collapse duplicate map sub-key entries in the search filter sidebar (HDX-4340) - -A map sub-field stored in `filterState` under dot notation (e.g. `LogAttributes.time`, -from a Lucene URL round-trip) and the same key returned by the facet query under -bracket notation (e.g. `LogAttributes['time']`) no longer render as two separate -accordion items. The merged entry keeps the bracket form so "Load more" stays -valid, and the user's selection still resolves via a tolerant filterState lookup. diff --git a/.changeset/describe-source-tool.md b/.changeset/describe-source-tool.md deleted file mode 100644 index 36a8669b66..0000000000 --- a/.changeset/describe-source-tool.md +++ /dev/null @@ -1,17 +0,0 @@ ---- -'@hyperdx/api': patch ---- - -feat(mcp): add hyperdx_describe_source tool and slim list_sources to catalog - -Add `hyperdx_describe_source` — returns full column schema, map attribute -keys, and sampled low-cardinality values (SeverityText, StatusCode, -ServiceName, etc.) for a single source. Uses existing rollup tables for -performant value sampling. - -Slim `hyperdx_list_sources` to a lightweight MongoDB-only catalog (no -ClickHouse queries). Source tools moved to a dedicated `tools/sources/` -module. - -All query tool descriptions and prompts updated to reference the two-step -`list_sources → describe_source` discovery workflow. diff --git a/.changeset/direct-read-map-default.md b/.changeset/direct-read-map-default.md deleted file mode 100644 index e528e5b259..0000000000 --- a/.changeset/direct-read-map-default.md +++ /dev/null @@ -1,44 +0,0 @@ ---- -"@hyperdx/common-utils": minor -"@hyperdx/app": patch ---- - -feat: default the direct_read map column optimization on supported ClickHouse versions - -The full-text-search logs schema (`00002_otel_logs.sql`) now ships with -`ResourceAttributeItems`, `ScopeAttributeItems`, and `LogAttributeItems` -ALIAS columns plus their `text(tokenizer='array')` skip indexes. The -traces schema (`00005_otel_traces.sql`) similarly gains -`ResourceAttributeItems` and `SpanAttributeItems` ALIAS columns with -matching items indexes. New installs and freshly migrated tables get -the optimization automatically — no manual `ALTER TABLE` required. - -Note: the traces table previously used only `bloom_filter` skip indexes -and worked on any ClickHouse version. The added `text(tokenizer='array')` -items indexes raise the minimum ClickHouse version required to **create** -the traces table to **>= 26.2**. Existing tables on older clusters are -unaffected (`CREATE TABLE IF NOT EXISTS` is a no-op). - -At query time, the app gates the `Map['key'] = 'value'` → -`has(, concat('key', '=', 'value'))` rewrite on the connected -ClickHouse server version (`SELECT version()`, cached per connection). -The gate only applies to **ALIAS** items columns, which are computed at -query time and therefore depend on the server being able to perform a -direct_read against the underlying Map's tuple storage. The direct_read -feature was backported into multiple stable 26.x release lines, so the -gate uses a per-branch minimum: - -- 26.2 line: >= 26.2.19.43 -- 26.3 line: >= 26.3.12.3 -- 26.4 line: >= 26.4.3.37 -- 26.5+ : always supported - -ALIAS items columns on servers below their branch's threshold continue -to compile filters into the original Map-subscript form. - -**MATERIALIZED items columns are always used when available**, regardless -of ClickHouse version. MATERIALIZED columns are physically stored on -disk, so `has(items, ...)` reads them directly and works on any -ClickHouse version that supports the text index itself. Operators who -want the optimization on servers below the backport cutoffs can -`ALTER TABLE` to materialize the items columns. diff --git a/.changeset/event-deltas-mcp-tool.md b/.changeset/event-deltas-mcp-tool.md deleted file mode 100644 index ab972b7ffc..0000000000 --- a/.changeset/event-deltas-mcp-tool.md +++ /dev/null @@ -1,14 +0,0 @@ ---- -'@hyperdx/api': patch -'@hyperdx/common-utils': patch ---- - -feat(mcp): add hyperdx_event_deltas tool - -Add `hyperdx_event_deltas` MCP tool that compares two row groups (target -vs baseline) and ranks properties by how much their value distributions -differ. Same algorithm as the in-app Event Deltas view. - -Extract shared event-deltas algorithm from the UI into -`@hyperdx/common-utils/src/core/eventDeltas.ts` so it can be used by -both the frontend and the MCP server. diff --git a/.changeset/filters-direct-read.md b/.changeset/filters-direct-read.md deleted file mode 100644 index 26182d6853..0000000000 --- a/.changeset/filters-direct-read.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -"@hyperdx/common-utils": minor -"@hyperdx/app": minor ---- - -feat: emit Lucene conditions from sidebar/dashboard filters to enable KV items direct_read optimization on Map columns - -Legacy `type: 'sql'` filters in URLs are automatically migrated to Lucene -on page load. The persisted `DashboardFilter.expression` in MongoDB is unchanged. diff --git a/.changeset/five-frogs-dream.md b/.changeset/five-frogs-dream.md deleted file mode 100644 index 49e30b33f5..0000000000 --- a/.changeset/five-frogs-dream.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@hyperdx/app": patch ---- - -feat: preserve compatible filters when switching sources diff --git a/.changeset/great-papayas-obey.md b/.changeset/great-papayas-obey.md deleted file mode 100644 index 6f4b2c8f49..0000000000 --- a/.changeset/great-papayas-obey.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@hyperdx/app": patch ---- - -fix: persist column widths in search results table diff --git a/.changeset/hdx-4090-swap-memory-limiter.md b/.changeset/hdx-4090-swap-memory-limiter.md deleted file mode 100644 index 8cf0fe0c97..0000000000 --- a/.changeset/hdx-4090-swap-memory-limiter.md +++ /dev/null @@ -1,73 +0,0 @@ ---- -'@hyperdx/api': minor -'@hyperdx/app': minor -'@hyperdx/otel-collector': minor ---- - -fix(otel-collector): allow `CUSTOM_OTELCOL_CONFIG_FILE` to override the -default `memory_limiter`, `batch` (and other pipeline processors) - -Pipeline `processors:` lists used to be defined in the OpAMP remote config -sent by the API (`packages/api/src/opamp/controllers/opampController.ts`). -That meant the remote config overwrote any pipeline `processors:` list a -user supplied via `CUSTOM_OTELCOL_CONFIG_FILE`, making it impossible to -substitute the default `memory_limiter` with one configured for -`limit_percentage`/`spike_limit_percentage` mode (#2145). - -The pipeline `processors:` lists now live in the bootstrap config -(`docker/otel-collector/config.yaml` for supervisor mode, and -`docker/otel-collector/config.standalone.yaml` for standalone mode). The -OpAMP remote config no longer sets `processors:` on these pipelines, so the -bootstrap+custom merge wins. Receivers and exporters are still configured -dynamically by the OpAMP controller. - -To override `memory_limiter`, define a new processor with a different name -in `CUSTOM_OTELCOL_CONFIG_FILE` and swap the pipeline `processors:` lists: - -```yaml -processors: - memory_limiter/custom: - check_interval: 5s - limit_percentage: 75 - spike_limit_percentage: 25 - -service: - pipelines: - traces: - processors: [memory_limiter/custom, batch] - metrics: - processors: [memory_limiter/custom, batch] - logs/out-default: - processors: [memory_limiter/custom, transform, batch] - logs/out-rrweb: - processors: [memory_limiter/custom, batch] -``` - -The default `memory_limiter` block defined in the base config is left in -the merged config but is no longer referenced by any pipeline; the -collector only instantiates `memory_limiter/custom` at runtime. - -The same swap pattern works for the `batch` processor (and any other base -processor). For example, to lower the export timeout on a specific -pipeline: - -```yaml -processors: - batch/lowlatency: - send_batch_size: 1000 - send_batch_max_size: 2000 - timeout: 500ms - -service: - pipelines: - traces: - processors: [memory_limiter, batch/lowlatency] - logs/out-default: - processors: [memory_limiter, transform, batch/lowlatency] -``` - -Lighter-weight env-var tuning is also available for the default `batch` -processor without writing a custom config file: -`HYPERDX_OTEL_BATCH_SEND_BATCH_SIZE`, -`HYPERDX_OTEL_BATCH_SEND_BATCH_MAX_SIZE`, and `HYPERDX_OTEL_BATCH_TIMEOUT`. -See the README for details. diff --git a/.changeset/hdx-4371-histogram-bucket-datetime.md b/.changeset/hdx-4371-histogram-bucket-datetime.md deleted file mode 100644 index 4494cc3a50..0000000000 --- a/.changeset/hdx-4371-histogram-bucket-datetime.md +++ /dev/null @@ -1,23 +0,0 @@ ---- -"@hyperdx/common-utils": patch ---- - -fix(charts): histogram bucket picks the highest-precision DateTime column when -Timestamp Column lists multiple columns - -When a source's `Timestamp Column` listed multiple columns (e.g. -`"EventDate, EventTime"` for partition-pruning), the histogram bucket was -built from only the first token. If that token was a `Date` column, every -row in a day collapsed into a single bar at midnight UTC of that day. - -The bucket resolver now walks the comma-split list, queries each column's -type via metadata, and returns the highest-precision DateTime / DateTime64 -token. Date columns are skipped. If no DateTime-typed token is found, the -original first-token behavior is preserved with a `console.warn`. - -The WHERE clause continues to use the multi-column form, so partition -pruning via the `Date` column keeps working. The same resolved column is -also used for the `argMin` / `argMax` / `min` / `max` time math in delta -expressions. - -Fixes HDX-4371. diff --git a/.changeset/hdx-4373-row-panel-empty-sections.md b/.changeset/hdx-4373-row-panel-empty-sections.md deleted file mode 100644 index a406fca3ad..0000000000 --- a/.changeset/hdx-4373-row-panel-empty-sections.md +++ /dev/null @@ -1,26 +0,0 @@ ---- -"@hyperdx/app": patch ---- - -fix(row-panel): hide empty attribute sections and stop showing "[Empty]" -when the source's body column isn't configured - -The row-expand side panel always rendered `Log/Span Attributes` and -`Resource Attributes` accordion sections, even when both were empty. The -body header fell back to a literal `[Empty]` paper in two visually -identical cases that meant different things: the body column was -configured but the value was empty, or the body column wasn't configured -on the source at all. - -The two attribute accordions now mirror the existing `topLevelAttributes` -pattern and only render when their content is non-empty. The body header -takes a new `bodyConfigured` prop: when `false` (source has neither body -nor implicit column expression configured), the body paper is suppressed -entirely. When `true` and the content is empty, the placeholder reads -"No body for this event." instead of `[Empty]`. - -`DBRowOverviewPanel` derives `bodyConfigured` from -`getEventBody(source) !== undefined`, which already returns `undefined` -when neither expression is set. - -Fixes HDX-4373. diff --git a/.changeset/hdx-4376-implicit-column-body-fallback.md b/.changeset/hdx-4376-implicit-column-body-fallback.md deleted file mode 100644 index 7d047e58f2..0000000000 --- a/.changeset/hdx-4376-implicit-column-body-fallback.md +++ /dev/null @@ -1,18 +0,0 @@ ---- -"@hyperdx/common-utils": patch -"@hyperdx/app": patch -"@hyperdx/api": patch ---- - -fix: bare-text Lucene search now falls back from Implicit Column Expression to -Body Expression on log sources - -Previously, a log source configured with `bodyExpression` set but -`implicitColumnExpression` unset threw `Can not search bare text without an -implicit column set.` on every bare-token search, even though the row panel -rendered correctly using the body column. - -Search now reuses the same one-way fallback that `getEventBody` already -implements: when no Implicit Column Expression is set, bare-text search runs -against the configured Body Expression. Trace sources are unchanged -(`spanNameExpression` is not a body equivalent for trace search). diff --git a/.changeset/hdx-4383-skip-string-inference-on-json-level.md b/.changeset/hdx-4383-skip-string-inference-on-json-level.md deleted file mode 100644 index a502eff2a5..0000000000 --- a/.changeset/hdx-4383-skip-string-inference-on-json-level.md +++ /dev/null @@ -1,43 +0,0 @@ ---- -"@hyperdx/otel-collector": patch ---- - -fix(otel-collector): skip string severity inference when JSON body has a -`level`/`severity` field - -When the log body parsed as JSON and contained a level-like field, the -pipeline still ran its `\b(alert|crit|emerg|fatal|error|err|warn|notice|debug|dbug|trace)` -keyword scan over the raw body string. The leading-only `\b` boundary -matched any word starting with a severity keyword, so bodies containing -words like `alertmanager`, `alerting`, `errors`, `warning`, etc. produced -the wrong severity. A Grafana sidecar log with body -`{"level":"INFO", "msg":"... mimir-alertmanager-dashboard ..."}` was being -tagged `SeverityText="fatal"`, `SeverityNumber=21` because `alert` matched -inside `alertmanager`, even though the JSON `level` said `INFO`. - -A new OTTL `log_statements` block in -`docker/otel-collector/config.yaml` runs between the existing JSON-parse -block and the string-inference block. It promotes a JSON-derived level -field (now in `log.attributes`) to `log.severity_text`, which causes the -string-inference block to be skipped via its existing -`severity_number == 0 and severity_text == ""` guard. The block is -case-insensitive across keys by enumerating common casings of common field -names used by mainstream logging frameworks: `level` / `Level` / `LEVEL` -(pino, winston, zerolog, zap, logrus, slog, Serilog, NLog), -`severity` / `Severity` / `SEVERITY` (Datadog, GCP Cloud Logging), and -`log.level` (Elastic ECS, flattened from nested JSON). Each `set` -self-guards on `severity_text == ""` so the first match wins (priority: -`level` > `severity` > `log.level`). The block as a whole is gated on no -producer-set severity, so explicit producer values are always preserved. - -`severity_number` is mapped via case-insensitive `(?i)` regex over -`severity_text`, mirroring the existing string-inference keyword set. -Unrecognized values (e.g. `"verbose"`) fall back to `INFO`, matching -block 2's else-branch. The existing `ConvertCase(severity_text, "lower")` -normalization is unchanged. - -Behavior preserved for: non-JSON bodies, JSON bodies without a level -field, and any log record where the producer already set -`severity_text` or `severity_number`. - -Fixes HDX-4383. diff --git a/.changeset/lower-body-text-index.md b/.changeset/lower-body-text-index.md deleted file mode 100644 index bd35d269a1..0000000000 --- a/.changeset/lower-body-text-index.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@hyperdx/common-utils": patch ---- - -fix: support text index on lower(Body) with no preprocessor diff --git a/.changeset/lucene-colon-in-map-key.md b/.changeset/lucene-colon-in-map-key.md deleted file mode 100644 index c74b10241a..0000000000 --- a/.changeset/lucene-colon-in-map-key.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -"@hyperdx/common-utils": patch ---- - -fix: escape colons in Lucene field names so filters on Map sub-keys containing -`:` (e.g. `LogAttributes['foo:bar']`) parse correctly - -`filtersToQuery` now backslash-escapes `:` and `\` in the emitted Lucene field -name, and `parseLuceneFilter` + the SQL serializer decode those placeholders -when consuming the AST so the original key is restored end-to-end. diff --git a/.changeset/mcp-dashboard-prompts-overhaul.md b/.changeset/mcp-dashboard-prompts-overhaul.md deleted file mode 100644 index 253f4dbef0..0000000000 --- a/.changeset/mcp-dashboard-prompts-overhaul.md +++ /dev/null @@ -1,15 +0,0 @@ ---- -'@hyperdx/api': patch ---- - -feat(mcp): rewrite dashboard authoring prompts and expose `filters` on `hyperdx_save_dashboard` - -The `create_dashboard` prompt now leads with a design checklist (alias every select item including number tiles, schema gap on `groupBy` so tables don't render `arrayElement(SpanAttributes, '...')` as the column header, RED columns with aliases, per-series `numberFormat` for durations, `groupByColumnsOnLeft` for inventory tables, dashboard-level filters instead of per-tile `where` literals, one-metric-per-tile for metric sources, required containers at five or more tiles, post-save validation of every tile, no title-recap markdown). The wall-of-JSON canonical example is gone; the `dashboard_examples` patterns carry the concrete shapes. - -The `dashboard_examples` set is replaced with four verified patterns (`service_inventory`, `service_detail`, `log_analytics`, `backend_dependencies`) plus the existing `infrastructure_sql`. Each non-SQL example leads with a "When to use" header and a "Why this shape" note so the model picks by intent, not by surface keyword match. Examples were built and rendered on a live dev stack before landing. - -The `query_guide` prompt gains a `DASHBOARD FILTERS` section that documents the `filters: [{ type, name, expression, sourceId, where?, whereLanguage? }]` shape, a `NUMBER FORMAT` section that explains the per-series vs. chart-level distinction, and a `PER-TILE TYPE CONSTRAINTS` note that metric tiles take exactly one select item per tile. - -`hyperdx_save_dashboard` now accepts `filters` on its input schema, reusing `externalDashboardFilterSchemaWithId` so the MCP and REST surfaces stay in lockstep and the existing `convertExternalFiltersToInternal` helper handles the conversion without translation. Filters round-trip through create, get, and update. - -Voice pass: every prompt string is now em-dash-free. diff --git a/.changeset/mcp-tool-quality-improvements.md b/.changeset/mcp-tool-quality-improvements.md deleted file mode 100644 index 13fa6e2a1e..0000000000 --- a/.changeset/mcp-tool-quality-improvements.md +++ /dev/null @@ -1,12 +0,0 @@ ---- -'@hyperdx/api': patch ---- - -feat(mcp): improve MCP tool quality — error hints, shared helpers, better messages - -Extract duplicated ClickHouse error handling into a shared helper with -pattern-matched error hints (DateTime64 casting, AS alias quoting, response -size limits) so agents get actionable guidance on common failures. Add -reusable mergeWhereIntoSelectItems() helper for consistent top-level where -injection. Improve source/connection-not-found messages to suggest calling -hyperdx_list_sources. diff --git a/.changeset/mcp-trace-tools.md b/.changeset/mcp-trace-tools.md deleted file mode 100644 index 55d6648dee..0000000000 --- a/.changeset/mcp-trace-tools.md +++ /dev/null @@ -1,14 +0,0 @@ ---- -'@hyperdx/api': patch ---- - -feat(mcp): add trace waterfall and breakdown tools - -Add `hyperdx_trace_waterfall` — fetch all spans in a single trace as a -parent/child waterfall tree with optional correlated logs. Supports -auto-pick by slowest, first error, or most recent trace. - -Add `hyperdx_trace_top_time_consuming_operations` — aggregate breakdown -of child operations consuming the most cumulative time across traces -matching a parent-span filter. Same algorithm as the in-app "Top Most -Time Consuming Operations" chart. diff --git a/.changeset/neat-pianos-cheat.md b/.changeset/neat-pianos-cheat.md deleted file mode 100644 index f3f7d03ba2..0000000000 --- a/.changeset/neat-pianos-cheat.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -"@hyperdx/app": patch -"@hyperdx/cli": patch ---- - -fix: only use pk and row uniqueness to look up a row diff --git a/.changeset/number-tile-static-color.md b/.changeset/number-tile-static-color.md deleted file mode 100644 index 875a658e8c..0000000000 --- a/.changeset/number-tile-static-color.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@hyperdx/app': patch ---- - -Number tile: pick a static color from the palette in Display Settings. The color picker stores a palette token (not a hex value) so the choice reflows correctly across light, dark, and IDE themes. diff --git a/.changeset/onclick-hover-hint.md b/.changeset/onclick-hover-hint.md deleted file mode 100644 index c529280ac2..0000000000 --- a/.changeset/onclick-hover-hint.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -"@hyperdx/common-utils": patch -"@hyperdx/app": patch ---- - -Dashboard table tiles configured with a row-click action now show a hover hint describing where the click will go (for example, `Search HyperDX Logs` or `Open dashboard "API Latency Drilldown"`). The cell wrapper is now a real link, so cmd-click and middle-click open the destination in a new tab, right-click shows the browser context menu with "Open in New Tab" and "Copy Link Address", and the destination URL appears in the browser status bar on hover. Keyboard users can Tab to a cell and press Enter to navigate, with a visible focus ring. diff --git a/.changeset/page-header-title-list-pages.md b/.changeset/page-header-title-list-pages.md deleted file mode 100644 index 29caa37a36..0000000000 --- a/.changeset/page-header-title-list-pages.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@hyperdx/app': patch ---- - -chore: use `PageHeader` `title` prop on Alerts, Dashboards, and Saved Searches list pages for consistency with the shared header API. diff --git a/.changeset/page-migrate-clickhouse.md b/.changeset/page-migrate-clickhouse.md deleted file mode 100644 index fb54609f0f..0000000000 --- a/.changeset/page-migrate-clickhouse.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@hyperdx/app': patch ---- - -chore: migrate the ClickHouse dashboard to shared `PageLayout` with breadcrumbs and a sticky header (connection selector, time range, refresh) instead of a duplicate page title. diff --git a/.changeset/page-migrate-kubernetes.md b/.changeset/page-migrate-kubernetes.md deleted file mode 100644 index e273f57a84..0000000000 --- a/.changeset/page-migrate-kubernetes.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@hyperdx/app': patch ---- - -chore: migrate the Kubernetes dashboard to shared `PageLayout` with breadcrumbs and a sticky header (log + metric sources, time range, refresh) instead of a duplicate page title. diff --git a/.changeset/page-migrate-service-map.md b/.changeset/page-migrate-service-map.md deleted file mode 100644 index 21019c4b57..0000000000 --- a/.changeset/page-migrate-service-map.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@hyperdx/app': patch ---- - -chore: migrate Service Map to shared `PageLayout` with a sticky toolbar (source, sampling, time range) and no duplicate page title. diff --git a/.changeset/perfect-ladybugs-scream.md b/.changeset/perfect-ladybugs-scream.md deleted file mode 100644 index 8689ccbe1a..0000000000 --- a/.changeset/perfect-ladybugs-scream.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@hyperdx/app": patch ---- - -refactor: Use new VirtualMultiSelect for dashboard filter inputs diff --git a/.changeset/purple-peaches-smash.md b/.changeset/purple-peaches-smash.md deleted file mode 100644 index d7fcfb631e..0000000000 --- a/.changeset/purple-peaches-smash.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@hyperdx/app": patch ---- - -fix: let "Load more" surface unselected values in exact filter mode diff --git a/.changeset/purple-pets-fail.md b/.changeset/purple-pets-fail.md deleted file mode 100644 index 13dd16eec0..0000000000 --- a/.changeset/purple-pets-fail.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -"@hyperdx/common-utils": patch -"@hyperdx/api": patch -"@hyperdx/app": patch ---- - -feat: Add source scoping to dashboard filters diff --git a/.changeset/real-trainers-camp.md b/.changeset/real-trainers-camp.md deleted file mode 100644 index e3b852a4b1..0000000000 --- a/.changeset/real-trainers-camp.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@hyperdx/app": minor ---- - -feat: upgrade to clickhouse-server 26.5 diff --git a/.changeset/share-session-stale-url.md b/.changeset/share-session-stale-url.md deleted file mode 100644 index 0f99d4b6f3..0000000000 --- a/.changeset/share-session-stale-url.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -'@hyperdx/app': patch ---- - -fix(app): copy correct session URL on first Share Session click - -The Share Session button captured `window.location.href` at render time, which ran before `nuqs` flushed `sid`/`sfrom`/`sto` into the URL. The button now reads the URL at click time via the shared `copyTextToClipboard` util, so the first copy always contains the session params (no reload needed). diff --git a/.changeset/sixty-adults-raise.md b/.changeset/sixty-adults-raise.md deleted file mode 100644 index 06f2b46458..0000000000 --- a/.changeset/sixty-adults-raise.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -"@hyperdx/common-utils": patch -"@hyperdx/api": patch -"@hyperdx/app": patch -"@hyperdx/cli": patch ---- - -feat: Add option for force-enabling/disabling text index support diff --git a/.changeset/wicked-rabbits-repeat.md b/.changeset/wicked-rabbits-repeat.md deleted file mode 100644 index 3e1e399cea..0000000000 --- a/.changeset/wicked-rabbits-repeat.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -"@hyperdx/common-utils": patch -"@hyperdx/api": patch -"@hyperdx/app": patch ---- - -feat: filters reflect all values, not search aware; filters use metadata MVs if available diff --git a/.env b/.env index 51a93a4f0e..6f07a637c7 100644 --- a/.env +++ b/.env @@ -8,8 +8,8 @@ NEXT_ALL_IN_ONE_IMAGE_NAME_DOCKERHUB=clickhouse/clickstack-all-in-one ALL_IN_ONE_IMAGE_NAME_DOCKERHUB=hyperdx/hyperdx-all-in-one NEXT_OTEL_COLLECTOR_IMAGE_NAME_DOCKERHUB=clickhouse/clickstack-otel-collector OTEL_COLLECTOR_IMAGE_NAME_DOCKERHUB=hyperdx/hyperdx-otel-collector -CODE_VERSION=2.27.0 -IMAGE_VERSION_SUB_TAG=.27.0 +CODE_VERSION=2.28.0 +IMAGE_VERSION_SUB_TAG=.28.0 IMAGE_VERSION=2 IMAGE_NIGHTLY_TAG=2-nightly IMAGE_LATEST_TAG=latest diff --git a/packages/api/CHANGELOG.md b/packages/api/CHANGELOG.md index 8af11351ba..5f4e8b8e17 100644 --- a/packages/api/CHANGELOG.md +++ b/packages/api/CHANGELOG.md @@ -1,5 +1,166 @@ # @hyperdx/api +## 2.28.0 + +### Minor Changes + +- 3123db53: feat: experimental promql support +- cb6a74ce: fix(otel-collector): allow `CUSTOM_OTELCOL_CONFIG_FILE` to override the + default `memory_limiter`, `batch` (and other pipeline processors) + + Pipeline `processors:` lists used to be defined in the OpAMP remote config + sent by the API (`packages/api/src/opamp/controllers/opampController.ts`). + That meant the remote config overwrote any pipeline `processors:` list a + user supplied via `CUSTOM_OTELCOL_CONFIG_FILE`, making it impossible to + substitute the default `memory_limiter` with one configured for + `limit_percentage`/`spike_limit_percentage` mode (#2145). + + The pipeline `processors:` lists now live in the bootstrap config + (`docker/otel-collector/config.yaml` for supervisor mode, and + `docker/otel-collector/config.standalone.yaml` for standalone mode). The + OpAMP remote config no longer sets `processors:` on these pipelines, so the + bootstrap+custom merge wins. Receivers and exporters are still configured + dynamically by the OpAMP controller. + + To override `memory_limiter`, define a new processor with a different name + in `CUSTOM_OTELCOL_CONFIG_FILE` and swap the pipeline `processors:` lists: + + ```yaml + processors: + memory_limiter/custom: + check_interval: 5s + limit_percentage: 75 + spike_limit_percentage: 25 + + service: + pipelines: + traces: + processors: [memory_limiter/custom, batch] + metrics: + processors: [memory_limiter/custom, batch] + logs/out-default: + processors: [memory_limiter/custom, transform, batch] + logs/out-rrweb: + processors: [memory_limiter/custom, batch] + ``` + + The default `memory_limiter` block defined in the base config is left in + the merged config but is no longer referenced by any pipeline; the + collector only instantiates `memory_limiter/custom` at runtime. + + The same swap pattern works for the `batch` processor (and any other base + processor). For example, to lower the export timeout on a specific + pipeline: + + ```yaml + processors: + batch/lowlatency: + send_batch_size: 1000 + send_batch_max_size: 2000 + timeout: 500ms + + service: + pipelines: + traces: + processors: [memory_limiter, batch/lowlatency] + logs/out-default: + processors: [memory_limiter, transform, batch/lowlatency] + ``` + + Lighter-weight env-var tuning is also available for the default `batch` + processor without writing a custom config file: + `HYPERDX_OTEL_BATCH_SEND_BATCH_SIZE`, + `HYPERDX_OTEL_BATCH_SEND_BATCH_MAX_SIZE`, and `HYPERDX_OTEL_BATCH_TIMEOUT`. + See the README for details. + +### Patch Changes + +- d1342121: feat(mcp): add hyperdx_describe_source tool and slim list_sources to catalog + + Add `hyperdx_describe_source` — returns full column schema, map attribute + keys, and sampled low-cardinality values (SeverityText, StatusCode, + ServiceName, etc.) for a single source. Uses existing rollup tables for + performant value sampling. + + Slim `hyperdx_list_sources` to a lightweight MongoDB-only catalog (no + ClickHouse queries). Source tools moved to a dedicated `tools/sources/` + module. + + All query tool descriptions and prompts updated to reference the two-step + `list_sources → describe_source` discovery workflow. + +- a945fa07: feat(mcp): add hyperdx_event_deltas tool + + Add `hyperdx_event_deltas` MCP tool that compares two row groups (target + vs baseline) and ranks properties by how much their value distributions + differ. Same algorithm as the in-app Event Deltas view. + + Extract shared event-deltas algorithm from the UI into + `@hyperdx/common-utils/src/core/eventDeltas.ts` so it can be used by + both the frontend and the MCP server. + +- e1c4381b: fix: bare-text Lucene search now falls back from Implicit Column Expression to + Body Expression on log sources + + Previously, a log source configured with `bodyExpression` set but + `implicitColumnExpression` unset threw `Can not search bare text without an +implicit column set.` on every bare-token search, even though the row panel + rendered correctly using the body column. + + Search now reuses the same one-way fallback that `getEventBody` already + implements: when no Implicit Column Expression is set, bare-text search runs + against the configured Body Expression. Trace sources are unchanged + (`spanNameExpression` is not a body equivalent for trace search). + +- c3a8aa55: feat(mcp): rewrite dashboard authoring prompts and expose `filters` on `hyperdx_save_dashboard` + + The `create_dashboard` prompt now leads with a design checklist (alias every select item including number tiles, schema gap on `groupBy` so tables don't render `arrayElement(SpanAttributes, '...')` as the column header, RED columns with aliases, per-series `numberFormat` for durations, `groupByColumnsOnLeft` for inventory tables, dashboard-level filters instead of per-tile `where` literals, one-metric-per-tile for metric sources, required containers at five or more tiles, post-save validation of every tile, no title-recap markdown). The wall-of-JSON canonical example is gone; the `dashboard_examples` patterns carry the concrete shapes. + + The `dashboard_examples` set is replaced with four verified patterns (`service_inventory`, `service_detail`, `log_analytics`, `backend_dependencies`) plus the existing `infrastructure_sql`. Each non-SQL example leads with a "When to use" header and a "Why this shape" note so the model picks by intent, not by surface keyword match. Examples were built and rendered on a live dev stack before landing. + + The `query_guide` prompt gains a `DASHBOARD FILTERS` section that documents the `filters: [{ type, name, expression, sourceId, where?, whereLanguage? }]` shape, a `NUMBER FORMAT` section that explains the per-series vs. chart-level distinction, and a `PER-TILE TYPE CONSTRAINTS` note that metric tiles take exactly one select item per tile. + + `hyperdx_save_dashboard` now accepts `filters` on its input schema, reusing `externalDashboardFilterSchemaWithId` so the MCP and REST surfaces stay in lockstep and the existing `convertExternalFiltersToInternal` helper handles the conversion without translation. Filters round-trip through create, get, and update. + + Voice pass: every prompt string is now em-dash-free. + +- a4b9fa85: feat(mcp): improve MCP tool quality — error hints, shared helpers, better messages + + Extract duplicated ClickHouse error handling into a shared helper with + pattern-matched error hints (DateTime64 casting, AS alias quoting, response + size limits) so agents get actionable guidance on common failures. Add + reusable mergeWhereIntoSelectItems() helper for consistent top-level where + injection. Improve source/connection-not-found messages to suggest calling + hyperdx_list_sources. + +- 07911fd2: feat(mcp): add trace waterfall and breakdown tools + + Add `hyperdx_trace_waterfall` — fetch all spans in a single trace as a + parent/child waterfall tree with optional correlated logs. Supports + auto-pick by slowest, first error, or most recent trace. + + Add `hyperdx_trace_top_time_consuming_operations` — aggregate breakdown + of child operations consuming the most cumulative time across traces + matching a parent-span filter. Same algorithm as the in-app "Top Most + Time Consuming Operations" chart. + +- 04a5a925: feat: Add source scoping to dashboard filters +- 8810ff0f: feat: Add option for force-enabling/disabling text index support +- a8eb27dc: feat: filters reflect all values, not search aware; filters use metadata MVs if available +- Updated dependencies [3123db53] +- Updated dependencies [dcab1cb6] +- Updated dependencies [a945fa07] +- Updated dependencies [1df7583d] +- Updated dependencies [6a5ac3e3] +- Updated dependencies [e1c4381b] +- Updated dependencies [b30dfe0a] +- Updated dependencies [dcb85826] +- Updated dependencies [b5148c85] +- Updated dependencies [04a5a925] +- Updated dependencies [8810ff0f] +- Updated dependencies [a8eb27dc] + - @hyperdx/common-utils@0.20.0 + ## 2.27.0 ### Minor Changes diff --git a/packages/api/package.json b/packages/api/package.json index b3e534a354..9874759096 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -1,6 +1,6 @@ { "name": "@hyperdx/api", - "version": "2.27.0", + "version": "2.28.0", "license": "MIT", "private": true, "engines": { @@ -11,7 +11,7 @@ "@ai-sdk/openai": "^3.0.47", "@braintree/sanitize-url": "^7.1.1", "@esm2cjs/p-queue": "^7.3.0", - "@hyperdx/common-utils": "^0.19.1", + "@hyperdx/common-utils": "^0.20.0", "@hyperdx/node-opentelemetry": "^0.9.0", "@hyperdx/passport-local-mongoose": "^9.0.1", "@modelcontextprotocol/sdk": "^1.27.1", diff --git a/packages/app/CHANGELOG.md b/packages/app/CHANGELOG.md index 86ccc6c5ff..f8c17791c2 100644 --- a/packages/app/CHANGELOG.md +++ b/packages/app/CHANGELOG.md @@ -1,5 +1,230 @@ # @hyperdx/app +## 2.28.0 + +### Minor Changes + +- 3123db53: feat: experimental promql support +- 1df7583d: feat: emit Lucene conditions from sidebar/dashboard filters to enable KV items direct_read optimization on Map columns + + Legacy `type: 'sql'` filters in URLs are automatically migrated to Lucene + on page load. The persisted `DashboardFilter.expression` in MongoDB is unchanged. + +- cb6a74ce: fix(otel-collector): allow `CUSTOM_OTELCOL_CONFIG_FILE` to override the + default `memory_limiter`, `batch` (and other pipeline processors) + + Pipeline `processors:` lists used to be defined in the OpAMP remote config + sent by the API (`packages/api/src/opamp/controllers/opampController.ts`). + That meant the remote config overwrote any pipeline `processors:` list a + user supplied via `CUSTOM_OTELCOL_CONFIG_FILE`, making it impossible to + substitute the default `memory_limiter` with one configured for + `limit_percentage`/`spike_limit_percentage` mode (#2145). + + The pipeline `processors:` lists now live in the bootstrap config + (`docker/otel-collector/config.yaml` for supervisor mode, and + `docker/otel-collector/config.standalone.yaml` for standalone mode). The + OpAMP remote config no longer sets `processors:` on these pipelines, so the + bootstrap+custom merge wins. Receivers and exporters are still configured + dynamically by the OpAMP controller. + + To override `memory_limiter`, define a new processor with a different name + in `CUSTOM_OTELCOL_CONFIG_FILE` and swap the pipeline `processors:` lists: + + ```yaml + processors: + memory_limiter/custom: + check_interval: 5s + limit_percentage: 75 + spike_limit_percentage: 25 + + service: + pipelines: + traces: + processors: [memory_limiter/custom, batch] + metrics: + processors: [memory_limiter/custom, batch] + logs/out-default: + processors: [memory_limiter/custom, transform, batch] + logs/out-rrweb: + processors: [memory_limiter/custom, batch] + ``` + + The default `memory_limiter` block defined in the base config is left in + the merged config but is no longer referenced by any pipeline; the + collector only instantiates `memory_limiter/custom` at runtime. + + The same swap pattern works for the `batch` processor (and any other base + processor). For example, to lower the export timeout on a specific + pipeline: + + ```yaml + processors: + batch/lowlatency: + send_batch_size: 1000 + send_batch_max_size: 2000 + timeout: 500ms + + service: + pipelines: + traces: + processors: [memory_limiter, batch/lowlatency] + logs/out-default: + processors: [memory_limiter, transform, batch/lowlatency] + ``` + + Lighter-weight env-var tuning is also available for the default `batch` + processor without writing a custom config file: + `HYPERDX_OTEL_BATCH_SEND_BATCH_SIZE`, + `HYPERDX_OTEL_BATCH_SEND_BATCH_MAX_SIZE`, and `HYPERDX_OTEL_BATCH_TIMEOUT`. + See the README for details. + +- b8f51ed9: feat: upgrade to clickhouse-server 26.5 + +### Patch Changes + +- 55926e5c: fix: "Add to Filters" on a JSON-typed ClickHouse column no longer produces an + unparseable Lucene query + + Previously, clicking "Add to Filters" on a field under a JSON column wrapped + the field path with `toString(...)` before handing it off as a Lucene filter + key. Lucene's grammar forbids parentheses inside field names, so the resulting + condition like `toString(JSONColumn.\`foo\`):"…"`failed to parse with`Expected … but ":" found.` + + The handler now passes the clean dot-notation path (e.g. `JSONColumn.foo`) + to the filter setter. + +- 1648c22c: feat(dashboard): add Table of Contents right rail with bulk collapse/expand + + Adds a toggleable right-rail Table of Contents to the dashboard page, plus + "Collapse all sections" and "Expand all sections" actions. All three live + under a new "View" section in the dashboard's existing menu. TOC visibility + is persisted per-user via localStorage; bulk collapse uses the same + per-viewer URL state as single-section toggling, so it's shareable via link + and does not change the dashboard's stored defaults. Clicking a TOC entry + scrolls the section into view, auto-expanding it first if collapsed. + +- 937e043a: fix: collapse duplicate map sub-key entries in the search filter sidebar (HDX-4340) + + A map sub-field stored in `filterState` under dot notation (e.g. `LogAttributes.time`, + from a Lucene URL round-trip) and the same key returned by the facet query under + bracket notation (e.g. `LogAttributes['time']`) no longer render as two separate + accordion items. The merged entry keeps the bracket form so "Load more" stays + valid, and the user's selection still resolves via a tolerant filterState lookup. + +- dcab1cb6: feat: default the direct_read map column optimization on supported ClickHouse versions + + The full-text-search logs schema (`00002_otel_logs.sql`) now ships with + `ResourceAttributeItems`, `ScopeAttributeItems`, and `LogAttributeItems` + ALIAS columns plus their `text(tokenizer='array')` skip indexes. The + traces schema (`00005_otel_traces.sql`) similarly gains + `ResourceAttributeItems` and `SpanAttributeItems` ALIAS columns with + matching items indexes. New installs and freshly migrated tables get + the optimization automatically — no manual `ALTER TABLE` required. + + Note: the traces table previously used only `bloom_filter` skip indexes + and worked on any ClickHouse version. The added `text(tokenizer='array')` + items indexes raise the minimum ClickHouse version required to **create** + the traces table to **>= 26.2**. Existing tables on older clusters are + unaffected (`CREATE TABLE IF NOT EXISTS` is a no-op). + + At query time, the app gates the `Map['key'] = 'value'` → + `has(, concat('key', '=', 'value'))` rewrite on the connected + ClickHouse server version (`SELECT version()`, cached per connection). + The gate only applies to **ALIAS** items columns, which are computed at + query time and therefore depend on the server being able to perform a + direct_read against the underlying Map's tuple storage. The direct_read + feature was backported into multiple stable 26.x release lines, so the + gate uses a per-branch minimum: + + - 26.2 line: >= 26.2.19.43 + - 26.3 line: >= 26.3.12.3 + - 26.4 line: >= 26.4.3.37 + - 26.5+ : always supported + + ALIAS items columns on servers below their branch's threshold continue + to compile filters into the original Map-subscript form. + + **MATERIALIZED items columns are always used when available**, regardless + of ClickHouse version. MATERIALIZED columns are physically stored on + disk, so `has(items, ...)` reads them directly and works on any + ClickHouse version that supports the text index itself. Operators who + want the optimization on servers below the backport cutoffs can + `ALTER TABLE` to materialize the items columns. + +- 923b544b: feat: preserve compatible filters when switching sources +- b94b8eff: fix: persist column widths in search results table +- 996a1139: fix(row-panel): hide empty attribute sections and stop showing "[Empty]" + when the source's body column isn't configured + + The row-expand side panel always rendered `Log/Span Attributes` and + `Resource Attributes` accordion sections, even when both were empty. The + body header fell back to a literal `[Empty]` paper in two visually + identical cases that meant different things: the body column was + configured but the value was empty, or the body column wasn't configured + on the source at all. + + The two attribute accordions now mirror the existing `topLevelAttributes` + pattern and only render when their content is non-empty. The body header + takes a new `bodyConfigured` prop: when `false` (source has neither body + nor implicit column expression configured), the body paper is suppressed + entirely. When `true` and the content is empty, the placeholder reads + "No body for this event." instead of `[Empty]`. + + `DBRowOverviewPanel` derives `bodyConfigured` from + `getEventBody(source) !== undefined`, which already returns `undefined` + when neither expression is set. + + Fixes HDX-4373. + +- e1c4381b: fix: bare-text Lucene search now falls back from Implicit Column Expression to + Body Expression on log sources + + Previously, a log source configured with `bodyExpression` set but + `implicitColumnExpression` unset threw `Can not search bare text without an +implicit column set.` on every bare-token search, even though the row panel + rendered correctly using the body column. + + Search now reuses the same one-way fallback that `getEventBody` already + implements: when no Implicit Column Expression is set, bare-text search runs + against the configured Body Expression. Trace sources are unchanged + (`spanNameExpression` is not a body equivalent for trace search). + +- 19cd7c91: fix: only use pk and row uniqueness to look up a row +- a44fa21b: Number tile: pick a static color from the palette in Display Settings. The color picker stores a palette token (not a hex value) so the choice reflows correctly across light, dark, and IDE themes. +- b5148c85: Dashboard table tiles configured with a row-click action now show a hover hint describing where the click will go (for example, `Search HyperDX Logs` or `Open dashboard "API Latency Drilldown"`). The cell wrapper is now a real link, so cmd-click and middle-click open the destination in a new tab, right-click shows the browser context menu with "Open in New Tab" and "Copy Link Address", and the destination URL appears in the browser status bar on hover. Keyboard users can Tab to a cell and press Enter to navigate, with a visible focus ring. +- 0e8a5b39: chore: use `PageHeader` `title` prop on Alerts, Dashboards, and Saved Searches list pages for consistency with the shared header API. +- 800081c5: chore: migrate the ClickHouse dashboard to shared `PageLayout` with breadcrumbs and a sticky header (connection selector, time range, refresh) instead of a duplicate page title. +- 41d67603: chore: migrate the Kubernetes dashboard to shared `PageLayout` with breadcrumbs and a sticky header (log + metric sources, time range, refresh) instead of a duplicate page title. +- 4d248bf4: chore: migrate Service Map to shared `PageLayout` with a sticky toolbar (source, sampling, time range) and no duplicate page title. +- 633eda61: refactor: Use new VirtualMultiSelect for dashboard filter inputs +- 8938b05e: fix: let "Load more" surface unselected values in exact filter mode +- 04a5a925: feat: Add source scoping to dashboard filters +- b24cb88c: fix(app): copy correct session URL on first Share Session click + + The Share Session button captured `window.location.href` at render time, which ran before `nuqs` flushed `sid`/`sfrom`/`sto` into the URL. The button now reads the URL at click time via the shared `copyTextToClipboard` util, so the first copy always contains the session params (no reload needed). + +- 8810ff0f: feat: Add option for force-enabling/disabling text index support +- a8eb27dc: feat: filters reflect all values, not search aware; filters use metadata MVs if available +- Updated dependencies [3123db53] +- Updated dependencies [d1342121] +- Updated dependencies [dcab1cb6] +- Updated dependencies [a945fa07] +- Updated dependencies [1df7583d] +- Updated dependencies [cb6a74ce] +- Updated dependencies [6a5ac3e3] +- Updated dependencies [e1c4381b] +- Updated dependencies [b30dfe0a] +- Updated dependencies [dcb85826] +- Updated dependencies [c3a8aa55] +- Updated dependencies [a4b9fa85] +- Updated dependencies [07911fd2] +- Updated dependencies [b5148c85] +- Updated dependencies [04a5a925] +- Updated dependencies [8810ff0f] +- Updated dependencies [a8eb27dc] + - @hyperdx/common-utils@0.20.0 + - @hyperdx/api@2.28.0 + ## 2.27.0 ### Patch Changes diff --git a/packages/app/package.json b/packages/app/package.json index 931a238491..86edb4b74e 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -1,6 +1,6 @@ { "name": "@hyperdx/app", - "version": "2.27.0", + "version": "2.28.0", "private": true, "license": "MIT", "engines": { @@ -38,7 +38,7 @@ "@hookform/resolvers": "^3.9.0", "@hyperdx/api": "workspace:*", "@hyperdx/browser": "^0.22.1", - "@hyperdx/common-utils": "^0.19.1", + "@hyperdx/common-utils": "^0.20.0", "@hyperdx/node-opentelemetry": "^0.9.0", "@mantine/core": "^9.0.0", "@mantine/dates": "^9.0.0", diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.md index 9097522a8b..4f7eececf6 100644 --- a/packages/cli/CHANGELOG.md +++ b/packages/cli/CHANGELOG.md @@ -1,5 +1,24 @@ # @hyperdx/cli +## 0.5.0 + +### Minor Changes + +- 3123db53: feat: experimental promql support + +### Patch Changes + +- b20275c9: fix(cli): exit with non-zero code when `upload-sourcemaps` fails + + The `upload-sourcemaps` command now exits with code 1 when uploads fail + (missing source maps, pre-signed URL request failure, authentication failure, + or any per-file upload failure after retries). Previously these failures were + logged to stderr but the process exited cleanly with code 0, causing CI + pipelines to treat failed uploads as successes. + +- 19cd7c91: fix: only use pk and row uniqueness to look up a row +- 8810ff0f: feat: Add option for force-enabling/disabling text index support + ## 0.4.1 ### Patch Changes diff --git a/packages/cli/package.json b/packages/cli/package.json index b177e6282b..497fc92d9e 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@hyperdx/cli", - "version": "0.4.1", + "version": "0.5.0", "license": "MIT", "type": "module", "repository": { @@ -41,7 +41,7 @@ "devDependencies": { "@clickhouse/client": "^1.12.1", "@clickhouse/client-common": "^1.12.1", - "@hyperdx/common-utils": "^0.19.0", + "@hyperdx/common-utils": "^0.20.0", "@jest/globals": "^30.2.0", "@types/crypto-js": "^4.2.2", "@types/jest": "^29.5.14", diff --git a/packages/common-utils/CHANGELOG.md b/packages/common-utils/CHANGELOG.md index ac146ef76e..f75e287c9b 100644 --- a/packages/common-utils/CHANGELOG.md +++ b/packages/common-utils/CHANGELOG.md @@ -1,5 +1,113 @@ # @hyperdx/common-utils +## 0.20.0 + +### Minor Changes + +- 3123db53: feat: experimental promql support +- dcab1cb6: feat: default the direct_read map column optimization on supported ClickHouse versions + + The full-text-search logs schema (`00002_otel_logs.sql`) now ships with + `ResourceAttributeItems`, `ScopeAttributeItems`, and `LogAttributeItems` + ALIAS columns plus their `text(tokenizer='array')` skip indexes. The + traces schema (`00005_otel_traces.sql`) similarly gains + `ResourceAttributeItems` and `SpanAttributeItems` ALIAS columns with + matching items indexes. New installs and freshly migrated tables get + the optimization automatically — no manual `ALTER TABLE` required. + + Note: the traces table previously used only `bloom_filter` skip indexes + and worked on any ClickHouse version. The added `text(tokenizer='array')` + items indexes raise the minimum ClickHouse version required to **create** + the traces table to **>= 26.2**. Existing tables on older clusters are + unaffected (`CREATE TABLE IF NOT EXISTS` is a no-op). + + At query time, the app gates the `Map['key'] = 'value'` → + `has(, concat('key', '=', 'value'))` rewrite on the connected + ClickHouse server version (`SELECT version()`, cached per connection). + The gate only applies to **ALIAS** items columns, which are computed at + query time and therefore depend on the server being able to perform a + direct_read against the underlying Map's tuple storage. The direct_read + feature was backported into multiple stable 26.x release lines, so the + gate uses a per-branch minimum: + + - 26.2 line: >= 26.2.19.43 + - 26.3 line: >= 26.3.12.3 + - 26.4 line: >= 26.4.3.37 + - 26.5+ : always supported + + ALIAS items columns on servers below their branch's threshold continue + to compile filters into the original Map-subscript form. + + **MATERIALIZED items columns are always used when available**, regardless + of ClickHouse version. MATERIALIZED columns are physically stored on + disk, so `has(items, ...)` reads them directly and works on any + ClickHouse version that supports the text index itself. Operators who + want the optimization on servers below the backport cutoffs can + `ALTER TABLE` to materialize the items columns. + +- 1df7583d: feat: emit Lucene conditions from sidebar/dashboard filters to enable KV items direct_read optimization on Map columns + + Legacy `type: 'sql'` filters in URLs are automatically migrated to Lucene + on page load. The persisted `DashboardFilter.expression` in MongoDB is unchanged. + +### Patch Changes + +- a945fa07: feat(mcp): add hyperdx_event_deltas tool + + Add `hyperdx_event_deltas` MCP tool that compares two row groups (target + vs baseline) and ranks properties by how much their value distributions + differ. Same algorithm as the in-app Event Deltas view. + + Extract shared event-deltas algorithm from the UI into + `@hyperdx/common-utils/src/core/eventDeltas.ts` so it can be used by + both the frontend and the MCP server. + +- 6a5ac3e3: fix(charts): histogram bucket picks the highest-precision DateTime column when + Timestamp Column lists multiple columns + + When a source's `Timestamp Column` listed multiple columns (e.g. + `"EventDate, EventTime"` for partition-pruning), the histogram bucket was + built from only the first token. If that token was a `Date` column, every + row in a day collapsed into a single bar at midnight UTC of that day. + + The bucket resolver now walks the comma-split list, queries each column's + type via metadata, and returns the highest-precision DateTime / DateTime64 + token. Date columns are skipped. If no DateTime-typed token is found, the + original first-token behavior is preserved with a `console.warn`. + + The WHERE clause continues to use the multi-column form, so partition + pruning via the `Date` column keeps working. The same resolved column is + also used for the `argMin` / `argMax` / `min` / `max` time math in delta + expressions. + + Fixes HDX-4371. + +- e1c4381b: fix: bare-text Lucene search now falls back from Implicit Column Expression to + Body Expression on log sources + + Previously, a log source configured with `bodyExpression` set but + `implicitColumnExpression` unset threw `Can not search bare text without an +implicit column set.` on every bare-token search, even though the row panel + rendered correctly using the body column. + + Search now reuses the same one-way fallback that `getEventBody` already + implements: when no Implicit Column Expression is set, bare-text search runs + against the configured Body Expression. Trace sources are unchanged + (`spanNameExpression` is not a body equivalent for trace search). + +- b30dfe0a: fix: support text index on lower(Body) with no preprocessor +- dcb85826: fix: escape colons in Lucene field names so filters on Map sub-keys containing + `:` (e.g. `LogAttributes['foo:bar']`) parse correctly + + `filtersToQuery` now backslash-escapes `:` and `\` in the emitted Lucene field + name, and `parseLuceneFilter` + the SQL serializer decode those placeholders + when consuming the AST so the original key is restored end-to-end. + +- b5148c85: Dashboard table tiles configured with a row-click action now show a hover hint describing where the click will go (for example, `Search HyperDX Logs` or `Open dashboard "API Latency Drilldown"`). The cell wrapper is now a real link, so cmd-click and middle-click open the destination in a new tab, right-click shows the browser context menu with "Open in New Tab" and "Copy Link Address", and the destination URL appears in the browser status bar on hover. Keyboard users can Tab to a cell and press Enter to navigate, with a visible focus ring. +- 04a5a925: feat: Add source scoping to dashboard filters +- 8810ff0f: feat: Add option for force-enabling/disabling text index support +- a8eb27dc: feat: filters reflect all values, not search aware; filters use metadata MVs if available + ## 0.19.1 ### Patch Changes diff --git a/packages/common-utils/package.json b/packages/common-utils/package.json index 4b2e6ff32d..551601deae 100644 --- a/packages/common-utils/package.json +++ b/packages/common-utils/package.json @@ -1,7 +1,7 @@ { "name": "@hyperdx/common-utils", "description": "Common utilities for HyperDX application", - "version": "0.19.1", + "version": "0.20.0", "license": "MIT", "private": true, "files": [ diff --git a/packages/otel-collector/CHANGELOG.md b/packages/otel-collector/CHANGELOG.md index 5aafa5636f..82e33e8c74 100644 --- a/packages/otel-collector/CHANGELOG.md +++ b/packages/otel-collector/CHANGELOG.md @@ -1,5 +1,120 @@ # @hyperdx/otel-collector +## 2.28.0 + +### Minor Changes + +- 3123db53: feat: experimental promql support +- cb6a74ce: fix(otel-collector): allow `CUSTOM_OTELCOL_CONFIG_FILE` to override the + default `memory_limiter`, `batch` (and other pipeline processors) + + Pipeline `processors:` lists used to be defined in the OpAMP remote config + sent by the API (`packages/api/src/opamp/controllers/opampController.ts`). + That meant the remote config overwrote any pipeline `processors:` list a + user supplied via `CUSTOM_OTELCOL_CONFIG_FILE`, making it impossible to + substitute the default `memory_limiter` with one configured for + `limit_percentage`/`spike_limit_percentage` mode (#2145). + + The pipeline `processors:` lists now live in the bootstrap config + (`docker/otel-collector/config.yaml` for supervisor mode, and + `docker/otel-collector/config.standalone.yaml` for standalone mode). The + OpAMP remote config no longer sets `processors:` on these pipelines, so the + bootstrap+custom merge wins. Receivers and exporters are still configured + dynamically by the OpAMP controller. + + To override `memory_limiter`, define a new processor with a different name + in `CUSTOM_OTELCOL_CONFIG_FILE` and swap the pipeline `processors:` lists: + + ```yaml + processors: + memory_limiter/custom: + check_interval: 5s + limit_percentage: 75 + spike_limit_percentage: 25 + + service: + pipelines: + traces: + processors: [memory_limiter/custom, batch] + metrics: + processors: [memory_limiter/custom, batch] + logs/out-default: + processors: [memory_limiter/custom, transform, batch] + logs/out-rrweb: + processors: [memory_limiter/custom, batch] + ``` + + The default `memory_limiter` block defined in the base config is left in + the merged config but is no longer referenced by any pipeline; the + collector only instantiates `memory_limiter/custom` at runtime. + + The same swap pattern works for the `batch` processor (and any other base + processor). For example, to lower the export timeout on a specific + pipeline: + + ```yaml + processors: + batch/lowlatency: + send_batch_size: 1000 + send_batch_max_size: 2000 + timeout: 500ms + + service: + pipelines: + traces: + processors: [memory_limiter, batch/lowlatency] + logs/out-default: + processors: [memory_limiter, transform, batch/lowlatency] + ``` + + Lighter-weight env-var tuning is also available for the default `batch` + processor without writing a custom config file: + `HYPERDX_OTEL_BATCH_SEND_BATCH_SIZE`, + `HYPERDX_OTEL_BATCH_SEND_BATCH_MAX_SIZE`, and `HYPERDX_OTEL_BATCH_TIMEOUT`. + See the README for details. + +### Patch Changes + +- ad3f1c9e: fix(otel-collector): skip string severity inference when JSON body has a + `level`/`severity` field + + When the log body parsed as JSON and contained a level-like field, the + pipeline still ran its `\b(alert|crit|emerg|fatal|error|err|warn|notice|debug|dbug|trace)` + keyword scan over the raw body string. The leading-only `\b` boundary + matched any word starting with a severity keyword, so bodies containing + words like `alertmanager`, `alerting`, `errors`, `warning`, etc. produced + the wrong severity. A Grafana sidecar log with body + `{"level":"INFO", "msg":"... mimir-alertmanager-dashboard ..."}` was being + tagged `SeverityText="fatal"`, `SeverityNumber=21` because `alert` matched + inside `alertmanager`, even though the JSON `level` said `INFO`. + + A new OTTL `log_statements` block in + `docker/otel-collector/config.yaml` runs between the existing JSON-parse + block and the string-inference block. It promotes a JSON-derived level + field (now in `log.attributes`) to `log.severity_text`, which causes the + string-inference block to be skipped via its existing + `severity_number == 0 and severity_text == ""` guard. The block is + case-insensitive across keys by enumerating common casings of common field + names used by mainstream logging frameworks: `level` / `Level` / `LEVEL` + (pino, winston, zerolog, zap, logrus, slog, Serilog, NLog), + `severity` / `Severity` / `SEVERITY` (Datadog, GCP Cloud Logging), and + `log.level` (Elastic ECS, flattened from nested JSON). Each `set` + self-guards on `severity_text == ""` so the first match wins (priority: + `level` > `severity` > `log.level`). The block as a whole is gated on no + producer-set severity, so explicit producer values are always preserved. + + `severity_number` is mapped via case-insensitive `(?i)` regex over + `severity_text`, mirroring the existing string-inference keyword set. + Unrecognized values (e.g. `"verbose"`) fall back to `INFO`, matching + block 2's else-branch. The existing `ConvertCase(severity_text, "lower")` + normalization is unchanged. + + Behavior preserved for: non-JSON bodies, JSON bodies without a level + field, and any log record where the producer already set + `severity_text` or `severity_number`. + + Fixes HDX-4383. + ## 2.27.0 ## 2.26.0 diff --git a/packages/otel-collector/package.json b/packages/otel-collector/package.json index d72289bf49..61b68c11c9 100644 --- a/packages/otel-collector/package.json +++ b/packages/otel-collector/package.json @@ -1,7 +1,7 @@ { "name": "@hyperdx/otel-collector", "description": "HyperDX OpenTelemetry Collector configuration and Docker image", - "version": "2.27.0", + "version": "2.28.0", "license": "MIT", "private": true } diff --git a/yarn.lock b/yarn.lock index 7a65e13a43..7b481164c2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4337,7 +4337,7 @@ __metadata: "@ai-sdk/openai": "npm:^3.0.47" "@braintree/sanitize-url": "npm:^7.1.1" "@esm2cjs/p-queue": "npm:^7.3.0" - "@hyperdx/common-utils": "npm:^0.19.1" + "@hyperdx/common-utils": "npm:^0.20.0" "@hyperdx/node-opentelemetry": "npm:^0.9.0" "@hyperdx/passport-local-mongoose": "npm:^9.0.1" "@modelcontextprotocol/sdk": "npm:^1.27.1" @@ -4426,7 +4426,7 @@ __metadata: "@hookform/resolvers": "npm:^3.9.0" "@hyperdx/api": "workspace:*" "@hyperdx/browser": "npm:^0.22.1" - "@hyperdx/common-utils": "npm:^0.19.1" + "@hyperdx/common-utils": "npm:^0.20.0" "@hyperdx/node-opentelemetry": "npm:^0.9.0" "@jedmao/location": "npm:^3.0.0" "@mantine/core": "npm:^9.0.0" @@ -4554,7 +4554,7 @@ __metadata: dependencies: "@clickhouse/client": "npm:^1.12.1" "@clickhouse/client-common": "npm:^1.12.1" - "@hyperdx/common-utils": "npm:^0.19.0" + "@hyperdx/common-utils": "npm:^0.20.0" "@jest/globals": "npm:^30.2.0" "@types/crypto-js": "npm:^4.2.2" "@types/jest": "npm:^29.5.14" @@ -4580,7 +4580,7 @@ __metadata: languageName: unknown linkType: soft -"@hyperdx/common-utils@npm:^0.19.0, @hyperdx/common-utils@npm:^0.19.1, @hyperdx/common-utils@workspace:packages/common-utils": +"@hyperdx/common-utils@npm:^0.20.0, @hyperdx/common-utils@workspace:packages/common-utils": version: 0.0.0-use.local resolution: "@hyperdx/common-utils@workspace:packages/common-utils" dependencies: