From eab2015cad6dadd456b9d07dddec53eae6bd89a7 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Sat, 22 Aug 2026 17:08:52 -0500 Subject: [PATCH 1/9] Isolate Elasticsearch 9.5 experiment resources --- src/Exceptionless.AppHost/appsettings.Development.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/Exceptionless.AppHost/appsettings.Development.json b/src/Exceptionless.AppHost/appsettings.Development.json index 0c208ae918..832d220e44 100644 --- a/src/Exceptionless.AppHost/appsettings.Development.json +++ b/src/Exceptionless.AppHost/appsettings.Development.json @@ -1,4 +1,11 @@ { + "Elasticsearch": { + "Port": 9215, + "ContainerName": "Exceptionless-Elasticsearch-9-5-Lookup", + "DataVolume": "exceptionless.elasticsearch-9-5-lookup.data.v1", + "KibanaPort": 5615, + "KibanaContainerName": "Exceptionless-Kibana-9-5-Lookup" + }, "Logging": { "LogLevel": { "Default": "Information", From fcaf3763afa102a1ebceb0c028b278bfd5fbfd8b Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Sat, 22 Aug 2026 18:06:27 -0500 Subject: [PATCH 2/9] Experiment with lookup joins for stack rollups --- .../design.md | 175 +++++++ .../proposal.md | 80 +++ .../specs/search-and-stacks/spec.md | 262 ++++++++++ .../tasks.md | 94 ++++ openspec/config.yaml | 24 + src/Exceptionless.Core/Bootstrapper.cs | 1 + .../Configuration/ElasticsearchOptions.cs | 2 + .../Configuration/Indexes/StackIndex.cs | 21 +- .../Services/StackRollupSearchService.cs | 487 ++++++++++++++++++ .../Api/Handlers/EventHandler.cs | 105 +++- .../src/routes/(app)/stack/+page.svelte | 35 +- .../Api/Endpoints/EventEndpointTests.cs | 89 ++++ .../Search/StackIndexTests.cs | 17 + tests/http/events.http | 6 + 14 files changed, 1382 insertions(+), 16 deletions(-) create mode 100644 openspec/changes/experiment-esql-lookup-join-stack-pagination/design.md create mode 100644 openspec/changes/experiment-esql-lookup-join-stack-pagination/proposal.md create mode 100644 openspec/changes/experiment-esql-lookup-join-stack-pagination/specs/search-and-stacks/spec.md create mode 100644 openspec/changes/experiment-esql-lookup-join-stack-pagination/tasks.md create mode 100644 openspec/config.yaml create mode 100644 src/Exceptionless.Core/Services/StackRollupSearchService.cs diff --git a/openspec/changes/experiment-esql-lookup-join-stack-pagination/design.md b/openspec/changes/experiment-esql-lookup-join-stack-pagination/design.md new file mode 100644 index 0000000000..ac51362e2c --- /dev/null +++ b/openspec/changes/experiment-esql-lookup-join-stack-pagination/design.md @@ -0,0 +1,175 @@ +# Design: ES|QL Lookup-Join Stack Pagination Experiment + +## Decision + +Introduce a narrow `IStackRollupSearchService` for the four event stack-rollup modes. The service may use `ElasticsearchClient` directly because Foundatio.Repositories does not expose ES|QL, while all ordinary stack hydration, project lookup, writes, and direct stack searches remain repository-backed. + +The query joins before aggregating: + +```text +FROM +| KEEP stack_id, count, date, +| RENAME AS event_user +| LOOKUP JOIN + ON stack_id == id AND is_deleted == false + AND QSTR(, {"default_operator": "AND"}) +| WHERE id IS NOT NULL +| STATS users = COUNT_DISTINCT(event_user), + total = SUM(COALESCE(count, 1)), + first_occurrence = MIN(date), + last_occurrence = MAX(date) + BY stack_id +| INLINE STATS total_stacks = COUNT(*) +| WHERE +| SORT , stack_id ASC +| LIMIT +``` + +The localhost Elasticsearch 9.5 capability spike rejected `QSTR` after `STATS` with `verification_exception: [QSTR] function cannot be used after STATS`. Joining before aggregation is therefore required to preserve the existing Lucene-style stack filter contract without translating that language into a second expression language. This removes the 20,000-stack-id materialization and offset-sized terms buckets from the primary rollup query, but the lookup processes matching event rows rather than one aggregated row per stack. The existing project-level `TotalUsers` helper remains repository-backed and can still invoke the legacy active-stack filter on a cache miss; replacing that auxiliary query is a separate follow-up. Those costs are explicit performance gates for the experiment. + +The spike also proved two less obvious constraints. Real event and stack mappings both expose fields such as `id`, so the event input must be projected with `KEEP` and renamed before the join or ES|QL rejects later references as ambiguous. In addition, `QSTR` defaults adjacent clauses to `OR`, while Exceptionless's repository parser treats them as implicit `AND`; every joined stack predicate therefore supplies `default_operator=AND`. + +Authorization, project, event-filter, and date predicates are supplied through the ES|QL REST request's Query DSL `filter`, built by the existing Foundatio event query builder. The prototype currently names the canonical event alias and relies on this pushdown filter. Resolving only the concrete daily indices remains a performance follow-up because a requested day can legitimately have no concrete index and ES|QL does not inherit the repository client's ignore-unavailable behavior. + +The exact physical event and stack field names must come from the index configuration rather than duplicated string literals. User values must be sent as ES|QL parameters; only allow-listed command fragments and validated index names may be composed into query text. + +## Existing paths and scope + +### Event stack-rollup path + +`EventHandler.GetInternalAsync` currently: + +1. validates with `EventStackQueryValidator`, +2. builds a Foundatio daily-event query, +3. lets `EventStackFilterQueryBuilder` search the stack index and inject up to 20,000 ids, +4. requests a terms aggregation sized to `skip + limit + 1`, +5. skips buckets in memory, +6. hydrates stack documents, and +7. joins aggregation metrics to `StackSummaryModel`. + +The experiment replaces steps 3 through 5 for cursor requests. Stack hydration and formatting remain shared with the current code. + +### Direct stack endpoint path + +`StackHandler.GetInternalAsync` searches only the versioned stack index and supports arbitrary stack sort expressions. It should continue using `IStackRepository`. A follow-up can add Foundatio search-after/before tokens with an appended `id` tie-breaker. Sending these stack-only reads through all daily event indices would add cost and change zero-event/time-range semantics without gaining anything from `LOOKUP JOIN`. + +### Legacy page path + +If `page` is present, retain the current aggregation/page implementation for compatibility and A/B comparison. Cursor and page parameters are mutually exclusive. The experiment must collect telemetry that distinguishes lookup-join, legacy-page, and fallback execution. + +## Index topology and migration + +Events remain in the existing `DailyIndex` indices. The ES|QL source must reuse the same start/end index resolution as `.Index(utcStart, utcEnd)` so it does not scan outside retention or include unrelated indices. ES|QL requires all selected shards to be available, so missing/closed daily indices need an explicit no-data or fallback policy rather than a broad wildcard. + +The current prototype deliberately uses the canonical event alias plus an exact Query DSL date filter until that missing-index policy is implemented. The benchmark must include the shard-fan-out cost of this choice; it is not production-ready evidence for the final daily-index resolution requirement. + +The canonical `StackIndex` is a `VersionedIndex`. Create its next version with: + +- `index.mode = lookup`, +- exactly one primary shard, regardless of the general `ElasticsearchOptions.NumberOfShards`, +- the configured replica count, and +- the existing mapping and alias. + +The versioned alias must resolve to exactly one concrete index before the feature is considered ready. Reindex all active and soft-deleted stack documents, validate document counts and representative mappings, then atomically switch the alias using the existing versioned-index migration mechanism. + +Lookup mode's one-primary-shard constraint is the central capacity tradeoff. Stacks are much smaller than events, but the experiment must measure shard size, document count, indexing/update throughput, patch latency, and query heap. Deployments that cannot fit their stack corpus or write rate on one primary shard must keep the feature disabled; the initial experiment must not introduce a dual-write lookup projection without a separate design. + +`index.mode` is a final creation-only setting. The stack index config applies lookup mode only while creating v2 and overrides Foundatio's existing-index settings update to send only mutable replica/priority settings; otherwise every later startup receives a harmless-looking but rejected `PUT _settings` request. + +Normal `IStackRepository` reads, writes, scripts, and deletes continue against the lookup-mode index. Replicas remain available for resilience. Log a clear startup readiness state with alias target, lookup mode, and primary shard count, without logging credentials or query values. + +## Filter preservation + +Continue using `EventStackQueryValidator` and `EventStackFilter` so the public Lucene-style filter contract and premium-feature classification remain unchanged. + +- Resolve the event-only filter through the current visitor and apply it immediately after `FROM`, using `QSTR` or the equivalent generated Query DSL filter. +- Resolve the stack filter through the current stack visitor, including physical field rewrites for `fixed`, `regressed`, `hidden`, and `stack_id`. +- Apply the stack predicate in the `LOOKUP JOIN` condition together with `stack_id == id`, before `STATS`. Elasticsearch 9.5 permits `QSTR` in this position but rejects it after aggregation; every supported filter fixture must still be proven. +- Require a non-null joined stack id and exclude soft-deleted stacks after the join, giving the left join effective inner-join behavior. +- Preserve the current handling of `@!`; do not reinterpret it as part of this refactor. +- Preserve organization/project authorization as an event-source predicate before `STATS`. Never depend on a joined stack field as the only authorization boundary. + +If a validated filter cannot be represented by the 9.5 query, route that request to the legacy implementation and increment a reason-labelled fallback metric. Do not partially apply a filter. + +## Mode semantics + +Use one query shape and an allow-listed mode definition: + +| Mode | Primary sort | Selection detail | +| --- | --- | --- | +| `stack_recent` | `last_occurrence DESC` | latest matching event in the resolved range | +| `stack_frequent` | `total DESC` | sum event `count`, preserving the existing default occurrence behavior | +| `stack_new` | `first_occurrence DESC` | preserve the existing stack first-occurrence range predicate added by `AddFirstOccurrenceFilter` | +| `stack_users` | `users DESC` | approximate distinct user count, matching current cardinality semantics | + +Every mode appends `stack_id ASC` as a tie-breaker. Sort is applied after `LOOKUP JOIN` because Elasticsearch does not preserve a sort performed before a lookup join. + +Hydrate canonical stack documents by returned ids and explicitly restore ES|QL row order before formatting summaries. Missing stack documents are excluded and measured; they must not reorder the remaining page. + +## Cursor contract + +Use a versioned, base64url-encoded JSON cursor containing: + +- cursor schema version, +- mode and canonical sort direction, +- typed primary sort value, +- stack id tie-breaker, +- resolved absolute UTC start/end, +- a hash of organization/project scope, normalized filter, mode, and sort contract, and +- direction (`before` or `after`) if required by the decoder. + +Do not include user identity, tokens, credentials, raw authorization data, or unbounded filter text. Reject unknown versions, malformed values, non-finite numeric values, and fingerprint mismatches with `400 Bad Request`. + +For canonical descending order `(metric DESC, stack_id ASC)`: + +- `after`: `metric < anchor OR (metric == anchor AND stack_id > anchor_id)` +- `before`: `metric > anchor OR (metric == anchor AND stack_id < anchor_id)` + +A `before` query reverses both sort directions, takes `limit + 1`, and reverses the selected items before returning them. Equivalent predicates must be generated for ascending sorts if reused later. Aggregated mode metrics must be non-null; direct stack sorts need an explicit null policy before they can reuse this cursor. + +The cursor freezes a relative time expression to its initially resolved absolute range. It does not freeze mutable event aggregates. New/backfilled events or stack status changes can move a stack across the anchor between requests; this is documented as live keyset pagination rather than snapshot pagination. + +## Totals and limits + +The existing `include=total` behavior must remain available. Because a cursor query limits rows after grouping, total count must be computed before the cursor predicate. Elasticsearch 9.5 supports `INLINE STATS total_stacks = COUNT(*)` between grouping and the cursor predicate, which the localhost spike proved returns the pre-cursor total on every selected row. Compare its result with the current cardinality behavior in tests and omit it when total is not requested. Do not increase `esql.query.result_truncation_max_size`. + +The ES|QL 10,000-row output limit does not prevent this design because the final output is `limit + 1`; `STATS` still processes the full selected source. The service must retain the API's existing limit clamp. + +## Capability, fallback, and errors + +Enable lookup-join execution only when: + +- the operational feature flag is enabled, +- Elasticsearch reports a compatible 9.5 capability, +- the stack alias resolves to one concrete index, +- that index has `index.mode=lookup` and one primary shard, and +- the query is cursor-based and representable without semantic loss. + +Fallback to the legacy query for feature-off, readiness mismatch, explicitly unsupported filters, and classified transient/circuit-breaker failures. Emit structured logs and counters with a bounded reason label. Invalid filters, invalid cursors, authorization failures, and plan-limit failures must retain their public error and must not be hidden by fallback. + +Use a timeout and cancellation token. Record duration, selected daily-index count, returned rows, mode, direction, fallback reason, and response allocation/size where available. Never log raw user filters or cursor contents at normal log levels. + +## Performance experiment + +Build a repeatable integration benchmark fixture with: + +- multiple daily event indices, +- at least tens of thousands of stacks for CI-scale measurement and an optional larger local profile, +- skewed occurrence/user distributions, +- ties on every mode metric, +- active, fixed, regressed, ignored, and soft-deleted stacks, and +- both event-only and stack-only filters. + +Compare legacy and ES|QL paths at the first page and equivalent depths near pages 100, 500, and the current maximum skip. Capture median and p95 wall time across repeated warm runs, request/response bytes, managed allocations, Elasticsearch took time if exposed, and failures/circuit-breakers. A noisy single run is not sufficient evidence. + +Promotion gate: result equivalence must be exact for ids/order/summary fields, shallow-page p95 must not materially regress, and deep-page work must remain bounded by `limit` instead of requested offset. If lookup-join heap/circuit-breaker behavior is worse for representative data, keep the experiment behind the flag or abandon it. + +## Official Elasticsearch references + +- [LOOKUP JOIN command](https://www.elastic.co/docs/reference/query-languages/esql/commands/lookup-join) +- [LOOKUP JOIN prerequisites and limitations](https://www.elastic.co/docs/reference/query-languages/esql/esql-lookup-join) +- [Index mode settings](https://www.elastic.co/docs/reference/elasticsearch/index-settings/index-modules) +- [ES|QL SORT and tie-breakers](https://www.elastic.co/docs/reference/query-languages/esql/commands/sort) +- [ES|QL LIMIT behavior](https://www.elastic.co/docs/reference/query-languages/esql/commands/limit) +- [ES|QL QSTR](https://www.elastic.co/docs/reference/query-languages/esql/functions-operators/search-functions/qstr) +- [ES|QL REST API](https://www.elastic.co/docs/reference/query-languages/esql/esql-rest) diff --git a/openspec/changes/experiment-esql-lookup-join-stack-pagination/proposal.md b/openspec/changes/experiment-esql-lookup-join-stack-pagination/proposal.md new file mode 100644 index 0000000000..5f8745ef7e --- /dev/null +++ b/openspec/changes/experiment-esql-lookup-join-stack-pagination/proposal.md @@ -0,0 +1,80 @@ +# Proposal: Experiment with ES|QL Lookup-Join Stack Pagination + +## Summary + +Replace the growing terms-aggregation/skip path used by event stack-rollup queries with an experimental Elasticsearch 9.5 ES|QL pipeline that: + +1. reads the existing daily event indices for the requested time range, +2. uses `LOOKUP JOIN` to attach and filter the canonical stack document from the versioned stack index, +3. aggregates the filtered event rows by `stack_id`, +4. applies a deterministic mode-specific sort with `stack_id` as the final tie-breaker, and +5. pages with opaque `before` and `after` keyset cursors. + +This specifically targets the `stack_recent`, `stack_frequent`, `stack_new`, and `stack_users` modes on event list endpoints. Those modes power the organization-level stacks page. It also establishes cursor behavior that direct stack endpoints can reuse, while leaving direct stack-only queries on `IStackRepository` unless an event join is actually needed. + +## User-visible behavior + +- The four existing stack-rollup modes return the same stack summary shape, filters, metrics, mode ordering, plan enforcement, and authorization boundaries. +- Requests without `page` use cursor paging. Responses include opaque `before` and `after` links when applicable. +- `after` moves forward in the canonical result order; `before` moves backward and returns items in the same canonical order. +- Existing `page` + `limit` requests remain supported by the legacy implementation during the experiment. +- A request cannot combine `page` with `before` or `after`, or combine `before` with `after`. +- Explicit `sort` remains invalid for the four stack-rollup modes. Their fixed sorts remain: + - `stack_recent`: last occurrence descending + - `stack_frequent`: summed occurrence count descending + - `stack_new`: first occurrence descending, restricted by the existing stack-first-occurrence time behavior + - `stack_users`: distinct users descending +- Cursor tokens are opaque and query-bound. Malformed cursors or cursors reused with a different scope, filter, time range, mode, or sort contract return `400 Bad Request`. +- Cursor paging provides a total deterministic order, not a point-in-time snapshot. Concurrent events can change aggregate values between requests. + +## Classification + +- **Type:** Experimental refactor, Elasticsearch index/query change, additive API pagination behavior +- **Affected areas:** Backend/API, Elasticsearch index configuration and reindexing, event/stack search, pagination headers, configuration, telemetry, integration tests, performance tests +- **OpenSpec justification:** This changes persisted Elasticsearch index settings, replaces a compatibility-sensitive aggregation/filtering path, introduces a new raw ES|QL execution boundary, and changes how public pagination links are produced. + +## Current implementation context + +`EventHandler.GetInternalAsync` implements the four stack modes by requesting a `terms` aggregation on `stack_id`. The bucket size grows with `skip + limit + 1`, the handler skips buckets in memory, fetches stack documents by id, and joins them to aggregation buckets. Stack-only predicates are handled by `EventStackFilterQueryBuilder`, which first searches the stack index, materializes up to 20,000 stack ids, and injects those ids into the event query. This makes deep paging increasingly expensive and creates an explicit document-limit failure mode. + +`StackHandler.GetInternalAsync` is a separate stack-only path. It pages the versioned stack index with page/limit and optionally runs daily-event aggregations for summary values. The initial lookup-join experiment must not force stack-only reads through daily event indices when no event-derived ordering or metrics are needed. + +## Compatibility boundaries + +- Preserve all existing event and stack routes. +- Preserve the `StackSummaryModel` response body and pagination headers. +- Preserve Lucene-style Exceptionless filter syntax and the existing split between event fields and stack fields, including special fields such as `fixed`, `regressed`, and `hidden`. +- Preserve free/premium filter validation and suspended-organization behavior before any ES|QL query executes. +- Preserve stack-mode rejection of explicit `sort`. +- Preserve page/limit behavior as a fallback during the experiment. +- Do not expose ES|QL syntax as a public API contract. +- Do not make a cursor token a durable SDK contract beyond being opaque and reusable with the same query. + +## Non-goals + +- Replacing ordinary event document search-after queries. +- Replacing direct stack-only repository searches with ES|QL when no event aggregation is required. +- Changing the documented Exceptionless filter language. +- Fixing or broadening mixed event/stack boolean filter semantics beyond current behavior. +- Providing point-in-time snapshot pagination for mutable aggregates. +- Raising the ES|QL cluster result limit. +- Supporting cross-cluster event indices in the first experiment. +- Removing the legacy aggregation implementation before correctness and performance gates pass. + +## Rollback and mitigation + +- Guard the ES|QL path with an operational feature flag/kill switch, disabled by default outside explicitly selected environments during the experiment. +- Route page-number requests and unsupported/capability-mismatch cases through the existing implementation. +- Keep the stack alias on the lookup-mode versioned index after an application rollback; normal repository reads and writes remain supported on that index mode. +- If lookup-index migration cannot complete, do not switch the versioned alias and keep the feature disabled. +- Do not silently retry an invalid user filter through the legacy path. Fallback is for capability/readiness/transient execution failures that are separately logged and measured. + +## Success criteria + +The experiment is eligible for follow-up production work only if it: + +- matches the legacy result ids, ordering, summary metrics, filters, and totals for representative fixtures; +- traverses forward and backward without duplicates for a stable fixture, including tied aggregate values; +- removes the `skip + limit` terms-bucket growth and stack-filter id materialization from the primary rollup query; +- demonstrates no material latency or allocation regression at shallow pages and a material improvement at deep pages on representative daily-index data; and +- can be disabled without an index rollback or public API break. diff --git a/openspec/changes/experiment-esql-lookup-join-stack-pagination/specs/search-and-stacks/spec.md b/openspec/changes/experiment-esql-lookup-join-stack-pagination/specs/search-and-stacks/spec.md new file mode 100644 index 0000000000..a041fa4020 --- /dev/null +++ b/openspec/changes/experiment-esql-lookup-join-stack-pagination/specs/search-and-stacks/spec.md @@ -0,0 +1,262 @@ +# Spec: Search and Stacks + +## MODIFIED Requirements + +### Requirement: Stack rollup modes preserve existing summaries and fixed ordering + +Event list endpoints executed in `stack_recent`, `stack_frequent`, `stack_new`, or `stack_users` mode MUST preserve the existing `StackSummaryModel` response contract, mode selection semantics, filters, authorization, plan enforcement, and fixed mode ordering. + +#### Scenario: Most frequent stacks + +Given matching events span one or more daily indices +And their stacks exist in the active versioned stack index +When an authorized caller requests `mode=stack_frequent` +Then results are ordered by summed occurrence count descending +And ties are ordered by stack id ascending +And every summary contains the same metrics and canonical stack fields as the legacy implementation. + +#### Scenario: Most recent stacks + +Given matching events span one or more daily indices +When an authorized caller requests `mode=stack_recent` +Then results are ordered by the latest matching event date descending +And ties are ordered by stack id ascending. + +#### Scenario: New stacks + +Given stacks have first occurrences both inside and outside the resolved time range +When an authorized caller requests `mode=stack_new` +Then only stacks selected by the existing stack-first-occurrence time behavior are returned +And results are ordered by first occurrence descending with stack id ascending as the tie-breaker. + +#### Scenario: Stacks by users + +Given matching events contain repeated and distinct user identities +When an authorized caller requests `mode=stack_users` +Then results are ordered by the existing approximate distinct-user metric descending +And ties are ordered by stack id ascending. + +#### Scenario: Explicit sort remains invalid + +Given a request uses any stack rollup mode +When it supplies an explicit `sort` +Then the API returns the existing bad-request behavior rather than overriding the mode sort. + +### Requirement: Stack rollup filters preserve Exceptionless query behavior + +Stack rollup execution MUST preserve the current Lucene-style filter contract and MUST apply event and stack predicates without widening authorization or returning soft-deleted stacks. + +#### Scenario: Event-only filter + +Given a valid filter references only event fields +When a stack rollup query executes +Then only events matching that filter contribute to stack metrics. + +#### Scenario: Stack-only filter + +Given a valid filter references stack status, tags, title, references, fixed state, regressed state, hidden state, or stack identifiers +When a stack rollup query executes +Then only stacks matching the equivalent current stack-filter behavior are returned. + +#### Scenario: Mixed event and stack filter + +Given a filter contains event and stack fields in a form supported by the current filter visitors +When a stack rollup query executes +Then its result ids match the legacy implementation for the same fixture. + +#### Scenario: Unsupported lossless translation + +Given a validated filter cannot be represented by the lookup-join query without changing semantics +When the experimental path evaluates the request +Then the whole request uses the legacy implementation +And no partial filter is executed. + +#### Scenario: Deleted or missing stack + +Given an event references a soft-deleted or missing stack document +When a stack rollup query executes +Then that stack is not returned. + +#### Scenario: Authorization is applied before aggregation + +Given a caller can access only selected organizations or projects +When a stack rollup query executes +Then inaccessible event documents do not contribute to aggregation metrics +And a joined stack field is not the sole authorization check. + +## ADDED Requirements + +### Requirement: Stack rollup modes support opaque before and after cursors + +Cursor-based stack rollup responses MUST define a deterministic total order and MUST provide opaque cursors that are valid only for the same logical query. + +#### Scenario: Forward traversal + +Given more matching stacks than the requested limit +When the caller follows each `after` link +Then each stable-fixture stack is returned exactly once in canonical mode order +And the last page does not advertise a next page. + +#### Scenario: Backward traversal + +Given the caller is on a page after the first page +When the caller follows the `before` link +Then the preceding items are returned in canonical mode order +And they match the page previously observed for the stable fixture. + +#### Scenario: Equal primary sort values + +Given multiple stacks have the same primary mode metric +When the caller traverses pages whose boundary falls within those ties +Then stack id provides a stable tie-breaker +And no tied stack is skipped or duplicated for the stable fixture. + +#### Scenario: Relative time range is frozen + +Given the initial request uses a relative time expression +When the caller follows a cursor after wall-clock time advances +Then the cursor uses the absolute UTC range resolved for the initial request. + +#### Scenario: Concurrent aggregate changes + +Given matching events or stack status change between cursor requests +When the caller follows a cursor +Then the API maintains its deterministic keyset comparison +But it does not promise point-in-time snapshot traversal. + +#### Scenario: Malformed cursor + +Given `before` or `after` is malformed, has an unknown version, or contains an invalid typed sort value +When the request is validated +Then the API returns `400 Bad Request` +And does not execute either Elasticsearch path. + +#### Scenario: Cursor reused with another query + +Given a valid cursor was issued for one scope, filter, time range, mode, or sort contract +When it is reused with a different logical query +Then the API returns `400 Bad Request`. + +#### Scenario: Conflicting pagination parameters + +Given a request supplies both `before` and `after`, or supplies `page` with either cursor +When the request is validated +Then the API returns `400 Bad Request`. + +### Requirement: Lookup-join execution requires a ready single-shard lookup index + +The experimental lookup-join path MUST execute only when the active stack alias resolves to one concrete lookup-mode index with one primary shard. + +#### Scenario: Ready lookup index + +Given the feature is enabled +And the stack alias resolves to one concrete index with `index.mode=lookup` and one primary shard +When a representable cursor stack-rollup request executes +Then the ES|QL lookup-join path may be used. + +#### Scenario: Multi-shard or standard-mode stack index + +Given the active stack index is not lookup mode or does not have exactly one primary shard +When a stack-rollup request executes +Then the legacy implementation is used +And readiness telemetry identifies the bounded reason. + +#### Scenario: Alias resolves to multiple concrete indices + +Given the stack alias resolves to multiple concrete indices +When a stack-rollup request executes +Then the lookup-join path is not used. + +#### Scenario: Lookup feature disabled + +Given the operational feature flag is disabled +When a stack-rollup request executes +Then existing legacy behavior is preserved without requiring an index rollback. + +### Requirement: Page-number stack rollups remain compatible during the experiment + +Explicit page-number requests MUST continue to use the existing page/limit behavior while cursor results are evaluated. + +#### Scenario: Explicit page request + +Given a request supplies `page` and `limit` without a cursor +When a stack rollup query executes +Then it returns the same result and page-link contract as the legacy implementation. + +#### Scenario: Cursor request avoids offset-sized buckets + +Given a cursor request follows a deep result boundary +When the lookup-join query executes +Then its final row limit is based on `limit + 1` +And it does not size a terms aggregation using the historical page offset. + +### Requirement: Lookup-join failures are observable and safely classified + +Lookup-join execution MUST preserve public validation and authorization errors, support cancellation and timeouts, and emit bounded operational telemetry for fallback. + +#### Scenario: Invalid filter remains invalid + +Given a user submits an invalid search filter +When query validation fails +Then the API returns the existing invalid-filter response +And does not retry through another implementation. + +#### Scenario: Transient lookup execution failure + +Given the lookup path is enabled and a classified transient Elasticsearch failure occurs +When the request can safely use the legacy implementation +Then the request falls back once +And telemetry records a bounded failure category without raw filters or cursors. + +#### Scenario: Request cancellation + +Given the HTTP request is cancelled +When ES|QL is executing +Then cancellation is propagated +And the handler does not start an unbounded fallback query. + +### Requirement: Lookup-mode migration preserves canonical stack repository behavior + +Migrating the versioned stack index to lookup mode MUST preserve active and soft-deleted stack documents and ordinary stack repository operations. + +#### Scenario: Versioned reindex completes + +Given an existing versioned stack index contains active and soft-deleted documents +When the lookup-mode version is built and validated +Then all documents and required mappings are present before the alias switches atomically. + +#### Scenario: Repository operations after cutover + +Given the stack alias points to the lookup-mode index +When Exceptionless reads, saves, patches, or soft-deletes a stack through `IStackRepository` +Then behavior matches the existing repository contract. + +#### Scenario: Migration is not ready + +Given reindex validation fails or the lookup index is not ready +When startup index configuration runs +Then the alias is not switched +And the lookup-join feature remains disabled. + +### Requirement: Experimental performance is measured against the legacy path + +The experimental PR MUST include a repeatable comparison using representative multi-day data before the lookup path is proposed as the default. + +#### Scenario: Correctness comparison + +Given identical representative fixtures +When legacy and lookup-join queries run for every mode, filter family, and measured page depth +Then result ids, order, summaries, and requested totals match. + +#### Scenario: Shallow and deep measurements + +Given repeated warm benchmark runs +When first-page and deep-page cases are measured +Then the report includes median and p95 wall time, allocations or response bytes where available, and failure/circuit-breaker counts +And no conclusion is based on a single run. + +#### Scenario: Promotion gate fails + +Given correctness differs, shallow latency materially regresses, or representative lookup joins create unacceptable heap/circuit-breaker pressure +When the experiment is reviewed +Then the feature remains disabled or the experiment is abandoned without removing the legacy path. diff --git a/openspec/changes/experiment-esql-lookup-join-stack-pagination/tasks.md b/openspec/changes/experiment-esql-lookup-join-stack-pagination/tasks.md new file mode 100644 index 0000000000..a5943ce43f --- /dev/null +++ b/openspec/changes/experiment-esql-lookup-join-stack-pagination/tasks.md @@ -0,0 +1,94 @@ +# Tasks: ES|QL Lookup-Join Stack Pagination Experiment + +## Design and capability spike + +- [x] 1. Prove the Elasticsearch 9.5 query shape against local Aspire Elasticsearch + - Create a temporary daily-event fixture and a single-shard lookup-mode stack index. + - Prove and document that stack `QSTR` requires `LOOKUP JOIN` before `STATS` in Elasticsearch 9.5. + - Prove the stack alias can be used when it resolves to exactly one concrete lookup index. + - Prove stack `QSTR`/Lucene-pushable predicates in the join condition, soft-delete exclusion, aggregate metrics, exact pre-cursor totals, tied forward cursors, and reverse-before ordering. + - Verification: localhost-only REST fixture on isolated Elasticsearch 9.5 port `9215`; no production endpoint. + +- [ ] 2. Add lookup-join capability/readiness detection and an operational kill switch + - Detect feature flag, server capability, alias target count, lookup mode, and primary shard count. + - Add bounded reason-labelled telemetry for disabled/unready/fallback states. + - Verification: targeted tests for ready, feature-off, standard-mode, multi-target alias, and wrong-shard states; `dotnet test --project tests/Exceptionless.Tests -- --filter-class `. + +## Stack index migration + +- [ ] 3. Create the next version of `StackIndex` in lookup mode + - Set `index.mode=lookup`, force one primary shard, retain configured replicas and current mapping/analysis. + - Warn when the general shard configuration is greater than one because stack lookup mode overrides it. + - Verification: index configuration test asserts mode, one primary, replicas, mapping aliases, and analyzer preservation. + +- [ ] 4. Validate versioned reindex and alias cutover + - Reindex active and soft-deleted stack documents. + - Verify counts and representative documents before atomic alias switch. + - Verify normal `IStackRepository` get/save/patch/delete behavior on the lookup-mode index. + - Verification: `dotnet test --project tests/Exceptionless.Tests -- --filter-class StackRepositoryTests` plus a dedicated migration integration test. + +## Query service and filters + +- [ ] 5. Add a narrow ES|QL stack-rollup search service + - Isolate direct client use behind `IStackRollupSearchService`. + - Reuse the daily-index resolver and pass values as parameters. + - Apply authorization/project/time predicates before aggregation. + - Verification: service integration tests over at least three daily indices and cancellation/timeout tests. + +- [x] 6. Preserve current event/stack filter semantics + - Reuse `EventStackQueryValidator` and `EventStackFilter` visitors. + - Apply event-only predicates at the source and stack predicates in the lookup join. + - Preserve special fields, `@!`, soft-delete exclusion, premium classification, and no-partial-filter fallback. + - Verification: port the full `CheckStackModeCounts`, deleted-stack, premium-search, mixed field, wildcard, range, missing/exists, and invalid-filter matrices to run against legacy and ES|QL paths. + +- [x] 7. Implement mode aggregation and ordered hydration + - Implement recent/frequent/new/users metrics and fixed mode sorts. + - Append `stack_id ASC` to every sort. + - Hydrate through repositories and restore ES|QL row order before formatting. + - Verification: parity tests assert ids, order, summary metrics, tags, project names, and missing-stack handling for every mode. + +## Cursor pagination and API integration + +- [ ] 8. Add versioned query-bound stack-rollup cursors + - Encode typed metric, stack id, resolved UTC range, mode/sort, fingerprint, and version. + - Implement forward and reverse keyset predicates and reverse-before result handling. + - Reject malformed, mismatched, dual-direction, and page-plus-cursor requests. + - Verification: unit tests for all modes, tied values, forward/back traversal, token version/type validation, relative time freezing, and tampering/mismatch errors. + +- [x] 9. Route cursor stack modes through ES|QL while preserving page fallback + - Keep explicit `page` requests on the legacy path. + - Preserve `sort` rejection and response body/status behavior. + - Populate `before`/`after` pagination links with opaque cursors. + - Verification: `dotnet test --project tests/Exceptionless.Tests -- --filter-class EventEndpointTests` and update `tests/http/*.http` for the additive cursor examples. + +- [ ] 10. Preserve optional total behavior + - Compute total before cursor filtering only when requested. + - Compare lookup-join total semantics with current cardinality behavior. + - Verification: endpoint tests for no total, first/middle/last page total, empty results, filters, and total parity. + +- [ ] 11. Add cursor paging to direct stack-only endpoints using Foundatio search-after, if included in the experimental PR + - Do not route stack-only reads through ES|QL. + - Preserve arbitrary stack sort and append a deterministic id tie-breaker. + - Keep explicit page requests compatible. + - Verification: `dotnet test --project tests/Exceptionless.Tests -- --filter-class StackEndpointTests` with every supported stack sort, ties, null policy, before/after, and page fallback. + +## Failure handling, performance, and completion + +- [ ] 12. Add classified fallback and observability + - Fall back only for feature/readiness/unsupported/transient categories. + - Preserve invalid filter/cursor, authorization, and plan errors. + - Record mode, direction, duration, selected index count, row count, and bounded fallback reason without raw filters/cursors. + - Verification: fault-injection tests for timeout, cancellation, circuit breaker, invalid query, and feature-off behavior. + +- [ ] 13. Add repeatable legacy-versus-ES|QL benchmark coverage + - Seed multiple daily indices, skewed metrics, ties, statuses, deletions, and large stack cardinality. + - Measure repeated warm first/deep pages and report median/p95, allocations, bytes, and failures. + - Verification: committed benchmark command/script and a checked-in results template; run locally against Aspire and attach measurements to the experimental PR. + +- [x] 14. Run targeted verification and strict OpenSpec validation + - `dotnet build` + - `dotnet test --project tests/Exceptionless.Tests -- --filter-class EventEndpointTests` + - `dotnet test --project tests/Exceptionless.Tests -- --filter-class StackEndpointTests` + - `dotnet test --project tests/Exceptionless.Tests -- --filter-class EventStackFilterQueryTests` + - localhost-only cursor smoke test through Aspire + - `openspec validate experiment-esql-lookup-join-stack-pagination --strict --no-interactive` diff --git a/openspec/config.yaml b/openspec/config.yaml new file mode 100644 index 0000000000..197993681f --- /dev/null +++ b/openspec/config.yaml @@ -0,0 +1,24 @@ +schema: spec-driven + +context: | + Project: exceptionless/Exceptionless. + Exceptionless is a real-time error monitoring platform on ASP.NET Core 10, Aspire, Foundatio.Repositories, Elasticsearch, and Svelte 5. + Event documents are stored in date-partitioned daily indices; stack documents are stored in a versioned index. + Preserve public API routes, response bodies, pagination headers, filter syntax, authorization, configuration keys, and persisted data unless an explicit compatibility decision says otherwise. + Use Foundatio.Repositories for ordinary repository work. A narrowly isolated Elasticsearch client call is acceptable when ES|QL cannot be expressed by Foundatio.Repositories. + Local verification must use Aspire/localhost and must not target production or staging. + +rules: + proposal: + - State the user-visible behavior, classification, affected areas, compatibility risks, non-goals, and rollback plan. + - Explain why OpenSpec is justified. + specs: + - Use ADDED, MODIFIED, and REMOVED requirements. + - Use Given/When/Then scenarios for observable behavior. + - Include negative, authorization, compatibility, and Elasticsearch failure scenarios where relevant. + design: + - Reference existing code paths before introducing abstractions. + - For Elasticsearch changes, cover index/schema impact, query behavior, migration/reindex strategy, performance, operations, and failure handling. + tasks: + - Make tasks independently reviewable and include a verification command or test for each task. + - Finish with strict OpenSpec validation and the smallest sufficient targeted tests. diff --git a/src/Exceptionless.Core/Bootstrapper.cs b/src/Exceptionless.Core/Bootstrapper.cs index 805a2f76d4..812476998c 100644 --- a/src/Exceptionless.Core/Bootstrapper.cs +++ b/src/Exceptionless.Core/Bootstrapper.cs @@ -77,6 +77,7 @@ public static void RegisterServices(IServiceCollection services, AppOptions appO services.AddSingleton(); services.AddSingleton(s => s.GetRequiredService().Client); + services.AddSingleton(); services.AddSingleton(s => s.GetRequiredService()); services.AddStartupAction(); diff --git a/src/Exceptionless.Core/Configuration/ElasticsearchOptions.cs b/src/Exceptionless.Core/Configuration/ElasticsearchOptions.cs index bce6d41cd7..7b96db9fe0 100644 --- a/src/Exceptionless.Core/Configuration/ElasticsearchOptions.cs +++ b/src/Exceptionless.Core/Configuration/ElasticsearchOptions.cs @@ -11,6 +11,7 @@ public class ElasticsearchOptions public int NumberOfReplicas { get; internal set; } public int FieldsLimit { get; internal set; } = 1500; public bool EnableMapperSizePlugin { get; internal set; } + public bool EnableStackRollupLookupJoin { get; internal set; } public string Scope { get; internal set; } = null!; public string ScopePrefix { get; internal set; } = null!; @@ -30,6 +31,7 @@ public static ElasticsearchOptions ReadFromConfiguration(IConfiguration config, options.DisableIndexConfiguration = config.GetValue(nameof(options.DisableIndexConfiguration), false); options.EnableSnapshotJobs = config.GetValue(nameof(options.EnableSnapshotJobs), String.IsNullOrEmpty(options.ScopePrefix) && appOptions.AppMode == AppMode.Production); + options.EnableStackRollupLookupJoin = config.GetValue(nameof(options.EnableStackRollupLookupJoin), appOptions.AppMode == AppMode.Development); options.ReindexCutOffDate = config.GetValue(nameof(options.ReindexCutOffDate), DateTime.MinValue); string? connectionString = config.GetConnectionString("Elasticsearch"); diff --git a/src/Exceptionless.Core/Repositories/Configuration/Indexes/StackIndex.cs b/src/Exceptionless.Core/Repositories/Configuration/Indexes/StackIndex.cs index 02b15bdd89..76c84023ba 100644 --- a/src/Exceptionless.Core/Repositories/Configuration/Indexes/StackIndex.cs +++ b/src/Exceptionless.Core/Repositories/Configuration/Indexes/StackIndex.cs @@ -17,7 +17,7 @@ public sealed class StackIndex : VersionedIndex private readonly ExceptionlessElasticConfiguration _configuration; - public StackIndex(ExceptionlessElasticConfiguration configuration) : base(configuration, configuration.Options.ScopePrefix + "stacks", 1) + public StackIndex(ExceptionlessElasticConfiguration configuration) : base(configuration, configuration.Options.ScopePrefix + "stacks", 2) { _configuration = configuration; } @@ -27,11 +27,28 @@ public override void ConfigureIndex(CreateIndexRequestDescriptor idx) base.ConfigureIndex(idx); idx.Settings(s => s .Analysis(a => BuildAnalysis(a)) - .NumberOfShards(_configuration.Options.NumberOfShards) + .Mode("lookup") + .NumberOfShards(1) .NumberOfReplicas(_configuration.Options.NumberOfReplicas) .Priority(5)); } + protected override Task UpdateIndexAsync(string name, Action? descriptor = null) + { + if (descriptor is not null) + return base.UpdateIndexAsync(name, descriptor); + + // index.mode is a final creation-only setting. Foundatio derives updates from + // ConfigureIndex, so keep the mutable settings explicit for existing indexes. + return base.UpdateIndexAsync(name, update => update + .Reopen(true) + .Settings(new IndexSettings + { + NumberOfReplicas = _configuration.Options.NumberOfReplicas, + Priority = 5 + })); + } + public override void ConfigureIndexMapping(TypeMappingDescriptor map) { map diff --git a/src/Exceptionless.Core/Services/StackRollupSearchService.cs b/src/Exceptionless.Core/Services/StackRollupSearchService.cs new file mode 100644 index 0000000000..d62a3c7eba --- /dev/null +++ b/src/Exceptionless.Core/Services/StackRollupSearchService.cs @@ -0,0 +1,487 @@ +using System.Diagnostics; +using System.Globalization; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.Esql; +using Elastic.Clients.Elasticsearch.QueryDsl; +using Exceptionless.Core.Configuration; +using Exceptionless.Core.Models; +using Exceptionless.Core.Models.Data; +using Exceptionless.Core.Repositories; +using Exceptionless.Core.Repositories.Configuration; +using Exceptionless.Core.Repositories.Queries; +using Foundatio.Repositories; +using Foundatio.Repositories.Elasticsearch.Queries.Builders; +using Foundatio.Repositories.Options; +using Microsoft.Extensions.Logging; + +namespace Exceptionless.Core.Services; + +public interface IStackRollupSearchService +{ + Task SearchAsync(StackRollupSearchRequest request, CancellationToken cancellationToken = default); +} + +public sealed record StackRollupSearchRequest( + AppFilter? AppFilter, + DateTime UtcStart, + DateTime UtcEnd, + TimeSpan Offset, + string? TimeExpression, + string? Filter, + string Mode, + int Limit, + string? Before, + string? After, + bool IncludeTotal); + +public sealed record StackRollupSearchResult( + IReadOnlyCollection Rows, + bool HasMore, + long? Total, + string? Before, + string? After); + +public sealed record StackRollupRow( + string StackId, + long Total, + long Users, + DateTime FirstOccurrence, + DateTime LastOccurrence); + +public sealed class InvalidStackRollupCursorException(string message) : Exception(message); + +public sealed class StackRollupSearchService : IStackRollupSearchService +{ + private const int CursorVersion = 1; + private static readonly TimeSpan ReadinessCacheDuration = TimeSpan.FromMinutes(1); + private readonly ElasticsearchClient _client; + private readonly ExceptionlessElasticConfiguration _configuration; + private readonly ElasticsearchOptions _options; + private readonly TimeProvider _timeProvider; + private readonly JsonSerializerOptions _serializerOptions; + private readonly EventStackFilter _eventStackFilter = new(); + private readonly ILogger _logger; + private readonly SemaphoreSlim _readinessLock = new(1, 1); + private StackRollupReadiness? _readiness; + private DateTimeOffset _readinessExpiresUtc; + + public StackRollupSearchService( + ElasticsearchClient client, + ExceptionlessElasticConfiguration configuration, + ElasticsearchOptions options, + TimeProvider timeProvider, + JsonSerializerOptions serializerOptions, + ILoggerFactory loggerFactory) + { + _client = client; + _configuration = configuration; + _options = options; + _timeProvider = timeProvider; + _serializerOptions = serializerOptions; + _logger = loggerFactory.CreateLogger(); + } + + public async Task SearchAsync(StackRollupSearchRequest request, CancellationToken cancellationToken = default) + { + if (!_options.EnableStackRollupLookupJoin) + return null; + + var readiness = await GetReadinessAsync(cancellationToken); + if (!readiness.IsReady) + { + _logger.LogDebug("Stack rollup lookup join is unavailable: {Reason}", readiness.Reason); + return null; + } + + if (!IsSupportedMode(request.Mode)) + return null; + + string fingerprint = CreateFingerprint(request); + StackRollupCursor? cursor = DecodeCursor(request.Before ?? request.After, request.Mode, fingerprint); + DateTime utcStart = cursor is null ? request.UtcStart : new DateTime(cursor.UtcStart, DateTimeKind.Utc); + DateTime utcEnd = cursor is null ? request.UtcEnd : new DateTime(cursor.UtcEnd, DateTimeKind.Utc); + string normalizedFilter = StripAlternateInversion(request.Filter); + if (request.Mode == "stack_new") + normalizedFilter = AddFirstOccurrenceFilter(utcStart, utcEnd, normalizedFilter); + string? eventFilter = await _eventStackFilter.GetEventFilterAsync(normalizedFilter); + string? stackFilter = (await _eventStackFilter.GetStackFilterAsync(normalizedFilter))?.Filter; + Query? sourceFilter = await BuildSourceFilterAsync(request.AppFilter, utcStart, utcEnd, eventFilter); + + var stopwatch = Stopwatch.StartNew(); + _logger.LogDebug( + "Executing ES|QL stack rollup mode {Mode}, direction {Direction}, range {UtcStart:o} to {UtcEnd:o}, event filter {HasEventFilter}, stack filter {HasStackFilter}", + request.Mode, + request.Before is not null ? "before" : request.After is not null ? "after" : "initial", + utcStart, + utcEnd, + !String.IsNullOrWhiteSpace(eventFilter), + !String.IsNullOrWhiteSpace(stackFilter)); + + var parameters = new List>>(); + string query = BuildQuery(request, cursor, stackFilter, parameters, countOnly: false); + var rows = await ExecuteRowsAsync(query, sourceFilter, parameters, cancellationToken); + long? total = request.IncludeTotal ? rows.FirstOrDefault()?.TotalStacks : null; + + if (request.IncludeTotal && total is null) + { + parameters.Clear(); + string countQuery = BuildQuery(request, cursor: null, stackFilter, parameters, countOnly: true); + total = await ExecuteTotalAsync(countQuery, sourceFilter, parameters, cancellationToken); + } + + bool isBefore = request.Before is not null; + bool hasExtra = rows.Count > request.Limit; + if (hasExtra) + rows.RemoveAt(rows.Count - 1); + + if (isBefore) + rows.Reverse(); + + bool hasPrevious = rows.Count > 0 && (isBefore ? hasExtra : request.After is not null); + bool hasNext = rows.Count > 0 && (isBefore || hasExtra); + string? before = hasPrevious ? EncodeCursor(rows[0], request, utcStart, utcEnd, fingerprint) : null; + string? after = hasNext ? EncodeCursor(rows[^1], request, utcStart, utcEnd, fingerprint) : null; + + _logger.LogDebug( + "Completed ES|QL stack rollup mode {Mode}, direction {Direction}, rows {RowCount}, has more {HasMore}, duration {DurationMs}ms", + request.Mode, + isBefore ? "before" : request.After is not null ? "after" : "initial", + rows.Count, + hasNext, + stopwatch.Elapsed.TotalMilliseconds); + + return new StackRollupSearchResult( + rows.Select(ToPublicRow).ToArray(), + hasNext, + total, + before, + after); + } + + private async Task BuildSourceFilterAsync(AppFilter? appFilter, DateTime utcStart, DateTime utcEnd, string? eventFilter) + { + var query = new RepositoryQuery() + .AppFilter(appFilter) + .DateRange(utcStart, utcEnd, (PersistentEvent e) => e.Date) + .FilterExpression(eventFilter); + + var options = new CommandOptions() + .TimeProvider(_timeProvider) + .ElasticIndex(_configuration.Events) + .DocumentType(typeof(PersistentEvent)); + var context = new QueryBuilderContext(query, options); + await _configuration.Events.QueryBuilder.BuildAsync(context); + return context.Filter; + } + + private string BuildQuery( + StackRollupSearchRequest request, + StackRollupCursor? cursor, + string? stackFilter, + ICollection>> parameters, + bool countOnly) + { + string eventIndex = ValidateIndexName(_configuration.Events.Name); + string stackIndex = ValidateIndexName(_configuration.Stacks.Name); + string userField = EscapeIdentifier(EventIndexExtensions.DataPath(Event.KnownDataKeys.UserInfo, user => user.Identity) + ".keyword"); + var query = new StringBuilder() + .Append("FROM ").Append(eventIndex) + .Append(" | KEEP stack_id, count, date, ").Append(userField) + .Append(" | RENAME ").Append(userField).Append(" AS event_user") + .Append(" | LOOKUP JOIN ").Append(stackIndex) + .Append(" ON stack_id == id AND is_deleted == false"); + + if (!String.IsNullOrWhiteSpace(stackFilter)) + { + query.Append(" AND QSTR(?stack_filter, {\"default_operator\": \"AND\"})"); + AddParameter(parameters, "stack_filter", FieldValue.String(stackFilter)); + } + + query + .Append(" | WHERE id IS NOT NULL") + .Append(" | STATS event_total = SUM(COALESCE(count, 1)), event_users = COUNT_DISTINCT(event_user), event_first = MIN(date), event_last = MAX(date) BY stack_id"); + + if (countOnly) + return query.Append(" | STATS total_stacks = COUNT(*) | KEEP total_stacks").ToString(); + + if (request.IncludeTotal) + query.Append(" | INLINE STATS total_stacks = COUNT(*)"); + + var mode = GetMode(request.Mode); + bool isBefore = request.Before is not null; + if (cursor is not null) + { + string primaryComparison = isBefore ? ">" : "<"; + string idComparison = isBefore ? "<" : ">"; + string metricParameter = mode.IsDate ? "TO_DATETIME(?cursor_metric)" : "?cursor_metric"; + query + .Append(" | WHERE ").Append(mode.Metric).Append(' ').Append(primaryComparison).Append(' ').Append(metricParameter) + .Append(" OR (").Append(mode.Metric).Append(" == ").Append(metricParameter) + .Append(" AND stack_id ").Append(idComparison).Append(" ?cursor_stack_id)"); + AddParameter(parameters, "cursor_metric", mode.IsDate + ? FieldValue.String(new DateTime(cursor.Metric, DateTimeKind.Utc).ToString("O", CultureInfo.InvariantCulture)) + : FieldValue.Long(cursor.Metric)); + AddParameter(parameters, "cursor_stack_id", FieldValue.String(cursor.StackId)); + } + + string primarySort = isBefore ? "ASC" : "DESC"; + string idSort = isBefore ? "DESC" : "ASC"; + query + .Append(" | SORT ").Append(mode.Metric).Append(' ').Append(primarySort).Append(", stack_id ").Append(idSort) + .Append(" | LIMIT ").Append(request.Limit + 1) + .Append(" | KEEP stack_id, event_total, event_users, event_first, event_last"); + + if (request.IncludeTotal) + query.Append(", total_stacks"); + + return query.ToString(); + } + + private async Task> ExecuteRowsAsync( + string query, + Query? sourceFilter, + ICollection>> parameters, + CancellationToken cancellationToken) + { + using var response = await _client.Esql.QueryAsync(new EsqlQueryRequest(query) + { + AllowPartialResults = false, + Columnar = false, + Filter = sourceFilter, + Format = EsqlFormat.Json, + Params = new Union>, ICollection>>>(parameters) + }, cancellationToken); + + if (!response.IsValidResponse) + { + _logger.LogWarning("Stack rollup lookup join failed with Elasticsearch status {StatusCode}", response.ApiCallDetails?.HttpStatusCode); + throw new ApplicationException("The experimental stack rollup query failed."); + } + + using var document = await JsonDocument.ParseAsync(response.Body, cancellationToken: cancellationToken); + return ReadRows(document.RootElement); + } + + private async Task ExecuteTotalAsync( + string query, + Query? sourceFilter, + ICollection>> parameters, + CancellationToken cancellationToken) + { + using var response = await _client.Esql.QueryAsync(new EsqlQueryRequest(query) + { + AllowPartialResults = false, + Columnar = false, + Filter = sourceFilter, + Format = EsqlFormat.Json, + Params = new Union>, ICollection>>>(parameters) + }, cancellationToken); + + if (!response.IsValidResponse) + throw new ApplicationException("The experimental stack rollup count query failed."); + + using var document = await JsonDocument.ParseAsync(response.Body, cancellationToken: cancellationToken); + if (!document.RootElement.TryGetProperty("values", out var values) || values.GetArrayLength() == 0) + return 0; + + return values[0][0].GetInt64(); + } + + private static List ReadRows(JsonElement root) + { + if (!root.TryGetProperty("columns", out var columns) || !root.TryGetProperty("values", out var values)) + throw new JsonException("The ES|QL stack rollup response did not contain columns and values."); + + var columnIndexes = columns.EnumerateArray() + .Select((column, index) => (Name: column.GetProperty("name").GetString(), Index: index)) + .Where(column => column.Name is not null) + .ToDictionary(column => column.Name!, column => column.Index, StringComparer.Ordinal); + + int stackIdIndex = columnIndexes["stack_id"]; + int totalIndex = columnIndexes["event_total"]; + int usersIndex = columnIndexes["event_users"]; + int firstIndex = columnIndexes["event_first"]; + int lastIndex = columnIndexes["event_last"]; + int? totalStacksIndex = columnIndexes.TryGetValue("total_stacks", out int index) ? index : null; + var rows = new List(); + foreach (var value in values.EnumerateArray()) + { + rows.Add(new StackRollupEsqlRow( + value[stackIdIndex].GetString() ?? throw new JsonException("A stack rollup row did not contain a stack id."), + value[totalIndex].GetInt64(), + value[usersIndex].GetInt64(), + value[firstIndex].GetDateTime().ToUniversalTime(), + value[lastIndex].GetDateTime().ToUniversalTime(), + totalStacksIndex.HasValue ? value[totalStacksIndex.Value].GetInt64() : null)); + } + + return rows; + } + + private async Task GetReadinessAsync(CancellationToken cancellationToken) + { + DateTimeOffset now = _timeProvider.GetUtcNow(); + if (_readiness is not null && now < _readinessExpiresUtc) + return _readiness; + + await _readinessLock.WaitAsync(cancellationToken); + try + { + now = _timeProvider.GetUtcNow(); + if (_readiness is not null && now < _readinessExpiresUtc) + return _readiness; + + _readiness = await CheckReadinessAsync(cancellationToken); + _readinessExpiresUtc = now.Add(ReadinessCacheDuration); + return _readiness; + } + finally + { + _readinessLock.Release(); + } + } + + private async Task CheckReadinessAsync(CancellationToken cancellationToken) + { + var info = await _client.InfoAsync(cancellationToken); + if (!info.IsValidResponse || !System.Version.TryParse(info.Version.Number, out var version) || version < new System.Version(9, 5)) + return new StackRollupReadiness(false, "elasticsearch-version"); + + var settings = await _client.Indices.GetSettingsAsync((Indices)_configuration.Stacks.Name, cancellationToken); + if (!settings.IsValidResponse || settings.Settings.Count != 1) + return new StackRollupReadiness(false, "stack-alias-target"); + + var indexSettings = settings.Settings.Single().Value.Settings?.Index; + if (indexSettings is null || !String.Equals(indexSettings.Mode, "lookup", StringComparison.OrdinalIgnoreCase)) + return new StackRollupReadiness(false, "stack-index-mode"); + + int shards = indexSettings.NumberOfShards is null + ? 0 + : indexSettings.NumberOfShards.Match(value => value, value => Int32.TryParse(value, out int parsed) ? parsed : 0); + return shards == 1 + ? new StackRollupReadiness(true, "ready") + : new StackRollupReadiness(false, "stack-primary-shards"); + } + + private string EncodeCursor(StackRollupEsqlRow row, StackRollupSearchRequest request, DateTime utcStart, DateTime utcEnd, string fingerprint) + { + var mode = GetMode(request.Mode); + long metric = mode.Metric switch + { + "event_total" => row.Total, + "event_users" => row.Users, + "event_first" => row.FirstOccurrence.Ticks, + "event_last" => row.LastOccurrence.Ticks, + _ => throw new InvalidOperationException("Unsupported stack rollup metric.") + }; + var cursor = new StackRollupCursor(CursorVersion, request.Mode, metric, row.StackId, utcStart.Ticks, utcEnd.Ticks, fingerprint); + byte[] bytes = JsonSerializer.SerializeToUtf8Bytes(cursor, _serializerOptions); + return Convert.ToBase64String(bytes).TrimEnd('=').Replace('+', '-').Replace('/', '_'); + } + + private StackRollupCursor? DecodeCursor(string? token, string mode, string fingerprint) + { + if (String.IsNullOrWhiteSpace(token)) + return null; + + try + { + string base64 = token.Replace('-', '+').Replace('_', '/'); + base64 = base64.PadRight(base64.Length + (4 - base64.Length % 4) % 4, '='); + var cursor = JsonSerializer.Deserialize(Convert.FromBase64String(base64), _serializerOptions); + if (cursor is null + || cursor.Version != CursorVersion + || !String.Equals(cursor.Mode, mode, StringComparison.Ordinal) + || !String.Equals(cursor.Fingerprint, fingerprint, StringComparison.Ordinal) + || String.IsNullOrWhiteSpace(cursor.StackId) + || cursor.UtcStart < DateTime.MinValue.Ticks + || cursor.UtcStart > DateTime.MaxValue.Ticks + || cursor.UtcEnd < DateTime.MinValue.Ticks + || cursor.UtcEnd > DateTime.MaxValue.Ticks + || cursor.UtcStart > cursor.UtcEnd + || GetMode(mode).IsDate && (cursor.Metric < DateTime.MinValue.Ticks || cursor.Metric > DateTime.MaxValue.Ticks)) + { + throw new InvalidStackRollupCursorException("The stack pagination cursor is not valid for this query."); + } + + return cursor; + } + catch (InvalidStackRollupCursorException) + { + throw; + } + catch (Exception ex) when (ex is FormatException or JsonException or OverflowException) + { + throw new InvalidStackRollupCursorException("The stack pagination cursor is malformed."); + } + } + + private static string CreateFingerprint(StackRollupSearchRequest request) + { + string organizations = String.Join(',', request.AppFilter?.Organizations.Select(organization => organization.Id).Order(StringComparer.Ordinal) ?? Enumerable.Empty()); + string projects = String.Join(',', request.AppFilter?.Projects?.Select(project => project.Id).Order(StringComparer.Ordinal) ?? Enumerable.Empty()); + string value = String.Join('\n', [ + request.Mode, + request.Filter ?? String.Empty, + request.TimeExpression ?? String.Empty, + request.Offset.Ticks.ToString(CultureInfo.InvariantCulture), + organizations, + projects, + request.AppFilter?.Stack?.Id ?? String.Empty + ]); + return Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(value))); + } + + private static StackRollupRow ToPublicRow(StackRollupEsqlRow row) => new( + row.StackId, + row.Total, + row.Users, + row.FirstOccurrence, + row.LastOccurrence); + + private static StackRollupMode GetMode(string mode) => mode switch + { + "stack_recent" => new StackRollupMode("event_last", true), + "stack_frequent" => new StackRollupMode("event_total", false), + "stack_new" => new StackRollupMode("event_first", true), + "stack_users" => new StackRollupMode("event_users", false), + _ => throw new InvalidOperationException("Unsupported stack rollup mode.") + }; + + private static bool IsSupportedMode(string mode) => mode is "stack_recent" or "stack_frequent" or "stack_new" or "stack_users"; + + private static string StripAlternateInversion(string? filter) => filter?.StartsWith("@!", StringComparison.Ordinal) == true ? filter[2..] : filter ?? String.Empty; + + private static string AddFirstOccurrenceFilter(DateTime utcStart, DateTime utcEnd, string? filter) + { + string range = $"first_occurrence:[\"{utcStart:O}\" TO \"{utcEnd:O}\"]"; + return String.IsNullOrWhiteSpace(filter) ? range : $"{range} ({filter})"; + } + + private static string ValidateIndexName(string index) + { + if (String.IsNullOrWhiteSpace(index) || index.Any(character => !Char.IsAsciiLetterOrDigit(character) && character is not '-' and not '_' and not '.')) + throw new InvalidOperationException("The configured Elasticsearch index alias cannot be used in ES|QL."); + + return index; + } + + private static string EscapeIdentifier(string field) => $"`{field.Replace("`", "``", StringComparison.Ordinal)}`"; + + private static void AddParameter(ICollection>> parameters, string name, FieldValue value) + => parameters.Add(new KeyValuePair>(name, [value])); + + private sealed record StackRollupMode(string Metric, bool IsDate); + private sealed record StackRollupReadiness(bool IsReady, string Reason); + private sealed record StackRollupCursor(int Version, string Mode, long Metric, string StackId, long UtcStart, long UtcEnd, string Fingerprint); + private sealed record StackRollupEsqlRow( + string StackId, + long Total, + long Users, + DateTime FirstOccurrence, + DateTime LastOccurrence, + long? TotalStacks); +} diff --git a/src/Exceptionless.Web/Api/Handlers/EventHandler.cs b/src/Exceptionless.Web/Api/Handlers/EventHandler.cs index c1bdb6d5c1..6df4e648dd 100644 --- a/src/Exceptionless.Web/Api/Handlers/EventHandler.cs +++ b/src/Exceptionless.Web/Api/Handlers/EventHandler.cs @@ -48,6 +48,7 @@ public class EventHandler( EventStackQueryValidator stackModeValidator, AppOptions appOptions, UsageService usageService, + IStackRollupSearchService stackRollupSearchService, TimeProvider timeProvider, LinkGenerator linkGenerator, ILoggerFactory loggerFactory) @@ -166,7 +167,7 @@ public async Task>> Handle(GetAllEvents message) var ti = TimeRangeParser.GetTimeInfo(message.Time, message.Offset, timeProvider, _allowedDateFields, DefaultDateField, organizations.GetRetentionUtcCutoff(appOptions.MaximumRetentionDays, timeProvider)); var sf = new AppFilter(organizations) { IsUserOrganizationsFilter = true }; - return await GetInternalAsync(sf, ti, httpContext, message.Filter, message.Sort, message.Mode, message.Page, message.Limit, message.Before, message.After, includeTotal: ShouldIncludeTotal(message.Include)); + return await GetInternalAsync(sf, ti, httpContext, message.Filter, message.Sort, message.Mode, message.Page, message.Limit, message.Before, message.After, includeTotal: ShouldIncludeTotal(message.Include), timeExpression: message.Time); } public async Task>> Handle(GetEventsByOrganization message) @@ -181,7 +182,7 @@ public async Task>> Handle(GetEventsByOrganization me var ti = TimeRangeParser.GetTimeInfo(message.Time, message.Offset, timeProvider, _allowedDateFields, DefaultDateField, organization.GetRetentionUtcCutoff(appOptions.MaximumRetentionDays, timeProvider)); var sf = new AppFilter(organization); - return await GetInternalAsync(sf, ti, httpContext, message.Filter, message.Sort, message.Mode, message.Page, message.Limit, message.Before, message.After, includeTotal: ShouldIncludeTotal(message.Include)); + return await GetInternalAsync(sf, ti, httpContext, message.Filter, message.Sort, message.Mode, message.Page, message.Limit, message.Before, message.After, includeTotal: ShouldIncludeTotal(message.Include), timeExpression: message.Time); } public async Task>> Handle(GetEventsByProject message) @@ -200,7 +201,7 @@ public async Task>> Handle(GetEventsByProject message var ti = TimeRangeParser.GetTimeInfo(message.Time, message.Offset, timeProvider, _allowedDateFields, DefaultDateField, organization.GetRetentionUtcCutoff(project, appOptions.MaximumRetentionDays, timeProvider)); var sf = new AppFilter(project, organization); - return await GetInternalAsync(sf, ti, httpContext, message.Filter, message.Sort, message.Mode, message.Page, message.Limit, message.Before, message.After, includeTotal: ShouldIncludeTotal(message.Include)); + return await GetInternalAsync(sf, ti, httpContext, message.Filter, message.Sort, message.Mode, message.Page, message.Limit, message.Before, message.After, includeTotal: ShouldIncludeTotal(message.Include), timeExpression: message.Time); } public async Task>> Handle(GetEventsByStack message) @@ -219,7 +220,7 @@ public async Task>> Handle(GetEventsByStack message) var ti = TimeRangeParser.GetTimeInfo(message.Time, message.Offset, timeProvider, _allowedDateFields, DefaultDateField, organization.GetRetentionUtcCutoff(stack, appOptions.MaximumRetentionDays, timeProvider)); var sf = new AppFilter(stack, organization); - return await GetInternalAsync(sf, ti, httpContext, message.Filter, message.Sort, message.Mode, message.Page, message.Limit, message.Before, message.After, includeTotal: ShouldIncludeTotal(message.Include)); + return await GetInternalAsync(sf, ti, httpContext, message.Filter, message.Sort, message.Mode, message.Page, message.Limit, message.Before, message.After, includeTotal: ShouldIncludeTotal(message.Include), timeExpression: message.Time); } public async Task>> Handle(GetEventsByReferenceId message) @@ -262,7 +263,7 @@ public async Task>> Handle(GetEventsBySessionId messa var ti = TimeRangeParser.GetTimeInfo(message.Time, message.Offset, timeProvider, _allowedDateFields, DefaultDateField, organizations.GetRetentionUtcCutoff(appOptions.MaximumRetentionDays, timeProvider)); var sf = new AppFilter(organizations) { IsUserOrganizationsFilter = true }; - return await GetInternalAsync(sf, ti, httpContext, $"(reference:{message.SessionId} OR ref.session:{message.SessionId}) {message.Filter}", message.Sort, message.Mode, message.Page, message.Limit, message.Before, message.After, premiumFeatureUpgradeMessage: ApiFilterPolicy.PremiumSessionUpgradeMessage, includeTotal: ShouldIncludeTotal(message.Include)); + return await GetInternalAsync(sf, ti, httpContext, $"(reference:{message.SessionId} OR ref.session:{message.SessionId}) {message.Filter}", message.Sort, message.Mode, message.Page, message.Limit, message.Before, message.After, premiumFeatureUpgradeMessage: ApiFilterPolicy.PremiumSessionUpgradeMessage, includeTotal: ShouldIncludeTotal(message.Include), timeExpression: message.Time); } public async Task>> Handle(GetEventsBySessionIdAndProject message) @@ -281,7 +282,7 @@ public async Task>> Handle(GetEventsBySessionIdAndPro var ti = TimeRangeParser.GetTimeInfo(message.Time, message.Offset, timeProvider, _allowedDateFields, DefaultDateField, organization.GetRetentionUtcCutoff(project, appOptions.MaximumRetentionDays, timeProvider)); var sf = new AppFilter(project, organization); - return await GetInternalAsync(sf, ti, httpContext, $"(reference:{message.SessionId} OR ref.session:{message.SessionId}) {message.Filter}", message.Sort, message.Mode, message.Page, message.Limit, message.Before, message.After, premiumFeatureUpgradeMessage: ApiFilterPolicy.PremiumSessionUpgradeMessage, includeTotal: ShouldIncludeTotal(message.Include)); + return await GetInternalAsync(sf, ti, httpContext, $"(reference:{message.SessionId} OR ref.session:{message.SessionId}) {message.Filter}", message.Sort, message.Mode, message.Page, message.Limit, message.Before, message.After, premiumFeatureUpgradeMessage: ApiFilterPolicy.PremiumSessionUpgradeMessage, includeTotal: ShouldIncludeTotal(message.Include), timeExpression: message.Time); } public async Task>> Handle(GetSessions message) @@ -293,7 +294,7 @@ public async Task>> Handle(GetSessions message) var ti = TimeRangeParser.GetTimeInfo(message.Time, message.Offset, timeProvider, _allowedDateFields, DefaultDateField, organizations.GetRetentionUtcCutoff(appOptions.MaximumRetentionDays, timeProvider)); var sf = new AppFilter(organizations) { IsUserOrganizationsFilter = true }; - return await GetInternalAsync(sf, ti, httpContext, $"type:{Event.KnownTypes.Session} {message.Filter}", message.Sort, message.Mode, message.Page, message.Limit, message.Before, message.After, premiumFeatureUpgradeMessage: ApiFilterPolicy.PremiumSessionUpgradeMessage, includeTotal: ShouldIncludeTotal(message.Include)); + return await GetInternalAsync(sf, ti, httpContext, $"type:{Event.KnownTypes.Session} {message.Filter}", message.Sort, message.Mode, message.Page, message.Limit, message.Before, message.After, premiumFeatureUpgradeMessage: ApiFilterPolicy.PremiumSessionUpgradeMessage, includeTotal: ShouldIncludeTotal(message.Include), timeExpression: message.Time); } public async Task>> Handle(GetSessionsByOrganization message) @@ -308,7 +309,7 @@ public async Task>> Handle(GetSessionsByOrganization var ti = TimeRangeParser.GetTimeInfo(message.Time, message.Offset, timeProvider, _allowedDateFields, DefaultDateField, organization.GetRetentionUtcCutoff(appOptions.MaximumRetentionDays, timeProvider)); var sf = new AppFilter(organization); - return await GetInternalAsync(sf, ti, httpContext, $"type:{Event.KnownTypes.Session} {message.Filter}", message.Sort, message.Mode, message.Page, message.Limit, message.Before, message.After, premiumFeatureUpgradeMessage: ApiFilterPolicy.PremiumSessionUpgradeMessage, includeTotal: ShouldIncludeTotal(message.Include)); + return await GetInternalAsync(sf, ti, httpContext, $"type:{Event.KnownTypes.Session} {message.Filter}", message.Sort, message.Mode, message.Page, message.Limit, message.Before, message.After, premiumFeatureUpgradeMessage: ApiFilterPolicy.PremiumSessionUpgradeMessage, includeTotal: ShouldIncludeTotal(message.Include), timeExpression: message.Time); } public async Task>> Handle(GetSessionsByProject message) @@ -327,7 +328,7 @@ public async Task>> Handle(GetSessionsByProject messa var ti = TimeRangeParser.GetTimeInfo(message.Time, message.Offset, timeProvider, _allowedDateFields, DefaultDateField, organization.GetRetentionUtcCutoff(project, appOptions.MaximumRetentionDays, timeProvider)); var sf = new AppFilter(project, organization); - return await GetInternalAsync(sf, ti, httpContext, $"type:{Event.KnownTypes.Session} {message.Filter}", message.Sort, message.Mode, message.Page, message.Limit, message.Before, message.After, premiumFeatureUpgradeMessage: ApiFilterPolicy.PremiumSessionUpgradeMessage, includeTotal: ShouldIncludeTotal(message.Include)); + return await GetInternalAsync(sf, ti, httpContext, $"type:{Event.KnownTypes.Session} {message.Filter}", message.Sort, message.Mode, message.Page, message.Limit, message.Before, message.After, premiumFeatureUpgradeMessage: ApiFilterPolicy.PremiumSessionUpgradeMessage, includeTotal: ShouldIncludeTotal(message.Include), timeExpression: message.Time); } public async Task Handle(SetEventUserDescription message) @@ -728,7 +729,7 @@ private async Task> CountInternalAsync(AppFilter sf, TimeInf return result; } - private async Task>> GetInternalAsync(AppFilter sf, TimeInfo ti, HttpContext httpContext, string? filter = null, string? sort = null, string? mode = null, int? page = null, int limit = 10, string? before = null, string? after = null, string? premiumFeatureUpgradeMessage = null, bool includeTotal = false) + private async Task>> GetInternalAsync(AppFilter sf, TimeInfo ti, HttpContext httpContext, string? filter = null, string? sort = null, string? mode = null, int? page = null, int limit = 10, string? before = null, string? after = null, string? premiumFeatureUpgradeMessage = null, bool includeTotal = false, string? timeExpression = null) { var currentUser = httpContext.Request.GetUser(); using var _ = _logger.BeginScope(new ExceptionlessState() @@ -755,6 +756,12 @@ private async Task>> GetInternalAsync(AppFilter sf, T if (skip > Pagination.MaximumSkip) return new PagedResult(Array.Empty(), false); + if (IsStackMode(mode) && before is not null && after is not null) + return Result.BadRequest("The before and after parameters cannot be used together."); + + if (IsStackMode(mode) && page.HasValue && (before is not null || after is not null)) + return Result.BadRequest("The page parameter cannot be combined with before or after."); + var pr = await GetQueryValidator(mode).ValidateQueryAsync(filter); if (!pr.IsValid) return Result.BadRequest(pr.Message ?? "Invalid filter."); @@ -797,6 +804,42 @@ private async Task>> GetInternalAsync(AppFilter sf, T if (!String.IsNullOrEmpty(sort)) return Result.BadRequest("Sort is not supported in stack mode."); + if (!page.HasValue) + { + var lookupResult = await stackRollupSearchService.SearchAsync(new StackRollupSearchRequest( + appliedAppFilter, + ti.Range.UtcStart, + ti.Range.UtcEnd, + ti.Offset, + timeExpression, + filter, + mode, + limit, + before, + after, + includeTotal), httpContext.RequestAborted); + + if (lookupResult is not null) + { + string[] lookupStackIds = lookupResult.Rows.Select(row => row.StackId).ToArray(); + var lookupStacks = (await stackRepository.GetByIdsAsync(lookupStackIds)) + .Select(stack => stack.ApplyOffset(ti.Offset)) + .ToList(); + var lookupSummaries = await GetStackSummariesAsync(lookupStacks, lookupResult.Rows, sf, ti); + + return new PagedResult( + lookupSummaries.Cast().ToList(), + lookupResult.HasMore, + Page: null, + lookupResult.Total, + lookupResult.Before, + lookupResult.After); + } + + if (before is not null || after is not null) + return Result.BadRequest("Stack cursor paging is not available for the current Elasticsearch index."); + } + var systemFilter = new RepositoryQuery() .AppFilter(appliedAppFilter) .EnforceEventStackFilter() @@ -843,6 +886,10 @@ private async Task>> GetInternalAsync(AppFilter sf, T return new PagedResult(events.Documents.Cast().ToList(), events.HasMore && !Pagination.NextPageExceedsSkipLimit(page, limit), page, includeTotal ? events.Total : null, events.Hits.FirstOrDefault()?.GetSortToken(serializer), events.Hits.LastOrDefault()?.GetSortToken(serializer)); } } + catch (InvalidStackRollupCursorException ex) + { + return Result.BadRequest(ex.Message); + } catch (ApplicationException ex) { string message = "An error has occurred: Please check your search filter."; @@ -948,6 +995,44 @@ private async Task> GetStackSummariesAsync(List> GetStackSummariesAsync(List stacks, IReadOnlyCollection rows, AppFilter sf, TimeInfo ti) + { + if (stacks.Count == 0) + return []; + + var stacksById = stacks.ToDictionary(stack => stack.Id, StringComparer.Ordinal); + var projects = await projectRepository.GetByIdsAsync(stacks.Select(stack => stack.ProjectId).Distinct().ToArray(), options => options.Cache()); + var projectNames = projects.ToDictionary(project => project.Id, project => project.Name); + var totalUsers = await GetUserCountByProjectIdsAsync(stacks, sf, ti.Range.UtcStart, ti.Range.UtcEnd); + var summaries = new List(rows.Count); + + foreach (var row in rows) + { + if (!stacksById.TryGetValue(row.StackId, out var stack)) + continue; + + var data = formattingPluginManager.GetStackSummaryData(stack); + summaries.Add(new StackSummaryModel + { + Id = data.Id, + TemplateKey = data.TemplateKey, + Data = data.Data, + ProjectId = stack.ProjectId, + ProjectName = projectNames.GetValueOrDefault(stack.ProjectId), + Tags = stack.Tags?.OfType().Order(StringComparer.OrdinalIgnoreCase).ToArray() ?? [], + Title = stack.Title, + Status = stack.Status, + FirstOccurrence = row.FirstOccurrence, + LastOccurrence = row.LastOccurrence, + Total = row.Total, + Users = row.Users, + TotalUsers = totalUsers.GetOrDefault(stack.ProjectId) + }); + } + + return summaries; + } + private async Task> GetUserCountByProjectIdsAsync(ICollection stacks, AppFilter sf, DateTime utcStart, DateTime utcEnd) { using var scopedCacheClient = new ScopedCacheClient(cacheClient, $"Project:user-count:{utcStart.Floor(TimeSpan.FromMinutes(15)).Ticks}-{utcEnd.Floor(TimeSpan.FromMinutes(15)).Ticks}"); diff --git a/src/Exceptionless.Web/ClientApp/src/routes/(app)/stack/+page.svelte b/src/Exceptionless.Web/ClientApp/src/routes/(app)/stack/+page.svelte index 00585957d1..b416799bdd 100644 --- a/src/Exceptionless.Web/ClientApp/src/routes/(app)/stack/+page.svelte +++ b/src/Exceptionless.Web/ClientApp/src/routes/(app)/stack/+page.svelte @@ -106,6 +106,8 @@ const PAGE_SIZE_PREFERENCE_KEY = 'event-stack-list-page-size'; const pageSizePreference = createPageSizePreference(PAGE_SIZE_PREFERENCE_KEY); const DEFAULT_PARAMS = { + after: undefined as string | undefined, + before: undefined as string | undefined, bot: undefined as string | undefined, filter: undefined as string | undefined, first: undefined as string | undefined, @@ -220,6 +222,8 @@ defaults: DEFAULT_PARAMS, history: 'push', schema: { + after: 'string', + before: 'string', bot: 'string', filter: 'string', first: 'string', @@ -487,7 +491,7 @@ queryFilterParams.version !== queryParams.version; const effectiveQueryWillChange = (filter || null) !== getEffectiveFilter() || time !== getQueryTime(); const shouldClearPaginationForFilter = shouldClearPagination && effectiveQueryWillChange; - const paginationWillChange = shouldClearPaginationForFilter && queryParams.page != null; + const paginationWillChange = shouldClearPaginationForFilter && (queryParams.after != null || queryParams.before != null || queryParams.page != null); updateFilterCache(filterCacheKey(filter), updatedFilters); @@ -498,6 +502,8 @@ } queryParams.update({ + after: shouldClearPaginationForFilter ? null : queryParams.after, + before: shouldClearPaginationForFilter ? null : queryParams.before, bot: queryFilterParams.bot, filter: newFilterParam, first: queryFilterParams.first, @@ -614,6 +620,18 @@ }); const eventsQueryParameters: GetEventsParams = $state({ + get after() { + return queryParams.after ?? undefined; + }, + set after(value) { + queryParams.after = value ?? null; + }, + get before() { + return queryParams.before ?? undefined; + }, + set before(value) { + queryParams.before = value ?? null; + }, get filter() { return getEffectiveFilter()!; }, @@ -650,10 +668,19 @@ const eventsQuery = getOrganizationEventsQuery({ enabled: () => !isSavedViewRoutePending, get params() { - return { + const params = { ...eventsQueryParameters, - include: 'total' as const + include: !eventsQueryParameters.after && !eventsQueryParameters.before ? ('total' as const) : undefined }; + + if (!eventsQueryParameters.after && !eventsQueryParameters.before) { + return params; + } + + const { page: ignoredPage, ...cursorParams } = params; + void ignoredPage; + + return cursorParams; }, route: { get organizationId() { @@ -673,7 +700,7 @@ }, defaultColumnVisibility: defaultStackColumnVisibility, enableColumnResizing: true, - paginationStrategy: 'offset', + paginationStrategy: 'cursor', get queryData() { return eventsQuery.data?.data ?? []; }, diff --git a/tests/Exceptionless.Tests/Api/Endpoints/EventEndpointTests.cs b/tests/Exceptionless.Tests/Api/Endpoints/EventEndpointTests.cs index f46babddc4..f20bcc987d 100644 --- a/tests/Exceptionless.Tests/Api/Endpoints/EventEndpointTests.cs +++ b/tests/Exceptionless.Tests/Api/Endpoints/EventEndpointTests.cs @@ -817,6 +817,95 @@ public async Task CanGetMostFrequentStackMode() Assert.Equal(2, results.Count); } + [Theory] + [InlineData("stack_recent")] + [InlineData("stack_frequent")] + [InlineData("stack_new")] + [InlineData("stack_users")] + public async Task CanPageStackModesWithCursors(string mode) + { + await CreateStacksAndEventsAsync(); + Log.SetLogLevel(LogLevel.Debug); + + var response = await SendRequestAsync(request => request + .AsGlobalAdminUser() + .AppendPath("events") + .QueryString("filter", $"project:{SampleDataService.FREE_PROJECT_ID} (status:open OR status:regressed)") + .QueryString("mode", mode) + .QueryString("limit", 1) + .QueryString("include", "total") + .StatusCodeShouldBeOk()); + + Assert.Equal("2", response.Headers.GetValues(Headers.ResultCount).Single()); + var firstPageLinks = ParseLinkHeaderValue(response.Headers.GetValues(HeaderNames.Link).ToArray()); + Assert.False(firstPageLinks.ContainsKey("previous")); + string? after = GetQueryStringValue(firstPageLinks["next"], "after"); + Assert.NotNull(after); + + var firstPage = await response.Content.ReadFromJsonAsync>(_jsonSerializerOptions, TestCancellationToken); + Assert.NotNull(firstPage); + string firstStackId = Assert.Single(firstPage).Id; + + response = await SendRequestAsync(request => request + .AsGlobalAdminUser() + .AppendPath("events") + .QueryString("filter", $"project:{SampleDataService.FREE_PROJECT_ID} (status:open OR status:regressed)") + .QueryString("mode", mode) + .QueryString("limit", 1) + .QueryString("include", "total") + .QueryString("after", after) + .StatusCodeShouldBeOk()); + + Assert.Equal("2", response.Headers.GetValues(Headers.ResultCount).Single()); + var secondPageLinks = ParseLinkHeaderValue(response.Headers.GetValues(HeaderNames.Link).ToArray()); + Assert.True(secondPageLinks.TryGetValue("previous", out string? previousLink)); + Assert.False(secondPageLinks.ContainsKey("next")); + string? before = GetQueryStringValue(previousLink, "before"); + Assert.NotNull(before); + + var secondPage = await response.Content.ReadFromJsonAsync>(_jsonSerializerOptions, TestCancellationToken); + Assert.NotNull(secondPage); + string secondStackId = Assert.Single(secondPage).Id; + Assert.NotEqual(firstStackId, secondStackId); + + response = await SendRequestAsync(request => request + .AsGlobalAdminUser() + .AppendPath("events") + .QueryString("filter", $"project:{SampleDataService.FREE_PROJECT_ID} (status:open OR status:regressed)") + .QueryString("mode", mode) + .QueryString("limit", 1) + .QueryString("before", before) + .StatusCodeShouldBeOk()); + + var previousPage = await response.Content.ReadFromJsonAsync>(_jsonSerializerOptions, TestCancellationToken); + Assert.NotNull(previousPage); + Assert.Equal(firstStackId, Assert.Single(previousPage).Id); + } + + [Theory] + [InlineData("before", "malformed", null, null)] + [InlineData("after", "malformed", null, null)] + [InlineData("before", "malformed", "after", "malformed")] + [InlineData("after", "malformed", "page", "1")] + public async Task RejectsInvalidStackCursorRequests(string firstName, string firstValue, string? secondName, string? secondValue) + { + await CreateStacksAndEventsAsync(); + + await SendRequestAsync(request => + { + request + .AsGlobalAdminUser() + .AppendPath("events") + .QueryString("mode", "stack_frequent") + .QueryString(firstName, firstValue); + + if (secondName is not null) + request.QueryString(secondName, secondValue); + + request.StatusCodeShouldBeBadRequest(); + }); + } + [Fact] public async Task CanGetProjectLevelMostFrequentStackMode() { diff --git a/tests/Exceptionless.Tests/Search/StackIndexTests.cs b/tests/Exceptionless.Tests/Search/StackIndexTests.cs index abea7b5c4f..79ec50f613 100644 --- a/tests/Exceptionless.Tests/Search/StackIndexTests.cs +++ b/tests/Exceptionless.Tests/Search/StackIndexTests.cs @@ -1,5 +1,7 @@ +using Elastic.Clients.Elasticsearch; using Exceptionless.Core.Models; using Exceptionless.Core.Repositories; +using Exceptionless.Core.Repositories.Configuration; using Exceptionless.Tests.Utility; using Foundatio.Repositories; using Foundatio.Repositories.Models; @@ -11,11 +13,13 @@ public sealed class StackIndexTests : IntegrationTestsBase { private readonly StackData _stackData; private readonly IStackRepository _repository; + private readonly ExceptionlessElasticConfiguration _configuration; public StackIndexTests(ITestOutputHelper output, AppWebHostFactory factory) : base(output, factory) { _stackData = GetService(); _repository = GetService(); + _configuration = GetService(); } protected override async Task ResetDataAsync() @@ -24,6 +28,19 @@ protected override async Task ResetDataAsync() await _stackData.CreateSearchDataAsync(); } + [Fact] + public async Task UsesSingleShardLookupModeIndexAsync() + { + var response = await _configuration.Client.Indices.GetSettingsAsync((Indices)_configuration.Stacks.Name, TestCancellationToken); + + Assert.True(response.IsValidResponse); + var settings = Assert.Single(response.Settings).Value.Settings?.Index; + Assert.NotNull(settings); + Assert.Equal("lookup", settings.Mode); + Assert.NotNull(settings.NumberOfShards); + Assert.Equal(1, settings.NumberOfShards.Match(value => value, value => Int32.TryParse(value, out int parsed) ? parsed : 0)); + } + [Theory] [InlineData("\"GET /Print\"", 3)] // Title [InlineData("\"my custom description\"", 1)] // Description diff --git a/tests/http/events.http b/tests/http/events.http index dda0f774f9..2405c2aa46 100644 --- a/tests/http/events.http +++ b/tests/http/events.http @@ -29,6 +29,12 @@ Authorization: Bearer {{token}} GET {{apiUrl}}/events?mode=stack_frequent&limit=10 Authorization: Bearer {{token}} +### + +### Stack summaries with cursor paging (copy the after value from the Link response header) +GET {{apiUrl}}/events?mode=stack_frequent&limit=10&after={{after}} +Authorization: Bearer {{token}} + ### @eventId = {{allEvents.response.body.$[0].id}} @stackId = {{allEvents.response.body.$[0].stack_id}} From b36f897b2cb9503d6f0f818bae4375ea9825a059 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Sat, 22 Aug 2026 20:07:57 -0500 Subject: [PATCH 3/9] Make lookup join stack paging mandatory --- .../design.md | 175 ------------ .../proposal.md | 80 ------ .../specs/search-and-stacks/spec.md | 262 ------------------ .../tasks.md | 94 ------- openspec/config.yaml | 24 -- .../Configuration/ElasticsearchOptions.cs | 2 - .../Services/StackRollupSearchService.cs | 18 +- .../Api/Handlers/EventHandler.cs | 149 +++------- .../src/routes/(app)/stack/+page.svelte | 10 +- .../Api/Endpoints/EventEndpointTests.cs | 4 +- 10 files changed, 43 insertions(+), 775 deletions(-) delete mode 100644 openspec/changes/experiment-esql-lookup-join-stack-pagination/design.md delete mode 100644 openspec/changes/experiment-esql-lookup-join-stack-pagination/proposal.md delete mode 100644 openspec/changes/experiment-esql-lookup-join-stack-pagination/specs/search-and-stacks/spec.md delete mode 100644 openspec/changes/experiment-esql-lookup-join-stack-pagination/tasks.md delete mode 100644 openspec/config.yaml diff --git a/openspec/changes/experiment-esql-lookup-join-stack-pagination/design.md b/openspec/changes/experiment-esql-lookup-join-stack-pagination/design.md deleted file mode 100644 index ac51362e2c..0000000000 --- a/openspec/changes/experiment-esql-lookup-join-stack-pagination/design.md +++ /dev/null @@ -1,175 +0,0 @@ -# Design: ES|QL Lookup-Join Stack Pagination Experiment - -## Decision - -Introduce a narrow `IStackRollupSearchService` for the four event stack-rollup modes. The service may use `ElasticsearchClient` directly because Foundatio.Repositories does not expose ES|QL, while all ordinary stack hydration, project lookup, writes, and direct stack searches remain repository-backed. - -The query joins before aggregating: - -```text -FROM -| KEEP stack_id, count, date, -| RENAME AS event_user -| LOOKUP JOIN - ON stack_id == id AND is_deleted == false - AND QSTR(, {"default_operator": "AND"}) -| WHERE id IS NOT NULL -| STATS users = COUNT_DISTINCT(event_user), - total = SUM(COALESCE(count, 1)), - first_occurrence = MIN(date), - last_occurrence = MAX(date) - BY stack_id -| INLINE STATS total_stacks = COUNT(*) -| WHERE -| SORT , stack_id ASC -| LIMIT -``` - -The localhost Elasticsearch 9.5 capability spike rejected `QSTR` after `STATS` with `verification_exception: [QSTR] function cannot be used after STATS`. Joining before aggregation is therefore required to preserve the existing Lucene-style stack filter contract without translating that language into a second expression language. This removes the 20,000-stack-id materialization and offset-sized terms buckets from the primary rollup query, but the lookup processes matching event rows rather than one aggregated row per stack. The existing project-level `TotalUsers` helper remains repository-backed and can still invoke the legacy active-stack filter on a cache miss; replacing that auxiliary query is a separate follow-up. Those costs are explicit performance gates for the experiment. - -The spike also proved two less obvious constraints. Real event and stack mappings both expose fields such as `id`, so the event input must be projected with `KEEP` and renamed before the join or ES|QL rejects later references as ambiguous. In addition, `QSTR` defaults adjacent clauses to `OR`, while Exceptionless's repository parser treats them as implicit `AND`; every joined stack predicate therefore supplies `default_operator=AND`. - -Authorization, project, event-filter, and date predicates are supplied through the ES|QL REST request's Query DSL `filter`, built by the existing Foundatio event query builder. The prototype currently names the canonical event alias and relies on this pushdown filter. Resolving only the concrete daily indices remains a performance follow-up because a requested day can legitimately have no concrete index and ES|QL does not inherit the repository client's ignore-unavailable behavior. - -The exact physical event and stack field names must come from the index configuration rather than duplicated string literals. User values must be sent as ES|QL parameters; only allow-listed command fragments and validated index names may be composed into query text. - -## Existing paths and scope - -### Event stack-rollup path - -`EventHandler.GetInternalAsync` currently: - -1. validates with `EventStackQueryValidator`, -2. builds a Foundatio daily-event query, -3. lets `EventStackFilterQueryBuilder` search the stack index and inject up to 20,000 ids, -4. requests a terms aggregation sized to `skip + limit + 1`, -5. skips buckets in memory, -6. hydrates stack documents, and -7. joins aggregation metrics to `StackSummaryModel`. - -The experiment replaces steps 3 through 5 for cursor requests. Stack hydration and formatting remain shared with the current code. - -### Direct stack endpoint path - -`StackHandler.GetInternalAsync` searches only the versioned stack index and supports arbitrary stack sort expressions. It should continue using `IStackRepository`. A follow-up can add Foundatio search-after/before tokens with an appended `id` tie-breaker. Sending these stack-only reads through all daily event indices would add cost and change zero-event/time-range semantics without gaining anything from `LOOKUP JOIN`. - -### Legacy page path - -If `page` is present, retain the current aggregation/page implementation for compatibility and A/B comparison. Cursor and page parameters are mutually exclusive. The experiment must collect telemetry that distinguishes lookup-join, legacy-page, and fallback execution. - -## Index topology and migration - -Events remain in the existing `DailyIndex` indices. The ES|QL source must reuse the same start/end index resolution as `.Index(utcStart, utcEnd)` so it does not scan outside retention or include unrelated indices. ES|QL requires all selected shards to be available, so missing/closed daily indices need an explicit no-data or fallback policy rather than a broad wildcard. - -The current prototype deliberately uses the canonical event alias plus an exact Query DSL date filter until that missing-index policy is implemented. The benchmark must include the shard-fan-out cost of this choice; it is not production-ready evidence for the final daily-index resolution requirement. - -The canonical `StackIndex` is a `VersionedIndex`. Create its next version with: - -- `index.mode = lookup`, -- exactly one primary shard, regardless of the general `ElasticsearchOptions.NumberOfShards`, -- the configured replica count, and -- the existing mapping and alias. - -The versioned alias must resolve to exactly one concrete index before the feature is considered ready. Reindex all active and soft-deleted stack documents, validate document counts and representative mappings, then atomically switch the alias using the existing versioned-index migration mechanism. - -Lookup mode's one-primary-shard constraint is the central capacity tradeoff. Stacks are much smaller than events, but the experiment must measure shard size, document count, indexing/update throughput, patch latency, and query heap. Deployments that cannot fit their stack corpus or write rate on one primary shard must keep the feature disabled; the initial experiment must not introduce a dual-write lookup projection without a separate design. - -`index.mode` is a final creation-only setting. The stack index config applies lookup mode only while creating v2 and overrides Foundatio's existing-index settings update to send only mutable replica/priority settings; otherwise every later startup receives a harmless-looking but rejected `PUT _settings` request. - -Normal `IStackRepository` reads, writes, scripts, and deletes continue against the lookup-mode index. Replicas remain available for resilience. Log a clear startup readiness state with alias target, lookup mode, and primary shard count, without logging credentials or query values. - -## Filter preservation - -Continue using `EventStackQueryValidator` and `EventStackFilter` so the public Lucene-style filter contract and premium-feature classification remain unchanged. - -- Resolve the event-only filter through the current visitor and apply it immediately after `FROM`, using `QSTR` or the equivalent generated Query DSL filter. -- Resolve the stack filter through the current stack visitor, including physical field rewrites for `fixed`, `regressed`, `hidden`, and `stack_id`. -- Apply the stack predicate in the `LOOKUP JOIN` condition together with `stack_id == id`, before `STATS`. Elasticsearch 9.5 permits `QSTR` in this position but rejects it after aggregation; every supported filter fixture must still be proven. -- Require a non-null joined stack id and exclude soft-deleted stacks after the join, giving the left join effective inner-join behavior. -- Preserve the current handling of `@!`; do not reinterpret it as part of this refactor. -- Preserve organization/project authorization as an event-source predicate before `STATS`. Never depend on a joined stack field as the only authorization boundary. - -If a validated filter cannot be represented by the 9.5 query, route that request to the legacy implementation and increment a reason-labelled fallback metric. Do not partially apply a filter. - -## Mode semantics - -Use one query shape and an allow-listed mode definition: - -| Mode | Primary sort | Selection detail | -| --- | --- | --- | -| `stack_recent` | `last_occurrence DESC` | latest matching event in the resolved range | -| `stack_frequent` | `total DESC` | sum event `count`, preserving the existing default occurrence behavior | -| `stack_new` | `first_occurrence DESC` | preserve the existing stack first-occurrence range predicate added by `AddFirstOccurrenceFilter` | -| `stack_users` | `users DESC` | approximate distinct user count, matching current cardinality semantics | - -Every mode appends `stack_id ASC` as a tie-breaker. Sort is applied after `LOOKUP JOIN` because Elasticsearch does not preserve a sort performed before a lookup join. - -Hydrate canonical stack documents by returned ids and explicitly restore ES|QL row order before formatting summaries. Missing stack documents are excluded and measured; they must not reorder the remaining page. - -## Cursor contract - -Use a versioned, base64url-encoded JSON cursor containing: - -- cursor schema version, -- mode and canonical sort direction, -- typed primary sort value, -- stack id tie-breaker, -- resolved absolute UTC start/end, -- a hash of organization/project scope, normalized filter, mode, and sort contract, and -- direction (`before` or `after`) if required by the decoder. - -Do not include user identity, tokens, credentials, raw authorization data, or unbounded filter text. Reject unknown versions, malformed values, non-finite numeric values, and fingerprint mismatches with `400 Bad Request`. - -For canonical descending order `(metric DESC, stack_id ASC)`: - -- `after`: `metric < anchor OR (metric == anchor AND stack_id > anchor_id)` -- `before`: `metric > anchor OR (metric == anchor AND stack_id < anchor_id)` - -A `before` query reverses both sort directions, takes `limit + 1`, and reverses the selected items before returning them. Equivalent predicates must be generated for ascending sorts if reused later. Aggregated mode metrics must be non-null; direct stack sorts need an explicit null policy before they can reuse this cursor. - -The cursor freezes a relative time expression to its initially resolved absolute range. It does not freeze mutable event aggregates. New/backfilled events or stack status changes can move a stack across the anchor between requests; this is documented as live keyset pagination rather than snapshot pagination. - -## Totals and limits - -The existing `include=total` behavior must remain available. Because a cursor query limits rows after grouping, total count must be computed before the cursor predicate. Elasticsearch 9.5 supports `INLINE STATS total_stacks = COUNT(*)` between grouping and the cursor predicate, which the localhost spike proved returns the pre-cursor total on every selected row. Compare its result with the current cardinality behavior in tests and omit it when total is not requested. Do not increase `esql.query.result_truncation_max_size`. - -The ES|QL 10,000-row output limit does not prevent this design because the final output is `limit + 1`; `STATS` still processes the full selected source. The service must retain the API's existing limit clamp. - -## Capability, fallback, and errors - -Enable lookup-join execution only when: - -- the operational feature flag is enabled, -- Elasticsearch reports a compatible 9.5 capability, -- the stack alias resolves to one concrete index, -- that index has `index.mode=lookup` and one primary shard, and -- the query is cursor-based and representable without semantic loss. - -Fallback to the legacy query for feature-off, readiness mismatch, explicitly unsupported filters, and classified transient/circuit-breaker failures. Emit structured logs and counters with a bounded reason label. Invalid filters, invalid cursors, authorization failures, and plan-limit failures must retain their public error and must not be hidden by fallback. - -Use a timeout and cancellation token. Record duration, selected daily-index count, returned rows, mode, direction, fallback reason, and response allocation/size where available. Never log raw user filters or cursor contents at normal log levels. - -## Performance experiment - -Build a repeatable integration benchmark fixture with: - -- multiple daily event indices, -- at least tens of thousands of stacks for CI-scale measurement and an optional larger local profile, -- skewed occurrence/user distributions, -- ties on every mode metric, -- active, fixed, regressed, ignored, and soft-deleted stacks, and -- both event-only and stack-only filters. - -Compare legacy and ES|QL paths at the first page and equivalent depths near pages 100, 500, and the current maximum skip. Capture median and p95 wall time across repeated warm runs, request/response bytes, managed allocations, Elasticsearch took time if exposed, and failures/circuit-breakers. A noisy single run is not sufficient evidence. - -Promotion gate: result equivalence must be exact for ids/order/summary fields, shallow-page p95 must not materially regress, and deep-page work must remain bounded by `limit` instead of requested offset. If lookup-join heap/circuit-breaker behavior is worse for representative data, keep the experiment behind the flag or abandon it. - -## Official Elasticsearch references - -- [LOOKUP JOIN command](https://www.elastic.co/docs/reference/query-languages/esql/commands/lookup-join) -- [LOOKUP JOIN prerequisites and limitations](https://www.elastic.co/docs/reference/query-languages/esql/esql-lookup-join) -- [Index mode settings](https://www.elastic.co/docs/reference/elasticsearch/index-settings/index-modules) -- [ES|QL SORT and tie-breakers](https://www.elastic.co/docs/reference/query-languages/esql/commands/sort) -- [ES|QL LIMIT behavior](https://www.elastic.co/docs/reference/query-languages/esql/commands/limit) -- [ES|QL QSTR](https://www.elastic.co/docs/reference/query-languages/esql/functions-operators/search-functions/qstr) -- [ES|QL REST API](https://www.elastic.co/docs/reference/query-languages/esql/esql-rest) diff --git a/openspec/changes/experiment-esql-lookup-join-stack-pagination/proposal.md b/openspec/changes/experiment-esql-lookup-join-stack-pagination/proposal.md deleted file mode 100644 index 5f8745ef7e..0000000000 --- a/openspec/changes/experiment-esql-lookup-join-stack-pagination/proposal.md +++ /dev/null @@ -1,80 +0,0 @@ -# Proposal: Experiment with ES|QL Lookup-Join Stack Pagination - -## Summary - -Replace the growing terms-aggregation/skip path used by event stack-rollup queries with an experimental Elasticsearch 9.5 ES|QL pipeline that: - -1. reads the existing daily event indices for the requested time range, -2. uses `LOOKUP JOIN` to attach and filter the canonical stack document from the versioned stack index, -3. aggregates the filtered event rows by `stack_id`, -4. applies a deterministic mode-specific sort with `stack_id` as the final tie-breaker, and -5. pages with opaque `before` and `after` keyset cursors. - -This specifically targets the `stack_recent`, `stack_frequent`, `stack_new`, and `stack_users` modes on event list endpoints. Those modes power the organization-level stacks page. It also establishes cursor behavior that direct stack endpoints can reuse, while leaving direct stack-only queries on `IStackRepository` unless an event join is actually needed. - -## User-visible behavior - -- The four existing stack-rollup modes return the same stack summary shape, filters, metrics, mode ordering, plan enforcement, and authorization boundaries. -- Requests without `page` use cursor paging. Responses include opaque `before` and `after` links when applicable. -- `after` moves forward in the canonical result order; `before` moves backward and returns items in the same canonical order. -- Existing `page` + `limit` requests remain supported by the legacy implementation during the experiment. -- A request cannot combine `page` with `before` or `after`, or combine `before` with `after`. -- Explicit `sort` remains invalid for the four stack-rollup modes. Their fixed sorts remain: - - `stack_recent`: last occurrence descending - - `stack_frequent`: summed occurrence count descending - - `stack_new`: first occurrence descending, restricted by the existing stack-first-occurrence time behavior - - `stack_users`: distinct users descending -- Cursor tokens are opaque and query-bound. Malformed cursors or cursors reused with a different scope, filter, time range, mode, or sort contract return `400 Bad Request`. -- Cursor paging provides a total deterministic order, not a point-in-time snapshot. Concurrent events can change aggregate values between requests. - -## Classification - -- **Type:** Experimental refactor, Elasticsearch index/query change, additive API pagination behavior -- **Affected areas:** Backend/API, Elasticsearch index configuration and reindexing, event/stack search, pagination headers, configuration, telemetry, integration tests, performance tests -- **OpenSpec justification:** This changes persisted Elasticsearch index settings, replaces a compatibility-sensitive aggregation/filtering path, introduces a new raw ES|QL execution boundary, and changes how public pagination links are produced. - -## Current implementation context - -`EventHandler.GetInternalAsync` implements the four stack modes by requesting a `terms` aggregation on `stack_id`. The bucket size grows with `skip + limit + 1`, the handler skips buckets in memory, fetches stack documents by id, and joins them to aggregation buckets. Stack-only predicates are handled by `EventStackFilterQueryBuilder`, which first searches the stack index, materializes up to 20,000 stack ids, and injects those ids into the event query. This makes deep paging increasingly expensive and creates an explicit document-limit failure mode. - -`StackHandler.GetInternalAsync` is a separate stack-only path. It pages the versioned stack index with page/limit and optionally runs daily-event aggregations for summary values. The initial lookup-join experiment must not force stack-only reads through daily event indices when no event-derived ordering or metrics are needed. - -## Compatibility boundaries - -- Preserve all existing event and stack routes. -- Preserve the `StackSummaryModel` response body and pagination headers. -- Preserve Lucene-style Exceptionless filter syntax and the existing split between event fields and stack fields, including special fields such as `fixed`, `regressed`, and `hidden`. -- Preserve free/premium filter validation and suspended-organization behavior before any ES|QL query executes. -- Preserve stack-mode rejection of explicit `sort`. -- Preserve page/limit behavior as a fallback during the experiment. -- Do not expose ES|QL syntax as a public API contract. -- Do not make a cursor token a durable SDK contract beyond being opaque and reusable with the same query. - -## Non-goals - -- Replacing ordinary event document search-after queries. -- Replacing direct stack-only repository searches with ES|QL when no event aggregation is required. -- Changing the documented Exceptionless filter language. -- Fixing or broadening mixed event/stack boolean filter semantics beyond current behavior. -- Providing point-in-time snapshot pagination for mutable aggregates. -- Raising the ES|QL cluster result limit. -- Supporting cross-cluster event indices in the first experiment. -- Removing the legacy aggregation implementation before correctness and performance gates pass. - -## Rollback and mitigation - -- Guard the ES|QL path with an operational feature flag/kill switch, disabled by default outside explicitly selected environments during the experiment. -- Route page-number requests and unsupported/capability-mismatch cases through the existing implementation. -- Keep the stack alias on the lookup-mode versioned index after an application rollback; normal repository reads and writes remain supported on that index mode. -- If lookup-index migration cannot complete, do not switch the versioned alias and keep the feature disabled. -- Do not silently retry an invalid user filter through the legacy path. Fallback is for capability/readiness/transient execution failures that are separately logged and measured. - -## Success criteria - -The experiment is eligible for follow-up production work only if it: - -- matches the legacy result ids, ordering, summary metrics, filters, and totals for representative fixtures; -- traverses forward and backward without duplicates for a stable fixture, including tied aggregate values; -- removes the `skip + limit` terms-bucket growth and stack-filter id materialization from the primary rollup query; -- demonstrates no material latency or allocation regression at shallow pages and a material improvement at deep pages on representative daily-index data; and -- can be disabled without an index rollback or public API break. diff --git a/openspec/changes/experiment-esql-lookup-join-stack-pagination/specs/search-and-stacks/spec.md b/openspec/changes/experiment-esql-lookup-join-stack-pagination/specs/search-and-stacks/spec.md deleted file mode 100644 index a041fa4020..0000000000 --- a/openspec/changes/experiment-esql-lookup-join-stack-pagination/specs/search-and-stacks/spec.md +++ /dev/null @@ -1,262 +0,0 @@ -# Spec: Search and Stacks - -## MODIFIED Requirements - -### Requirement: Stack rollup modes preserve existing summaries and fixed ordering - -Event list endpoints executed in `stack_recent`, `stack_frequent`, `stack_new`, or `stack_users` mode MUST preserve the existing `StackSummaryModel` response contract, mode selection semantics, filters, authorization, plan enforcement, and fixed mode ordering. - -#### Scenario: Most frequent stacks - -Given matching events span one or more daily indices -And their stacks exist in the active versioned stack index -When an authorized caller requests `mode=stack_frequent` -Then results are ordered by summed occurrence count descending -And ties are ordered by stack id ascending -And every summary contains the same metrics and canonical stack fields as the legacy implementation. - -#### Scenario: Most recent stacks - -Given matching events span one or more daily indices -When an authorized caller requests `mode=stack_recent` -Then results are ordered by the latest matching event date descending -And ties are ordered by stack id ascending. - -#### Scenario: New stacks - -Given stacks have first occurrences both inside and outside the resolved time range -When an authorized caller requests `mode=stack_new` -Then only stacks selected by the existing stack-first-occurrence time behavior are returned -And results are ordered by first occurrence descending with stack id ascending as the tie-breaker. - -#### Scenario: Stacks by users - -Given matching events contain repeated and distinct user identities -When an authorized caller requests `mode=stack_users` -Then results are ordered by the existing approximate distinct-user metric descending -And ties are ordered by stack id ascending. - -#### Scenario: Explicit sort remains invalid - -Given a request uses any stack rollup mode -When it supplies an explicit `sort` -Then the API returns the existing bad-request behavior rather than overriding the mode sort. - -### Requirement: Stack rollup filters preserve Exceptionless query behavior - -Stack rollup execution MUST preserve the current Lucene-style filter contract and MUST apply event and stack predicates without widening authorization or returning soft-deleted stacks. - -#### Scenario: Event-only filter - -Given a valid filter references only event fields -When a stack rollup query executes -Then only events matching that filter contribute to stack metrics. - -#### Scenario: Stack-only filter - -Given a valid filter references stack status, tags, title, references, fixed state, regressed state, hidden state, or stack identifiers -When a stack rollup query executes -Then only stacks matching the equivalent current stack-filter behavior are returned. - -#### Scenario: Mixed event and stack filter - -Given a filter contains event and stack fields in a form supported by the current filter visitors -When a stack rollup query executes -Then its result ids match the legacy implementation for the same fixture. - -#### Scenario: Unsupported lossless translation - -Given a validated filter cannot be represented by the lookup-join query without changing semantics -When the experimental path evaluates the request -Then the whole request uses the legacy implementation -And no partial filter is executed. - -#### Scenario: Deleted or missing stack - -Given an event references a soft-deleted or missing stack document -When a stack rollup query executes -Then that stack is not returned. - -#### Scenario: Authorization is applied before aggregation - -Given a caller can access only selected organizations or projects -When a stack rollup query executes -Then inaccessible event documents do not contribute to aggregation metrics -And a joined stack field is not the sole authorization check. - -## ADDED Requirements - -### Requirement: Stack rollup modes support opaque before and after cursors - -Cursor-based stack rollup responses MUST define a deterministic total order and MUST provide opaque cursors that are valid only for the same logical query. - -#### Scenario: Forward traversal - -Given more matching stacks than the requested limit -When the caller follows each `after` link -Then each stable-fixture stack is returned exactly once in canonical mode order -And the last page does not advertise a next page. - -#### Scenario: Backward traversal - -Given the caller is on a page after the first page -When the caller follows the `before` link -Then the preceding items are returned in canonical mode order -And they match the page previously observed for the stable fixture. - -#### Scenario: Equal primary sort values - -Given multiple stacks have the same primary mode metric -When the caller traverses pages whose boundary falls within those ties -Then stack id provides a stable tie-breaker -And no tied stack is skipped or duplicated for the stable fixture. - -#### Scenario: Relative time range is frozen - -Given the initial request uses a relative time expression -When the caller follows a cursor after wall-clock time advances -Then the cursor uses the absolute UTC range resolved for the initial request. - -#### Scenario: Concurrent aggregate changes - -Given matching events or stack status change between cursor requests -When the caller follows a cursor -Then the API maintains its deterministic keyset comparison -But it does not promise point-in-time snapshot traversal. - -#### Scenario: Malformed cursor - -Given `before` or `after` is malformed, has an unknown version, or contains an invalid typed sort value -When the request is validated -Then the API returns `400 Bad Request` -And does not execute either Elasticsearch path. - -#### Scenario: Cursor reused with another query - -Given a valid cursor was issued for one scope, filter, time range, mode, or sort contract -When it is reused with a different logical query -Then the API returns `400 Bad Request`. - -#### Scenario: Conflicting pagination parameters - -Given a request supplies both `before` and `after`, or supplies `page` with either cursor -When the request is validated -Then the API returns `400 Bad Request`. - -### Requirement: Lookup-join execution requires a ready single-shard lookup index - -The experimental lookup-join path MUST execute only when the active stack alias resolves to one concrete lookup-mode index with one primary shard. - -#### Scenario: Ready lookup index - -Given the feature is enabled -And the stack alias resolves to one concrete index with `index.mode=lookup` and one primary shard -When a representable cursor stack-rollup request executes -Then the ES|QL lookup-join path may be used. - -#### Scenario: Multi-shard or standard-mode stack index - -Given the active stack index is not lookup mode or does not have exactly one primary shard -When a stack-rollup request executes -Then the legacy implementation is used -And readiness telemetry identifies the bounded reason. - -#### Scenario: Alias resolves to multiple concrete indices - -Given the stack alias resolves to multiple concrete indices -When a stack-rollup request executes -Then the lookup-join path is not used. - -#### Scenario: Lookup feature disabled - -Given the operational feature flag is disabled -When a stack-rollup request executes -Then existing legacy behavior is preserved without requiring an index rollback. - -### Requirement: Page-number stack rollups remain compatible during the experiment - -Explicit page-number requests MUST continue to use the existing page/limit behavior while cursor results are evaluated. - -#### Scenario: Explicit page request - -Given a request supplies `page` and `limit` without a cursor -When a stack rollup query executes -Then it returns the same result and page-link contract as the legacy implementation. - -#### Scenario: Cursor request avoids offset-sized buckets - -Given a cursor request follows a deep result boundary -When the lookup-join query executes -Then its final row limit is based on `limit + 1` -And it does not size a terms aggregation using the historical page offset. - -### Requirement: Lookup-join failures are observable and safely classified - -Lookup-join execution MUST preserve public validation and authorization errors, support cancellation and timeouts, and emit bounded operational telemetry for fallback. - -#### Scenario: Invalid filter remains invalid - -Given a user submits an invalid search filter -When query validation fails -Then the API returns the existing invalid-filter response -And does not retry through another implementation. - -#### Scenario: Transient lookup execution failure - -Given the lookup path is enabled and a classified transient Elasticsearch failure occurs -When the request can safely use the legacy implementation -Then the request falls back once -And telemetry records a bounded failure category without raw filters or cursors. - -#### Scenario: Request cancellation - -Given the HTTP request is cancelled -When ES|QL is executing -Then cancellation is propagated -And the handler does not start an unbounded fallback query. - -### Requirement: Lookup-mode migration preserves canonical stack repository behavior - -Migrating the versioned stack index to lookup mode MUST preserve active and soft-deleted stack documents and ordinary stack repository operations. - -#### Scenario: Versioned reindex completes - -Given an existing versioned stack index contains active and soft-deleted documents -When the lookup-mode version is built and validated -Then all documents and required mappings are present before the alias switches atomically. - -#### Scenario: Repository operations after cutover - -Given the stack alias points to the lookup-mode index -When Exceptionless reads, saves, patches, or soft-deletes a stack through `IStackRepository` -Then behavior matches the existing repository contract. - -#### Scenario: Migration is not ready - -Given reindex validation fails or the lookup index is not ready -When startup index configuration runs -Then the alias is not switched -And the lookup-join feature remains disabled. - -### Requirement: Experimental performance is measured against the legacy path - -The experimental PR MUST include a repeatable comparison using representative multi-day data before the lookup path is proposed as the default. - -#### Scenario: Correctness comparison - -Given identical representative fixtures -When legacy and lookup-join queries run for every mode, filter family, and measured page depth -Then result ids, order, summaries, and requested totals match. - -#### Scenario: Shallow and deep measurements - -Given repeated warm benchmark runs -When first-page and deep-page cases are measured -Then the report includes median and p95 wall time, allocations or response bytes where available, and failure/circuit-breaker counts -And no conclusion is based on a single run. - -#### Scenario: Promotion gate fails - -Given correctness differs, shallow latency materially regresses, or representative lookup joins create unacceptable heap/circuit-breaker pressure -When the experiment is reviewed -Then the feature remains disabled or the experiment is abandoned without removing the legacy path. diff --git a/openspec/changes/experiment-esql-lookup-join-stack-pagination/tasks.md b/openspec/changes/experiment-esql-lookup-join-stack-pagination/tasks.md deleted file mode 100644 index a5943ce43f..0000000000 --- a/openspec/changes/experiment-esql-lookup-join-stack-pagination/tasks.md +++ /dev/null @@ -1,94 +0,0 @@ -# Tasks: ES|QL Lookup-Join Stack Pagination Experiment - -## Design and capability spike - -- [x] 1. Prove the Elasticsearch 9.5 query shape against local Aspire Elasticsearch - - Create a temporary daily-event fixture and a single-shard lookup-mode stack index. - - Prove and document that stack `QSTR` requires `LOOKUP JOIN` before `STATS` in Elasticsearch 9.5. - - Prove the stack alias can be used when it resolves to exactly one concrete lookup index. - - Prove stack `QSTR`/Lucene-pushable predicates in the join condition, soft-delete exclusion, aggregate metrics, exact pre-cursor totals, tied forward cursors, and reverse-before ordering. - - Verification: localhost-only REST fixture on isolated Elasticsearch 9.5 port `9215`; no production endpoint. - -- [ ] 2. Add lookup-join capability/readiness detection and an operational kill switch - - Detect feature flag, server capability, alias target count, lookup mode, and primary shard count. - - Add bounded reason-labelled telemetry for disabled/unready/fallback states. - - Verification: targeted tests for ready, feature-off, standard-mode, multi-target alias, and wrong-shard states; `dotnet test --project tests/Exceptionless.Tests -- --filter-class `. - -## Stack index migration - -- [ ] 3. Create the next version of `StackIndex` in lookup mode - - Set `index.mode=lookup`, force one primary shard, retain configured replicas and current mapping/analysis. - - Warn when the general shard configuration is greater than one because stack lookup mode overrides it. - - Verification: index configuration test asserts mode, one primary, replicas, mapping aliases, and analyzer preservation. - -- [ ] 4. Validate versioned reindex and alias cutover - - Reindex active and soft-deleted stack documents. - - Verify counts and representative documents before atomic alias switch. - - Verify normal `IStackRepository` get/save/patch/delete behavior on the lookup-mode index. - - Verification: `dotnet test --project tests/Exceptionless.Tests -- --filter-class StackRepositoryTests` plus a dedicated migration integration test. - -## Query service and filters - -- [ ] 5. Add a narrow ES|QL stack-rollup search service - - Isolate direct client use behind `IStackRollupSearchService`. - - Reuse the daily-index resolver and pass values as parameters. - - Apply authorization/project/time predicates before aggregation. - - Verification: service integration tests over at least three daily indices and cancellation/timeout tests. - -- [x] 6. Preserve current event/stack filter semantics - - Reuse `EventStackQueryValidator` and `EventStackFilter` visitors. - - Apply event-only predicates at the source and stack predicates in the lookup join. - - Preserve special fields, `@!`, soft-delete exclusion, premium classification, and no-partial-filter fallback. - - Verification: port the full `CheckStackModeCounts`, deleted-stack, premium-search, mixed field, wildcard, range, missing/exists, and invalid-filter matrices to run against legacy and ES|QL paths. - -- [x] 7. Implement mode aggregation and ordered hydration - - Implement recent/frequent/new/users metrics and fixed mode sorts. - - Append `stack_id ASC` to every sort. - - Hydrate through repositories and restore ES|QL row order before formatting. - - Verification: parity tests assert ids, order, summary metrics, tags, project names, and missing-stack handling for every mode. - -## Cursor pagination and API integration - -- [ ] 8. Add versioned query-bound stack-rollup cursors - - Encode typed metric, stack id, resolved UTC range, mode/sort, fingerprint, and version. - - Implement forward and reverse keyset predicates and reverse-before result handling. - - Reject malformed, mismatched, dual-direction, and page-plus-cursor requests. - - Verification: unit tests for all modes, tied values, forward/back traversal, token version/type validation, relative time freezing, and tampering/mismatch errors. - -- [x] 9. Route cursor stack modes through ES|QL while preserving page fallback - - Keep explicit `page` requests on the legacy path. - - Preserve `sort` rejection and response body/status behavior. - - Populate `before`/`after` pagination links with opaque cursors. - - Verification: `dotnet test --project tests/Exceptionless.Tests -- --filter-class EventEndpointTests` and update `tests/http/*.http` for the additive cursor examples. - -- [ ] 10. Preserve optional total behavior - - Compute total before cursor filtering only when requested. - - Compare lookup-join total semantics with current cardinality behavior. - - Verification: endpoint tests for no total, first/middle/last page total, empty results, filters, and total parity. - -- [ ] 11. Add cursor paging to direct stack-only endpoints using Foundatio search-after, if included in the experimental PR - - Do not route stack-only reads through ES|QL. - - Preserve arbitrary stack sort and append a deterministic id tie-breaker. - - Keep explicit page requests compatible. - - Verification: `dotnet test --project tests/Exceptionless.Tests -- --filter-class StackEndpointTests` with every supported stack sort, ties, null policy, before/after, and page fallback. - -## Failure handling, performance, and completion - -- [ ] 12. Add classified fallback and observability - - Fall back only for feature/readiness/unsupported/transient categories. - - Preserve invalid filter/cursor, authorization, and plan errors. - - Record mode, direction, duration, selected index count, row count, and bounded fallback reason without raw filters/cursors. - - Verification: fault-injection tests for timeout, cancellation, circuit breaker, invalid query, and feature-off behavior. - -- [ ] 13. Add repeatable legacy-versus-ES|QL benchmark coverage - - Seed multiple daily indices, skewed metrics, ties, statuses, deletions, and large stack cardinality. - - Measure repeated warm first/deep pages and report median/p95, allocations, bytes, and failures. - - Verification: committed benchmark command/script and a checked-in results template; run locally against Aspire and attach measurements to the experimental PR. - -- [x] 14. Run targeted verification and strict OpenSpec validation - - `dotnet build` - - `dotnet test --project tests/Exceptionless.Tests -- --filter-class EventEndpointTests` - - `dotnet test --project tests/Exceptionless.Tests -- --filter-class StackEndpointTests` - - `dotnet test --project tests/Exceptionless.Tests -- --filter-class EventStackFilterQueryTests` - - localhost-only cursor smoke test through Aspire - - `openspec validate experiment-esql-lookup-join-stack-pagination --strict --no-interactive` diff --git a/openspec/config.yaml b/openspec/config.yaml deleted file mode 100644 index 197993681f..0000000000 --- a/openspec/config.yaml +++ /dev/null @@ -1,24 +0,0 @@ -schema: spec-driven - -context: | - Project: exceptionless/Exceptionless. - Exceptionless is a real-time error monitoring platform on ASP.NET Core 10, Aspire, Foundatio.Repositories, Elasticsearch, and Svelte 5. - Event documents are stored in date-partitioned daily indices; stack documents are stored in a versioned index. - Preserve public API routes, response bodies, pagination headers, filter syntax, authorization, configuration keys, and persisted data unless an explicit compatibility decision says otherwise. - Use Foundatio.Repositories for ordinary repository work. A narrowly isolated Elasticsearch client call is acceptable when ES|QL cannot be expressed by Foundatio.Repositories. - Local verification must use Aspire/localhost and must not target production or staging. - -rules: - proposal: - - State the user-visible behavior, classification, affected areas, compatibility risks, non-goals, and rollback plan. - - Explain why OpenSpec is justified. - specs: - - Use ADDED, MODIFIED, and REMOVED requirements. - - Use Given/When/Then scenarios for observable behavior. - - Include negative, authorization, compatibility, and Elasticsearch failure scenarios where relevant. - design: - - Reference existing code paths before introducing abstractions. - - For Elasticsearch changes, cover index/schema impact, query behavior, migration/reindex strategy, performance, operations, and failure handling. - tasks: - - Make tasks independently reviewable and include a verification command or test for each task. - - Finish with strict OpenSpec validation and the smallest sufficient targeted tests. diff --git a/src/Exceptionless.Core/Configuration/ElasticsearchOptions.cs b/src/Exceptionless.Core/Configuration/ElasticsearchOptions.cs index 7b96db9fe0..bce6d41cd7 100644 --- a/src/Exceptionless.Core/Configuration/ElasticsearchOptions.cs +++ b/src/Exceptionless.Core/Configuration/ElasticsearchOptions.cs @@ -11,7 +11,6 @@ public class ElasticsearchOptions public int NumberOfReplicas { get; internal set; } public int FieldsLimit { get; internal set; } = 1500; public bool EnableMapperSizePlugin { get; internal set; } - public bool EnableStackRollupLookupJoin { get; internal set; } public string Scope { get; internal set; } = null!; public string ScopePrefix { get; internal set; } = null!; @@ -31,7 +30,6 @@ public static ElasticsearchOptions ReadFromConfiguration(IConfiguration config, options.DisableIndexConfiguration = config.GetValue(nameof(options.DisableIndexConfiguration), false); options.EnableSnapshotJobs = config.GetValue(nameof(options.EnableSnapshotJobs), String.IsNullOrEmpty(options.ScopePrefix) && appOptions.AppMode == AppMode.Production); - options.EnableStackRollupLookupJoin = config.GetValue(nameof(options.EnableStackRollupLookupJoin), appOptions.AppMode == AppMode.Development); options.ReindexCutOffDate = config.GetValue(nameof(options.ReindexCutOffDate), DateTime.MinValue); string? connectionString = config.GetConnectionString("Elasticsearch"); diff --git a/src/Exceptionless.Core/Services/StackRollupSearchService.cs b/src/Exceptionless.Core/Services/StackRollupSearchService.cs index d62a3c7eba..86c6030278 100644 --- a/src/Exceptionless.Core/Services/StackRollupSearchService.cs +++ b/src/Exceptionless.Core/Services/StackRollupSearchService.cs @@ -21,7 +21,7 @@ namespace Exceptionless.Core.Services; public interface IStackRollupSearchService { - Task SearchAsync(StackRollupSearchRequest request, CancellationToken cancellationToken = default); + Task SearchAsync(StackRollupSearchRequest request, CancellationToken cancellationToken = default); } public sealed record StackRollupSearchRequest( @@ -59,7 +59,6 @@ public sealed class StackRollupSearchService : IStackRollupSearchService private static readonly TimeSpan ReadinessCacheDuration = TimeSpan.FromMinutes(1); private readonly ElasticsearchClient _client; private readonly ExceptionlessElasticConfiguration _configuration; - private readonly ElasticsearchOptions _options; private readonly TimeProvider _timeProvider; private readonly JsonSerializerOptions _serializerOptions; private readonly EventStackFilter _eventStackFilter = new(); @@ -71,34 +70,29 @@ public sealed class StackRollupSearchService : IStackRollupSearchService public StackRollupSearchService( ElasticsearchClient client, ExceptionlessElasticConfiguration configuration, - ElasticsearchOptions options, TimeProvider timeProvider, JsonSerializerOptions serializerOptions, ILoggerFactory loggerFactory) { _client = client; _configuration = configuration; - _options = options; _timeProvider = timeProvider; _serializerOptions = serializerOptions; _logger = loggerFactory.CreateLogger(); } - public async Task SearchAsync(StackRollupSearchRequest request, CancellationToken cancellationToken = default) + public async Task SearchAsync(StackRollupSearchRequest request, CancellationToken cancellationToken = default) { - if (!_options.EnableStackRollupLookupJoin) - return null; + if (!IsSupportedMode(request.Mode)) + throw new ArgumentOutOfRangeException(nameof(request), request.Mode, "Unsupported stack rollup mode."); var readiness = await GetReadinessAsync(cancellationToken); if (!readiness.IsReady) { - _logger.LogDebug("Stack rollup lookup join is unavailable: {Reason}", readiness.Reason); - return null; + _logger.LogError("Stack rollup lookup join prerequisite failed: {Reason}", readiness.Reason); + throw new InvalidOperationException($"The stack rollup lookup join prerequisite failed: {readiness.Reason}."); } - if (!IsSupportedMode(request.Mode)) - return null; - string fingerprint = CreateFingerprint(request); StackRollupCursor? cursor = DecodeCursor(request.Before ?? request.After, request.Mode, fingerprint); DateTime utcStart = cursor is null ? request.UtcStart : new DateTime(cursor.UtcStart, DateTimeKind.Utc); diff --git a/src/Exceptionless.Web/Api/Handlers/EventHandler.cs b/src/Exceptionless.Web/Api/Handlers/EventHandler.cs index 6df4e648dd..77a270531f 100644 --- a/src/Exceptionless.Web/Api/Handlers/EventHandler.cs +++ b/src/Exceptionless.Web/Api/Handlers/EventHandler.cs @@ -750,18 +750,19 @@ private async Task>> GetInternalAsync(AppFilter sf, T .SetHttpContext(httpContext) ); + bool isStackMode = IsStackMode(mode); + if (isStackMode && before is not null && after is not null) + return Result.BadRequest("The before and after parameters cannot be used together."); + + if (isStackMode && page.HasValue) + return Result.BadRequest("The page parameter is not supported in stack mode. Use before or after cursor pagination."); + int resolvedPage = Pagination.GetPage(page.GetValueOrDefault(1)); limit = Pagination.GetLimit(limit); int skip = Pagination.GetSkip(resolvedPage, limit); if (skip > Pagination.MaximumSkip) return new PagedResult(Array.Empty(), false); - if (IsStackMode(mode) && before is not null && after is not null) - return Result.BadRequest("The before and after parameters cannot be used together."); - - if (IsStackMode(mode) && page.HasValue && (before is not null || after is not null)) - return Result.BadRequest("The page parameter cannot be combined with before or after."); - var pr = await GetQueryValidator(mode).ValidateQueryAsync(filter); if (!pr.IsValid) return Result.BadRequest(pr.Message ?? "Invalid filter."); @@ -804,83 +805,32 @@ private async Task>> GetInternalAsync(AppFilter sf, T if (!String.IsNullOrEmpty(sort)) return Result.BadRequest("Sort is not supported in stack mode."); - if (!page.HasValue) - { - var lookupResult = await stackRollupSearchService.SearchAsync(new StackRollupSearchRequest( - appliedAppFilter, - ti.Range.UtcStart, - ti.Range.UtcEnd, - ti.Offset, - timeExpression, - filter, - mode, - limit, - before, - after, - includeTotal), httpContext.RequestAborted); - - if (lookupResult is not null) - { - string[] lookupStackIds = lookupResult.Rows.Select(row => row.StackId).ToArray(); - var lookupStacks = (await stackRepository.GetByIdsAsync(lookupStackIds)) - .Select(stack => stack.ApplyOffset(ti.Offset)) - .ToList(); - var lookupSummaries = await GetStackSummariesAsync(lookupStacks, lookupResult.Rows, sf, ti); - - return new PagedResult( - lookupSummaries.Cast().ToList(), - lookupResult.HasMore, - Page: null, - lookupResult.Total, - lookupResult.Before, - lookupResult.After); - } - - if (before is not null || after is not null) - return Result.BadRequest("Stack cursor paging is not available for the current Elasticsearch index."); - } - - var systemFilter = new RepositoryQuery() - .AppFilter(appliedAppFilter) - .EnforceEventStackFilter() - .DateRange(ti.Range.UtcStart, ti.Range.UtcEnd, (PersistentEvent e) => e.Date) - .Index(ti.Range.UtcStart, ti.Range.UtcEnd); - - string? stackAggregations = mode switch - { - "stack_recent" => "cardinality:user sum:count~1 min:date -max:date", - "stack_frequent" => "cardinality:user -sum:count~1 min:date max:date", - "stack_new" => "cardinality:user sum:count~1 -min:date max:date", - "stack_users" => "-cardinality:user sum:count~1 min:date max:date", - _ => null - }; - - if (mode == "stack_new") - filter = AddFirstOccurrenceFilter(ti.Range, filter); - - string aggregationExpression = includeTotal - ? $"cardinality:stack_id terms:(stack_id~{Pagination.GetSkip(resolvedPage + 1, limit) + 1} {stackAggregations})" - : $"terms:(stack_id~{Pagination.GetSkip(resolvedPage + 1, limit) + 1} {stackAggregations})"; - - var countResponse = await eventRepository.CountAsync(q => q - .SystemFilter(systemFilter) - .FilterExpression(filter) - .EnforceEventStackFilter() - .AggregationsExpression(aggregationExpression), - o => o.TrackTotalHits(false)); - - var stackTerms = countResponse.Aggregations.Terms("terms_stack_id"); - if (stackTerms is null || stackTerms.Buckets.Count == 0) - return new PagedResult(Array.Empty(), false); - - string[] stackIds = stackTerms.Buckets.Skip(skip).Take(limit + 1).Select(t => t.Key).ToArray(); - var stacks = (await stackRepository.GetByIdsAsync(stackIds)).Select(s => s.ApplyOffset(ti.Offset)).ToList(); - - var stackSummaries = await GetStackSummariesAsync(stacks, stackTerms.Buckets, sf, ti); - - double? totalStackCount = countResponse.Aggregations.Cardinality("cardinality_stack_id")?.Value; - long? total = includeTotal && totalStackCount.HasValue ? Convert.ToInt64(totalStackCount.Value) : null; - return new PagedResult(stackSummaries.Take(limit).Cast().ToList(), stackSummaries.Count > limit && !Pagination.NextPageExceedsSkipLimit(resolvedPage, limit), resolvedPage, total); + var lookupResult = await stackRollupSearchService.SearchAsync(new StackRollupSearchRequest( + appliedAppFilter, + ti.Range.UtcStart, + ti.Range.UtcEnd, + ti.Offset, + timeExpression, + filter, + mode, + limit, + before, + after, + includeTotal), httpContext.RequestAborted); + + string[] lookupStackIds = lookupResult.Rows.Select(row => row.StackId).ToArray(); + var lookupStacks = (await stackRepository.GetByIdsAsync(lookupStackIds)) + .Select(stack => stack.ApplyOffset(ti.Offset)) + .ToList(); + var lookupSummaries = await GetStackSummariesAsync(lookupStacks, lookupResult.Rows, sf, ti); + + return new PagedResult( + lookupSummaries.Cast().ToList(), + lookupResult.HasMore, + Page: null, + lookupResult.Total, + lookupResult.Before, + lookupResult.After); default: events = await GetEventsInternalAsync(appliedAppFilter, ti, filter, sort, page, limit, before, after, includeTotal); return new PagedResult(events.Documents.Cast().ToList(), events.HasMore && !Pagination.NextPageExceedsSkipLimit(page, limit), page, includeTotal ? events.Total : null, events.Hits.FirstOrDefault()?.GetSortToken(serializer), events.Hits.LastOrDefault()?.GetSortToken(serializer)); @@ -962,39 +912,6 @@ private Task> GetEventsInternalAsync(AppFilter? sys : o.SearchBeforeToken(before, serializer).SearchAfterToken(after, serializer).PageLimit(limit).TrackTotalHits(includeTotal)); } - private async Task> GetStackSummariesAsync(List stacks, IReadOnlyCollection> stackTerms, AppFilter sf, TimeInfo ti) - { - if (stacks.Count == 0) - return new List(0); - - var projects = await projectRepository.GetByIdsAsync(stacks.Select(s => s.ProjectId).Distinct().ToArray(), o => o.Cache()); - var projectNames = projects.ToDictionary(p => p.Id, p => p.Name); - var totalUsers = await GetUserCountByProjectIdsAsync(stacks, sf, ti.Range.UtcStart, ti.Range.UtcEnd); - return stacks.Join(stackTerms, s => s.Id, tk => tk.Key, (stack, term) => - { - var data = formattingPluginManager.GetStackSummaryData(stack); - var summary = new StackSummaryModel - { - Id = data.Id, - TemplateKey = data.TemplateKey, - Data = data.Data, - ProjectId = stack.ProjectId, - ProjectName = projectNames.GetValueOrDefault(stack.ProjectId), - Tags = stack.Tags?.OfType().Order(StringComparer.OrdinalIgnoreCase).ToArray() ?? [], - Title = stack.Title, - Status = stack.Status, - FirstOccurrence = term.Aggregations.Min("min_date")?.Value ?? stack.FirstOccurrence, - LastOccurrence = term.Aggregations.Max("max_date")?.Value ?? stack.LastOccurrence, - Total = (long)(term.Aggregations.Sum("sum_count")?.Value ?? term.Total.GetValueOrDefault()), - - Users = term.Aggregations.Cardinality("cardinality_user")?.Value.GetValueOrDefault() ?? 0, - TotalUsers = totalUsers.GetOrDefault(stack.ProjectId) - }; - - return summary; - }).ToList(); - } - private async Task> GetStackSummariesAsync(List stacks, IReadOnlyCollection rows, AppFilter sf, TimeInfo ti) { if (stacks.Count == 0) diff --git a/src/Exceptionless.Web/ClientApp/src/routes/(app)/stack/+page.svelte b/src/Exceptionless.Web/ClientApp/src/routes/(app)/stack/+page.svelte index b416799bdd..1062e853ea 100644 --- a/src/Exceptionless.Web/ClientApp/src/routes/(app)/stack/+page.svelte +++ b/src/Exceptionless.Web/ClientApp/src/routes/(app)/stack/+page.svelte @@ -668,19 +668,13 @@ const eventsQuery = getOrganizationEventsQuery({ enabled: () => !isSavedViewRoutePending, get params() { - const params = { + const { page: ignoredPage, ...params } = { ...eventsQueryParameters, include: !eventsQueryParameters.after && !eventsQueryParameters.before ? ('total' as const) : undefined }; - - if (!eventsQueryParameters.after && !eventsQueryParameters.before) { - return params; - } - - const { page: ignoredPage, ...cursorParams } = params; void ignoredPage; - return cursorParams; + return params; }, route: { get organizationId() { diff --git a/tests/Exceptionless.Tests/Api/Endpoints/EventEndpointTests.cs b/tests/Exceptionless.Tests/Api/Endpoints/EventEndpointTests.cs index f20bcc987d..4ad2340486 100644 --- a/tests/Exceptionless.Tests/Api/Endpoints/EventEndpointTests.cs +++ b/tests/Exceptionless.Tests/Api/Endpoints/EventEndpointTests.cs @@ -886,8 +886,8 @@ public async Task CanPageStackModesWithCursors(string mode) [InlineData("before", "malformed", null, null)] [InlineData("after", "malformed", null, null)] [InlineData("before", "malformed", "after", "malformed")] - [InlineData("after", "malformed", "page", "1")] - public async Task RejectsInvalidStackCursorRequests(string firstName, string firstValue, string? secondName, string? secondValue) + [InlineData("page", "1", null, null)] + public async Task GetEvents_StackPaginationIsInvalid_ReturnsBadRequest(string firstName, string firstValue, string? secondName, string? secondValue) { await CreateStacksAndEventsAsync(); From 91fce991fe2397fb45eb26cc0c6ed340565c1154 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Sun, 23 Aug 2026 01:39:42 -0500 Subject: [PATCH 4/9] Refactor stack rollups around ES|QL lookup joins --- .../Exceptionless.AppHost.csproj | 1 - .../Extensions/ElasticsearchExtensions.cs | 36 +- .../Services/StackRollupSearchService.cs | 301 +++++-- .../Api/Endpoints/EventEndpoints.cs | 15 +- .../Api/Endpoints/StackEndpoints.cs | 81 +- .../Api/Handlers/EventHandler.cs | 171 +--- .../Api/Handlers/StackHandler.cs | 108 +-- .../Api/Handlers/StackRollupHandler.cs | 301 +++++++ .../Api/Messages/EventMessages.cs | 6 +- .../Api/Messages/StackMessages.cs | 12 +- .../app/frequent-controller.js | 4 +- .../ClientApp.angular/app/new-controller.js | 21 +- .../ClientApp.angular/app/users-controller.js | 4 +- .../components/stack/stack-service.js | 58 +- .../e2e/tests/list-query-cache.e2e.ts | 6 +- .../e2e/tests/stack-effects-chaos.e2e.ts | 10 +- .../ClientApp/e2e/tests/stack-triage.e2e.ts | 35 + .../src/lib/features/events/api.svelte.ts | 5 +- .../components/table/events-data-table.svelte | 10 +- .../components/table/options.svelte.test.ts | 2 +- .../src/lib/features/stacks/api.svelte.ts | 104 ++- .../src/lib/features/stacks/api.test.ts | 26 +- .../stacks/components/table/options.svelte.ts | 2 +- .../(components)/navigation-command.svelte | 4 +- .../ClientApp/src/routes/(app)/+layout.svelte | 12 +- .../project/[projectId]/stacks/+page.svelte | 29 +- .../src/routes/(app)/stack/+page.svelte | 119 +-- .../Api/Data/endpoint-manifest.json | 84 ++ .../Exceptionless.Tests/Api/Data/openapi.json | 823 ++++++++++++++++-- .../EventEndpointTests.PremiumSearch.cs | 27 +- .../Api/Endpoints/EventEndpointTests.cs | 213 +++-- .../Api/Endpoints/StackEndpointTests.cs | 66 ++ .../Api/OpenApiSnapshotTests.cs | 19 + .../Performance/StackRollupBenchmarkTests.cs | 342 ++++++++ tests/http/events.http | 4 +- 35 files changed, 2376 insertions(+), 685 deletions(-) create mode 100644 src/Exceptionless.Web/Api/Handlers/StackRollupHandler.cs create mode 100644 tests/Exceptionless.Tests/Performance/StackRollupBenchmarkTests.cs diff --git a/src/Exceptionless.AppHost/Exceptionless.AppHost.csproj b/src/Exceptionless.AppHost/Exceptionless.AppHost.csproj index 16cc44ac96..d2a4d8d9ff 100644 --- a/src/Exceptionless.AppHost/Exceptionless.AppHost.csproj +++ b/src/Exceptionless.AppHost/Exceptionless.AppHost.csproj @@ -12,7 +12,6 @@ - diff --git a/src/Exceptionless.AppHost/Extensions/ElasticsearchExtensions.cs b/src/Exceptionless.AppHost/Extensions/ElasticsearchExtensions.cs index 4022e89583..9a7762d4dc 100644 --- a/src/Exceptionless.AppHost/Extensions/ElasticsearchExtensions.cs +++ b/src/Exceptionless.AppHost/Extensions/ElasticsearchExtensions.cs @@ -1,4 +1,4 @@ -using Elastic.Clients.Elasticsearch; +using System.Text.Json; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Diagnostics.HealthChecks; @@ -136,19 +136,25 @@ public async Task CheckHealthAsync(HealthCheckContext context if (string.IsNullOrEmpty(connectionString)) return new HealthCheckResult(context.Registration.FailureStatus, "Connection string not available."); - using var settings = new ElasticsearchClientSettings(new Uri(connectionString)); - var client = new ElasticsearchClient(settings); - var response = await client.Cluster.HealthAsync( - request => request.WaitForStatus(Elastic.Clients.Elasticsearch.HealthStatus.Yellow), - cancellationToken); - bool isReady = response.IsValidResponse - && !response.TimedOut - && response.Status is Elastic.Clients.Elasticsearch.HealthStatus.Yellow or Elastic.Clients.Elasticsearch.HealthStatus.Green; - if (isReady) - return HealthCheckResult.Healthy(); - - return new HealthCheckResult( - context.Registration.FailureStatus, - $"Elasticsearch cluster health check failed. Timed out: {response.TimedOut}; status: {response.Status}. {response.DebugInformation}"); + try + { + using var client = new HttpClient { BaseAddress = new Uri(connectionString), Timeout = TimeSpan.FromSeconds(10) }; + using var response = await client.GetAsync("_cluster/health?wait_for_status=yellow&timeout=5s", cancellationToken); + if (!response.IsSuccessStatusCode) + return new HealthCheckResult(context.Registration.FailureStatus, $"Elasticsearch cluster health returned HTTP {(int)response.StatusCode}."); + + await using var responseStream = await response.Content.ReadAsStreamAsync(cancellationToken); + using var document = await JsonDocument.ParseAsync(responseStream, cancellationToken: cancellationToken); + bool timedOut = document.RootElement.TryGetProperty("timed_out", out var timedOutElement) && timedOutElement.GetBoolean(); + string? status = document.RootElement.TryGetProperty("status", out var statusElement) ? statusElement.GetString() : null; + if (!timedOut && status is "yellow" or "green") + return HealthCheckResult.Healthy(); + + return new HealthCheckResult(context.Registration.FailureStatus, $"Elasticsearch cluster health check failed. Timed out: {timedOut}; status: {status ?? "unknown"}."); + } + catch (Exception ex) when (ex is HttpRequestException or JsonException or TaskCanceledException) + { + return new HealthCheckResult(context.Registration.FailureStatus, "Elasticsearch cluster health check request failed.", ex); + } } } diff --git a/src/Exceptionless.Core/Services/StackRollupSearchService.cs b/src/Exceptionless.Core/Services/StackRollupSearchService.cs index 86c6030278..08e09acdc7 100644 --- a/src/Exceptionless.Core/Services/StackRollupSearchService.cs +++ b/src/Exceptionless.Core/Services/StackRollupSearchService.cs @@ -22,6 +22,8 @@ namespace Exceptionless.Core.Services; public interface IStackRollupSearchService { Task SearchAsync(StackRollupSearchRequest request, CancellationToken cancellationToken = default); + Task GetStatsAsync(StackRollupStatsRequest request, CancellationToken cancellationToken = default); + Task> GetProjectUserCountsAsync(StackRollupProjectUsersRequest request, CancellationToken cancellationToken = default); } public sealed record StackRollupSearchRequest( @@ -31,7 +33,7 @@ public sealed record StackRollupSearchRequest( TimeSpan Offset, string? TimeExpression, string? Filter, - string Mode, + string? Sort, int Limit, string? Before, string? After, @@ -44,6 +46,28 @@ public sealed record StackRollupSearchResult( string? Before, string? After); +public sealed record StackRollupStatsRequest( + AppFilter? AppFilter, + DateTime UtcStart, + DateTime UtcEnd, + TimeSpan Offset, + string? Filter, + int BucketCount = 50); + +public sealed record StackRollupStatsResult( + long TotalEvents, + long TotalStacks, + long NewStacks, + IReadOnlyCollection Buckets); + +public sealed record StackRollupStatsBucket(DateTime Date, long Events, long Stacks); + +public sealed record StackRollupProjectUsersRequest( + AppFilter AppFilter, + DateTime UtcStart, + DateTime UtcEnd, + IReadOnlyCollection ProjectIds); + public sealed record StackRollupRow( string StackId, long Total, @@ -83,31 +107,22 @@ public StackRollupSearchService( public async Task SearchAsync(StackRollupSearchRequest request, CancellationToken cancellationToken = default) { - if (!IsSupportedMode(request.Mode)) - throw new ArgumentOutOfRangeException(nameof(request), request.Mode, "Unsupported stack rollup mode."); - - var readiness = await GetReadinessAsync(cancellationToken); - if (!readiness.IsReady) - { - _logger.LogError("Stack rollup lookup join prerequisite failed: {Reason}", readiness.Reason); - throw new InvalidOperationException($"The stack rollup lookup join prerequisite failed: {readiness.Reason}."); - } + var sort = GetSort(request.Sort); + await EnsureReadyAsync(cancellationToken); string fingerprint = CreateFingerprint(request); - StackRollupCursor? cursor = DecodeCursor(request.Before ?? request.After, request.Mode, fingerprint); + StackRollupCursor? cursor = DecodeCursor(request.Before ?? request.After, sort.Value, fingerprint); DateTime utcStart = cursor is null ? request.UtcStart : new DateTime(cursor.UtcStart, DateTimeKind.Utc); DateTime utcEnd = cursor is null ? request.UtcEnd : new DateTime(cursor.UtcEnd, DateTimeKind.Utc); string normalizedFilter = StripAlternateInversion(request.Filter); - if (request.Mode == "stack_new") - normalizedFilter = AddFirstOccurrenceFilter(utcStart, utcEnd, normalizedFilter); string? eventFilter = await _eventStackFilter.GetEventFilterAsync(normalizedFilter); string? stackFilter = (await _eventStackFilter.GetStackFilterAsync(normalizedFilter))?.Filter; Query? sourceFilter = await BuildSourceFilterAsync(request.AppFilter, utcStart, utcEnd, eventFilter); var stopwatch = Stopwatch.StartNew(); _logger.LogDebug( - "Executing ES|QL stack rollup mode {Mode}, direction {Direction}, range {UtcStart:o} to {UtcEnd:o}, event filter {HasEventFilter}, stack filter {HasStackFilter}", - request.Mode, + "Executing ES|QL stack rollup sort {Sort}, direction {Direction}, range {UtcStart:o} to {UtcEnd:o}, event filter {HasEventFilter}, stack filter {HasStackFilter}", + sort.Value, request.Before is not null ? "before" : request.After is not null ? "after" : "initial", utcStart, utcEnd, @@ -115,14 +130,14 @@ public async Task SearchAsync(StackRollupSearchRequest !String.IsNullOrWhiteSpace(stackFilter)); var parameters = new List>>(); - string query = BuildQuery(request, cursor, stackFilter, parameters, countOnly: false); + string query = BuildQuery(request, sort, cursor, stackFilter, parameters, countOnly: false); var rows = await ExecuteRowsAsync(query, sourceFilter, parameters, cancellationToken); long? total = request.IncludeTotal ? rows.FirstOrDefault()?.TotalStacks : null; if (request.IncludeTotal && total is null) { parameters.Clear(); - string countQuery = BuildQuery(request, cursor: null, stackFilter, parameters, countOnly: true); + string countQuery = BuildQuery(request, sort, cursor: null, stackFilter, parameters, countOnly: true); total = await ExecuteTotalAsync(countQuery, sourceFilter, parameters, cancellationToken); } @@ -136,12 +151,12 @@ public async Task SearchAsync(StackRollupSearchRequest bool hasPrevious = rows.Count > 0 && (isBefore ? hasExtra : request.After is not null); bool hasNext = rows.Count > 0 && (isBefore || hasExtra); - string? before = hasPrevious ? EncodeCursor(rows[0], request, utcStart, utcEnd, fingerprint) : null; - string? after = hasNext ? EncodeCursor(rows[^1], request, utcStart, utcEnd, fingerprint) : null; + string? before = hasPrevious ? EncodeCursor(rows[0], sort, utcStart, utcEnd, fingerprint) : null; + string? after = hasNext ? EncodeCursor(rows[^1], sort, utcStart, utcEnd, fingerprint) : null; _logger.LogDebug( - "Completed ES|QL stack rollup mode {Mode}, direction {Direction}, rows {RowCount}, has more {HasMore}, duration {DurationMs}ms", - request.Mode, + "Completed ES|QL stack rollup sort {Sort}, direction {Direction}, rows {RowCount}, has more {HasMore}, duration {DurationMs}ms", + sort.Value, isBefore ? "before" : request.After is not null ? "after" : "initial", rows.Count, hasNext, @@ -155,6 +170,86 @@ public async Task SearchAsync(StackRollupSearchRequest after); } + public async Task GetStatsAsync(StackRollupStatsRequest request, CancellationToken cancellationToken = default) + { + await EnsureReadyAsync(cancellationToken); + string normalizedFilter = StripAlternateInversion(request.Filter); + string? eventFilter = await _eventStackFilter.GetEventFilterAsync(normalizedFilter); + string? stackFilter = (await _eventStackFilter.GetStackFilterAsync(normalizedFilter))?.Filter; + Query? sourceFilter = await BuildSourceFilterAsync(request.AppFilter, request.UtcStart, request.UtcEnd, eventFilter); + var parameters = new List>>(); + string query = BuildStatsQuery(request, stackFilter, parameters); + + using var response = await _client.Esql.QueryAsync(new EsqlQueryRequest(query) + { + AllowPartialResults = false, + Columnar = false, + Filter = sourceFilter, + Format = EsqlFormat.Json, + Params = new Union>, ICollection>>>(parameters) + }, cancellationToken); + + if (!response.IsValidResponse) + { + _logger.LogWarning("Stack rollup stats lookup join failed with Elasticsearch status {StatusCode}", response.ApiCallDetails?.HttpStatusCode); + throw new ApplicationException("The stack rollup stats query failed."); + } + + using var document = await JsonDocument.ParseAsync(response.Body, cancellationToken: cancellationToken); + return ReadStats(document.RootElement); + } + + public async Task> GetProjectUserCountsAsync(StackRollupProjectUsersRequest request, CancellationToken cancellationToken = default) + { + if (request.ProjectIds.Count == 0) + return new Dictionary(); + + await EnsureReadyAsync(cancellationToken); + Query? sourceFilter = await BuildSourceFilterAsync(request.AppFilter, request.UtcStart, request.UtcEnd, eventFilter: null); + string eventIndex = ValidateIndexName(_configuration.Events.Name); + string stackIndex = ValidateIndexName(_configuration.Stacks.Name); + string userField = EscapeIdentifier(EventIndexExtensions.DataPath(Event.KnownDataKeys.UserInfo, user => user.Identity) + ".keyword"); + var parameters = new List>>(); + AddParameter(parameters, "project_ids", request.ProjectIds.Select(FieldValue.String).ToArray()); + string query = new StringBuilder() + .Append("FROM ").Append(eventIndex) + .Append(" | KEEP stack_id, project_id, ").Append(userField) + .Append(" | RENAME project_id AS event_project_id, ").Append(userField).Append(" AS event_user") + .Append(" | LOOKUP JOIN ").Append(stackIndex).Append(" ON stack_id == id AND is_deleted == false") + .Append(" | WHERE id IS NOT NULL AND event_project_id IN (?project_ids)") + .Append(" | STATS users = COUNT_DISTINCT(event_user) BY project_id = event_project_id") + .Append(" | KEEP project_id, users") + .ToString(); + + using var response = await _client.Esql.QueryAsync(new EsqlQueryRequest(query) + { + AllowPartialResults = false, + Columnar = false, + Filter = sourceFilter, + Format = EsqlFormat.Json, + Params = new Union>, ICollection>>>(parameters) + }, cancellationToken); + + if (!response.IsValidResponse) + { + _logger.LogWarning("Stack rollup project user lookup join failed with Elasticsearch status {StatusCode}", response.ApiCallDetails?.HttpStatusCode); + throw new ApplicationException("The stack rollup project user query failed."); + } + + using var document = await JsonDocument.ParseAsync(response.Body, cancellationToken: cancellationToken); + return ReadProjectUserCounts(document.RootElement); + } + + private async Task EnsureReadyAsync(CancellationToken cancellationToken) + { + var readiness = await GetReadinessAsync(cancellationToken); + if (readiness.IsReady) + return; + + _logger.LogError("Stack rollup lookup join prerequisite failed: {Reason}", readiness.Reason); + throw new InvalidOperationException($"The stack rollup lookup join prerequisite failed: {readiness.Reason}."); + } + private async Task BuildSourceFilterAsync(AppFilter? appFilter, DateTime utcStart, DateTime utcEnd, string? eventFilter) { var query = new RepositoryQuery() @@ -173,6 +268,7 @@ public async Task SearchAsync(StackRollupSearchRequest private string BuildQuery( StackRollupSearchRequest request, + StackRollupSort sort, StackRollupCursor? cursor, string? stackFilter, ICollection>> parameters, @@ -204,27 +300,29 @@ private string BuildQuery( if (request.IncludeTotal) query.Append(" | INLINE STATS total_stacks = COUNT(*)"); - var mode = GetMode(request.Mode); bool isBefore = request.Before is not null; if (cursor is not null) { - string primaryComparison = isBefore ? ">" : "<"; + string primaryComparison = isBefore + ? sort.Ascending ? "<" : ">" + : sort.Ascending ? ">" : "<"; string idComparison = isBefore ? "<" : ">"; - string metricParameter = mode.IsDate ? "TO_DATETIME(?cursor_metric)" : "?cursor_metric"; + string metricParameter = sort.IsDate ? "TO_DATETIME(?cursor_metric)" : "?cursor_metric"; query - .Append(" | WHERE ").Append(mode.Metric).Append(' ').Append(primaryComparison).Append(' ').Append(metricParameter) - .Append(" OR (").Append(mode.Metric).Append(" == ").Append(metricParameter) + .Append(" | WHERE ").Append(sort.Metric).Append(' ').Append(primaryComparison).Append(' ').Append(metricParameter) + .Append(" OR (").Append(sort.Metric).Append(" == ").Append(metricParameter) .Append(" AND stack_id ").Append(idComparison).Append(" ?cursor_stack_id)"); - AddParameter(parameters, "cursor_metric", mode.IsDate + AddParameter(parameters, "cursor_metric", sort.IsDate ? FieldValue.String(new DateTime(cursor.Metric, DateTimeKind.Utc).ToString("O", CultureInfo.InvariantCulture)) : FieldValue.Long(cursor.Metric)); AddParameter(parameters, "cursor_stack_id", FieldValue.String(cursor.StackId)); } - string primarySort = isBefore ? "ASC" : "DESC"; + bool queryAscending = isBefore ? !sort.Ascending : sort.Ascending; + string primarySort = queryAscending ? "ASC" : "DESC"; string idSort = isBefore ? "DESC" : "ASC"; query - .Append(" | SORT ").Append(mode.Metric).Append(' ').Append(primarySort).Append(", stack_id ").Append(idSort) + .Append(" | SORT ").Append(sort.Metric).Append(' ').Append(primarySort).Append(", stack_id ").Append(idSort) .Append(" | LIMIT ").Append(request.Limit + 1) .Append(" | KEEP stack_id, event_total, event_users, event_first, event_last"); @@ -234,6 +332,39 @@ private string BuildQuery( return query.ToString(); } + private string BuildStatsQuery( + StackRollupStatsRequest request, + string? stackFilter, + ICollection>> parameters) + { + string eventIndex = ValidateIndexName(_configuration.Events.Name); + string stackIndex = ValidateIndexName(_configuration.Stacks.Name); + int bucketCount = Math.Clamp(request.BucketCount, 1, 100); + string timeZone = FormatTimeZone(request.Offset); + string utcStart = request.UtcStart.ToString("O", CultureInfo.InvariantCulture); + string utcEnd = request.UtcEnd.ToString("O", CultureInfo.InvariantCulture); + var query = new StringBuilder() + .Append("SET time_zone = \"").Append(timeZone).Append("\"; FROM ").Append(eventIndex) + .Append(" | KEEP stack_id, count, date, is_first_occurrence") + .Append(" | LOOKUP JOIN ").Append(stackIndex) + .Append(" ON stack_id == id AND is_deleted == false"); + + if (!String.IsNullOrWhiteSpace(stackFilter)) + { + query.Append(" AND QSTR(?stack_filter, {\"default_operator\": \"AND\"})"); + AddParameter(parameters, "stack_filter", FieldValue.String(stackFilter)); + } + + return query + .Append(" | WHERE id IS NOT NULL") + .Append(" | INLINE STATS total_events = SUM(COALESCE(count, 1)), total_stacks = COUNT_DISTINCT(stack_id, 40000), new_stacks = SUM(CASE(is_first_occurrence, 1, 0))") + .Append(" | STATS events = SUM(COALESCE(count, 1)), stacks = COUNT_DISTINCT(stack_id), total_events = MAX(total_events), total_stacks = MAX(total_stacks), new_stacks = MAX(new_stacks)") + .Append(" BY bucket = BUCKET(date, ").Append(bucketCount).Append(", \"").Append(utcStart).Append("\", \"").Append(utcEnd).Append("\")") + .Append(" | SORT bucket | LIMIT ").Append(bucketCount + 2) + .Append(" | KEEP bucket, events, stacks, total_events, total_stacks, new_stacks") + .ToString(); + } + private async Task> ExecuteRowsAsync( string query, Query? sourceFilter, @@ -315,6 +446,62 @@ private static List ReadRows(JsonElement root) return rows; } + private static StackRollupStatsResult ReadStats(JsonElement root) + { + if (!root.TryGetProperty("columns", out var columns) || !root.TryGetProperty("values", out var values)) + throw new JsonException("The ES|QL stack rollup stats response did not contain columns and values."); + if (values.GetArrayLength() == 0) + return new StackRollupStatsResult(0, 0, 0, []); + + var columnIndexes = GetColumnIndexes(columns); + int bucketIndex = columnIndexes["bucket"]; + int eventsIndex = columnIndexes["events"]; + int stacksIndex = columnIndexes["stacks"]; + int totalEventsIndex = columnIndexes["total_events"]; + int totalStacksIndex = columnIndexes["total_stacks"]; + int newStacksIndex = columnIndexes["new_stacks"]; + var buckets = new List(values.GetArrayLength()); + long totalEvents = 0; + long totalStacks = 0; + long newStacks = 0; + foreach (var value in values.EnumerateArray()) + { + totalEvents = value[totalEventsIndex].GetInt64(); + totalStacks = value[totalStacksIndex].GetInt64(); + newStacks = value[newStacksIndex].GetInt64(); + buckets.Add(new StackRollupStatsBucket( + value[bucketIndex].GetDateTimeOffset().UtcDateTime, + value[eventsIndex].GetInt64(), + value[stacksIndex].GetInt64())); + } + + return new StackRollupStatsResult(totalEvents, totalStacks, newStacks, buckets); + } + + private static IReadOnlyDictionary ReadProjectUserCounts(JsonElement root) + { + if (!root.TryGetProperty("columns", out var columns) || !root.TryGetProperty("values", out var values)) + throw new JsonException("The ES|QL project user response did not contain columns and values."); + + var columnIndexes = GetColumnIndexes(columns); + int projectIndex = columnIndexes["project_id"]; + int usersIndex = columnIndexes["users"]; + var result = new Dictionary(StringComparer.Ordinal); + foreach (var value in values.EnumerateArray()) + { + string projectId = value[projectIndex].GetString() ?? throw new JsonException("A project user row did not contain a project id."); + result[projectId] = value[usersIndex].GetInt64(); + } + + return result; + } + + private static IReadOnlyDictionary GetColumnIndexes(JsonElement columns) + => columns.EnumerateArray() + .Select((column, index) => (Name: column.GetProperty("name").GetString(), Index: index)) + .Where(column => column.Name is not null) + .ToDictionary(column => column.Name!, column => column.Index, StringComparer.Ordinal); + private async Task GetReadinessAsync(CancellationToken cancellationToken) { DateTimeOffset now = _timeProvider.GetUtcNow(); @@ -360,10 +547,9 @@ private async Task CheckReadinessAsync(CancellationToken c : new StackRollupReadiness(false, "stack-primary-shards"); } - private string EncodeCursor(StackRollupEsqlRow row, StackRollupSearchRequest request, DateTime utcStart, DateTime utcEnd, string fingerprint) + private string EncodeCursor(StackRollupEsqlRow row, StackRollupSort sort, DateTime utcStart, DateTime utcEnd, string fingerprint) { - var mode = GetMode(request.Mode); - long metric = mode.Metric switch + long metric = sort.Metric switch { "event_total" => row.Total, "event_users" => row.Users, @@ -371,12 +557,12 @@ private string EncodeCursor(StackRollupEsqlRow row, StackRollupSearchRequest req "event_last" => row.LastOccurrence.Ticks, _ => throw new InvalidOperationException("Unsupported stack rollup metric.") }; - var cursor = new StackRollupCursor(CursorVersion, request.Mode, metric, row.StackId, utcStart.Ticks, utcEnd.Ticks, fingerprint); + var cursor = new StackRollupCursor(CursorVersion, sort.Value, metric, row.StackId, utcStart.Ticks, utcEnd.Ticks, fingerprint); byte[] bytes = JsonSerializer.SerializeToUtf8Bytes(cursor, _serializerOptions); return Convert.ToBase64String(bytes).TrimEnd('=').Replace('+', '-').Replace('/', '_'); } - private StackRollupCursor? DecodeCursor(string? token, string mode, string fingerprint) + private StackRollupCursor? DecodeCursor(string? token, string sort, string fingerprint) { if (String.IsNullOrWhiteSpace(token)) return null; @@ -388,7 +574,7 @@ private string EncodeCursor(StackRollupEsqlRow row, StackRollupSearchRequest req var cursor = JsonSerializer.Deserialize(Convert.FromBase64String(base64), _serializerOptions); if (cursor is null || cursor.Version != CursorVersion - || !String.Equals(cursor.Mode, mode, StringComparison.Ordinal) + || !String.Equals(cursor.Sort, sort, StringComparison.Ordinal) || !String.Equals(cursor.Fingerprint, fingerprint, StringComparison.Ordinal) || String.IsNullOrWhiteSpace(cursor.StackId) || cursor.UtcStart < DateTime.MinValue.Ticks @@ -396,7 +582,7 @@ private string EncodeCursor(StackRollupEsqlRow row, StackRollupSearchRequest req || cursor.UtcEnd < DateTime.MinValue.Ticks || cursor.UtcEnd > DateTime.MaxValue.Ticks || cursor.UtcStart > cursor.UtcEnd - || GetMode(mode).IsDate && (cursor.Metric < DateTime.MinValue.Ticks || cursor.Metric > DateTime.MaxValue.Ticks)) + || GetSort(sort).IsDate && (cursor.Metric < DateTime.MinValue.Ticks || cursor.Metric > DateTime.MaxValue.Ticks)) { throw new InvalidStackRollupCursorException("The stack pagination cursor is not valid for this query."); } @@ -418,7 +604,7 @@ private static string CreateFingerprint(StackRollupSearchRequest request) string organizations = String.Join(',', request.AppFilter?.Organizations.Select(organization => organization.Id).Order(StringComparer.Ordinal) ?? Enumerable.Empty()); string projects = String.Join(',', request.AppFilter?.Projects?.Select(project => project.Id).Order(StringComparer.Ordinal) ?? Enumerable.Empty()); string value = String.Join('\n', [ - request.Mode, + GetSort(request.Sort).Value, request.Filter ?? String.Empty, request.TimeExpression ?? String.Empty, request.Offset.Ticks.ToString(CultureInfo.InvariantCulture), @@ -436,25 +622,21 @@ private static string CreateFingerprint(StackRollupSearchRequest request) row.FirstOccurrence, row.LastOccurrence); - private static StackRollupMode GetMode(string mode) => mode switch + private static StackRollupSort GetSort(string? sort) => (String.IsNullOrWhiteSpace(sort) ? "-total" : sort.Trim()) switch { - "stack_recent" => new StackRollupMode("event_last", true), - "stack_frequent" => new StackRollupMode("event_total", false), - "stack_new" => new StackRollupMode("event_first", true), - "stack_users" => new StackRollupMode("event_users", false), - _ => throw new InvalidOperationException("Unsupported stack rollup mode.") + "total" => new StackRollupSort("total", "event_total", false, true), + "-total" => new StackRollupSort("-total", "event_total", false, false), + "users" => new StackRollupSort("users", "event_users", false, true), + "-users" => new StackRollupSort("-users", "event_users", false, false), + "first_occurrence" => new StackRollupSort("first_occurrence", "event_first", true, true), + "-first_occurrence" => new StackRollupSort("-first_occurrence", "event_first", true, false), + "last_occurrence" => new StackRollupSort("last_occurrence", "event_last", true, true), + "-last_occurrence" => new StackRollupSort("-last_occurrence", "event_last", true, false), + _ => throw new ArgumentOutOfRangeException(nameof(sort), sort, "Unsupported stack rollup sort.") }; - private static bool IsSupportedMode(string mode) => mode is "stack_recent" or "stack_frequent" or "stack_new" or "stack_users"; - private static string StripAlternateInversion(string? filter) => filter?.StartsWith("@!", StringComparison.Ordinal) == true ? filter[2..] : filter ?? String.Empty; - private static string AddFirstOccurrenceFilter(DateTime utcStart, DateTime utcEnd, string? filter) - { - string range = $"first_occurrence:[\"{utcStart:O}\" TO \"{utcEnd:O}\"]"; - return String.IsNullOrWhiteSpace(filter) ? range : $"{range} ({filter})"; - } - private static string ValidateIndexName(string index) { if (String.IsNullOrWhiteSpace(index) || index.Any(character => !Char.IsAsciiLetterOrDigit(character) && character is not '-' and not '_' and not '.')) @@ -465,12 +647,25 @@ private static string ValidateIndexName(string index) private static string EscapeIdentifier(string field) => $"`{field.Replace("`", "``", StringComparison.Ordinal)}`"; + private static string FormatTimeZone(TimeSpan offset) + { + if (offset < TimeSpan.FromHours(-14) || offset > TimeSpan.FromHours(14)) + throw new ArgumentOutOfRangeException(nameof(offset), offset, "The stack rollup time zone offset must be between -14:00 and +14:00."); + + string sign = offset < TimeSpan.Zero ? "-" : "+"; + var absolute = offset.Duration(); + return $"{sign}{(int)absolute.TotalHours:00}:{absolute.Minutes:00}"; + } + private static void AddParameter(ICollection>> parameters, string name, FieldValue value) => parameters.Add(new KeyValuePair>(name, [value])); - private sealed record StackRollupMode(string Metric, bool IsDate); + private static void AddParameter(ICollection>> parameters, string name, ICollection values) + => parameters.Add(new KeyValuePair>(name, values)); + + private sealed record StackRollupSort(string Value, string Metric, bool IsDate, bool Ascending); private sealed record StackRollupReadiness(bool IsReady, string Reason); - private sealed record StackRollupCursor(int Version, string Mode, long Metric, string StackId, long UtcStart, long UtcEnd, string Fingerprint); + private sealed record StackRollupCursor(int Version, string Sort, long Metric, string StackId, long UtcStart, long UtcEnd, string Fingerprint); private sealed record StackRollupEsqlRow( string StackId, long Total, diff --git a/src/Exceptionless.Web/Api/Endpoints/EventEndpoints.cs b/src/Exceptionless.Web/Api/Endpoints/EventEndpoints.cs index 05ad674605..e6b39bb57d 100644 --- a/src/Exceptionless.Web/Api/Endpoints/EventEndpoints.cs +++ b/src/Exceptionless.Web/Api/Endpoints/EventEndpoints.cs @@ -26,8 +26,8 @@ public static IEndpointRouteBuilder MapEventEndpoints(this IEndpointRouteBuilder .WithTags("Event"); // Count - group.MapGet("events/count", async (HttpContext httpContext, IMediator mediator, IMediatorResultMapper resultMapper, string? filter = null, string? aggregations = null, string? time = null, string? offset = null, string? mode = null) - => (await mediator.InvokeAsync>(new GetEventCount(filter, aggregations, time, offset, mode, httpContext))).ToHttpResult(resultMapper)) + group.MapGet("events/count", async (HttpContext httpContext, IMediator mediator, IMediatorResultMapper resultMapper, string? filter = null, string? aggregations = null, string? time = null, string? offset = null) + => (await mediator.InvokeAsync>(new GetEventCount(filter, aggregations, time, offset, httpContext))).ToHttpResult(resultMapper)) .RequireAuthorization(AuthorizationRoles.EventsReadPolicy) .Produces() .ProducesProblem(StatusCodes.Status400BadRequest) @@ -39,7 +39,6 @@ public static IEndpointRouteBuilder MapEventEndpoints(this IEndpointRouteBuilder ["aggregations"] = "A list of values you want returned. Example: avg:value cardinality:value sum:users max:value min:value", ["time"] = "The time filter that limits the data being returned to a specific date range.", ["offset"] = "The time offset in minutes that controls what data is returned based on the time filter. This is used for time zone support.", - ["mode"] = "If no mode is set then the whole event object will be returned. If the mode is set to summary than a lightweight object will be returned.", }, ResponseDescriptions = new() { ["400"] = "Invalid filter.", @@ -47,8 +46,8 @@ public static IEndpointRouteBuilder MapEventEndpoints(this IEndpointRouteBuilder } }); - group.MapGet("organizations/{organizationId:objectid}/events/count", async (string organizationId, HttpContext httpContext, IMediator mediator, IMediatorResultMapper resultMapper, string? filter = null, string? aggregations = null, string? time = null, string? offset = null, string? mode = null) - => (await mediator.InvokeAsync>(new GetEventCountByOrganization(organizationId, filter, aggregations, time, offset, mode, httpContext))).ToHttpResult(resultMapper)) + group.MapGet("organizations/{organizationId:objectid}/events/count", async (string organizationId, HttpContext httpContext, IMediator mediator, IMediatorResultMapper resultMapper, string? filter = null, string? aggregations = null, string? time = null, string? offset = null) + => (await mediator.InvokeAsync>(new GetEventCountByOrganization(organizationId, filter, aggregations, time, offset, httpContext))).ToHttpResult(resultMapper)) .RequireAuthorization(AuthorizationRoles.EventsReadPolicy) .Produces() .ProducesProblem(StatusCodes.Status400BadRequest) @@ -61,7 +60,6 @@ public static IEndpointRouteBuilder MapEventEndpoints(this IEndpointRouteBuilder ["aggregations"] = "A list of values you want returned. Example: avg:value cardinality:value sum:users max:value min:value", ["time"] = "The time filter that limits the data being returned to a specific date range.", ["offset"] = "The time offset in minutes that controls what data is returned based on the time filter. This is used for time zone support.", - ["mode"] = "If no mode is set then the whole event object will be returned. If the mode is set to summary than a lightweight object will be returned.", }, ResponseDescriptions = new() { ["400"] = "Invalid filter.", @@ -69,8 +67,8 @@ public static IEndpointRouteBuilder MapEventEndpoints(this IEndpointRouteBuilder } }); - group.MapGet("projects/{projectId:objectid}/events/count", async (string projectId, HttpContext httpContext, IMediator mediator, IMediatorResultMapper resultMapper, string? filter = null, string? aggregations = null, string? time = null, string? offset = null, string? mode = null) - => (await mediator.InvokeAsync>(new GetEventCountByProject(projectId, filter, aggregations, time, offset, mode, httpContext))).ToHttpResult(resultMapper)) + group.MapGet("projects/{projectId:objectid}/events/count", async (string projectId, HttpContext httpContext, IMediator mediator, IMediatorResultMapper resultMapper, string? filter = null, string? aggregations = null, string? time = null, string? offset = null) + => (await mediator.InvokeAsync>(new GetEventCountByProject(projectId, filter, aggregations, time, offset, httpContext))).ToHttpResult(resultMapper)) .RequireAuthorization(AuthorizationRoles.EventsReadPolicy) .Produces() .ProducesProblem(StatusCodes.Status400BadRequest) @@ -83,7 +81,6 @@ public static IEndpointRouteBuilder MapEventEndpoints(this IEndpointRouteBuilder ["aggregations"] = "A list of values you want returned. Example: avg:value cardinality:value sum:users max:value min:value", ["time"] = "The time filter that limits the data being returned to a specific date range.", ["offset"] = "The time offset in minutes that controls what data is returned based on the time filter. This is used for time zone support.", - ["mode"] = "If mode is set to stack_new, then additional filters will be added.", }, ResponseDescriptions = new() { ["400"] = "Invalid filter.", diff --git a/src/Exceptionless.Web/Api/Endpoints/StackEndpoints.cs b/src/Exceptionless.Web/Api/Endpoints/StackEndpoints.cs index c88d1e757a..6e3a4b20bb 100644 --- a/src/Exceptionless.Web/Api/Endpoints/StackEndpoints.cs +++ b/src/Exceptionless.Web/Api/Endpoints/StackEndpoints.cs @@ -2,6 +2,7 @@ using Exceptionless.Core.Authorization; using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; +using Exceptionless.Core.Services; using Exceptionless.Web.Api.Filters; using Exceptionless.Web.Api.Infrastructure; using Exceptionless.Web.Api.Messages; @@ -248,8 +249,8 @@ public static IEndpointRouteBuilder MapStackEndpoints(this IEndpointRouteBuilder }); // Get all - group.MapGet("stacks", async (HttpContext httpContext, IMediator mediator, IMediatorResultMapper resultMapper, string? filter = null, string? sort = null, string? time = null, string? offset = null, string? mode = null, int page = 1, int limit = 10) - => (await mediator.InvokeAsync>>(new GetAllStacks(filter, sort, time, offset, mode, page, limit, httpContext))).ToHttpResult(resultMapper)) + group.MapGet("stacks", async (HttpContext httpContext, IMediator mediator, IMediatorResultMapper resultMapper, string? filter = null, string? sort = null, string? time = null, string? offset = null, int limit = 10, string? before = null, string? after = null) + => (await mediator.InvokeAsync>>(new GetAllStacks(filter, sort, time, offset, limit, before, after, httpContext))).ToHttpResult(resultMapper)) .RequireAuthorization(AuthorizationRoles.StacksReadPolicy) .Produces>() .ProducesProblem(StatusCodes.Status400BadRequest) @@ -261,9 +262,9 @@ public static IEndpointRouteBuilder MapStackEndpoints(this IEndpointRouteBuilder ["sort"] = "Controls the sort order that the data is returned in. In this example -date returns the results descending by date.", ["time"] = "The time filter that limits the data being returned to a specific date range.", ["offset"] = "The time offset in minutes that controls what data is returned based on the time filter. This is used for time zone support.", - ["mode"] = "If no mode is set then the whole stack object will be returned. If the mode is set to summary than a lightweight object will be returned.", - ["page"] = "The page parameter is used for pagination. This value must be greater than 0.", ["limit"] = "A limit on the number of objects to be returned. Limit can range between 1 and 100 items.", + ["before"] = "A cursor that returns the previous page for this exact filter and sort.", + ["after"] = "A cursor that returns the next page for this exact filter and sort.", }, ResponseDescriptions = new() { ["400"] = "Invalid filter.", @@ -272,8 +273,8 @@ public static IEndpointRouteBuilder MapStackEndpoints(this IEndpointRouteBuilder }); // Get by organization - group.MapGet("organizations/{organizationId:objectid}/stacks", async (string organizationId, HttpContext httpContext, IMediator mediator, IMediatorResultMapper resultMapper, string? filter = null, string? sort = null, string? time = null, string? offset = null, string? mode = null, int page = 1, int limit = 10) - => (await mediator.InvokeAsync>>(new GetStacksByOrganization(organizationId, filter, sort, time, offset, mode, page, limit, httpContext))).ToHttpResult(resultMapper)) + group.MapGet("organizations/{organizationId:objectid}/stacks", async (string organizationId, HttpContext httpContext, IMediator mediator, IMediatorResultMapper resultMapper, string? filter = null, string? sort = null, string? time = null, string? offset = null, int limit = 10, string? before = null, string? after = null) + => (await mediator.InvokeAsync>>(new GetStacksByOrganization(organizationId, filter, sort, time, offset, limit, before, after, httpContext))).ToHttpResult(resultMapper)) .RequireAuthorization(AuthorizationRoles.StacksReadPolicy) .Produces>() .ProducesProblem(StatusCodes.Status400BadRequest) @@ -287,9 +288,9 @@ public static IEndpointRouteBuilder MapStackEndpoints(this IEndpointRouteBuilder ["sort"] = "Controls the sort order that the data is returned in. In this example -date returns the results descending by date.", ["time"] = "The time filter that limits the data being returned to a specific date range.", ["offset"] = "The time offset in minutes that controls what data is returned based on the time filter. This is used for time zone support.", - ["mode"] = "If no mode is set then the whole stack object will be returned. If the mode is set to summary than a lightweight object will be returned.", - ["page"] = "The page parameter is used for pagination. This value must be greater than 0.", ["limit"] = "A limit on the number of objects to be returned. Limit can range between 1 and 100 items.", + ["before"] = "A cursor that returns the previous page for this exact filter and sort.", + ["after"] = "A cursor that returns the next page for this exact filter and sort.", }, ResponseDescriptions = new() { ["400"] = "Invalid filter.", @@ -299,8 +300,8 @@ public static IEndpointRouteBuilder MapStackEndpoints(this IEndpointRouteBuilder }); // Get by project - group.MapGet("projects/{projectId:objectid}/stacks", async (string projectId, HttpContext httpContext, IMediator mediator, IMediatorResultMapper resultMapper, string? filter = null, string? sort = null, string? time = null, string? offset = null, string? mode = null, int page = 1, int limit = 10) - => (await mediator.InvokeAsync>>(new GetStacksByProject(projectId, filter, sort, time, offset, mode, page, limit, httpContext))).ToHttpResult(resultMapper)) + group.MapGet("projects/{projectId:objectid}/stacks", async (string projectId, HttpContext httpContext, IMediator mediator, IMediatorResultMapper resultMapper, string? filter = null, string? sort = null, string? time = null, string? offset = null, int limit = 10, string? before = null, string? after = null) + => (await mediator.InvokeAsync>>(new GetStacksByProject(projectId, filter, sort, time, offset, limit, before, after, httpContext))).ToHttpResult(resultMapper)) .RequireAuthorization(AuthorizationRoles.StacksReadPolicy) .Produces>() .ProducesProblem(StatusCodes.Status400BadRequest) @@ -314,9 +315,9 @@ public static IEndpointRouteBuilder MapStackEndpoints(this IEndpointRouteBuilder ["sort"] = "Controls the sort order that the data is returned in. In this example -date returns the results descending by date.", ["time"] = "The time filter that limits the data being returned to a specific date range.", ["offset"] = "The time offset in minutes that controls what data is returned based on the time filter. This is used for time zone support.", - ["mode"] = "If no mode is set then the whole stack object will be returned. If the mode is set to summary than a lightweight object will be returned.", - ["page"] = "The page parameter is used for pagination. This value must be greater than 0.", ["limit"] = "A limit on the number of objects to be returned. Limit can range between 1 and 100 items.", + ["before"] = "A cursor that returns the previous page for this exact filter and sort.", + ["after"] = "A cursor that returns the next page for this exact filter and sort.", }, ResponseDescriptions = new() { ["400"] = "Invalid filter.", @@ -325,6 +326,62 @@ public static IEndpointRouteBuilder MapStackEndpoints(this IEndpointRouteBuilder } }); + MapStackRollupEndpoints(group); return endpoints; } + + private static void MapStackRollupEndpoints(RouteGroupBuilder group) + { + group.MapGet("stack-rollups/stats", async (HttpContext httpContext, IMediator mediator, IMediatorResultMapper resultMapper, string? filter = null, string? time = null, string? offset = null) + => (await mediator.InvokeAsync>(new GetStackRollupStats(filter, time, offset, httpContext))).ToHttpResult(resultMapper)) + .RequireAuthorization(AuthorizationRoles.StacksReadPolicy) + .Produces() + .ProducesProblem(StatusCodes.Status400BadRequest) + .ProducesProblem(StatusCodes.Status426UpgradeRequired) + .WithSummary("Get stack rollup statistics"); + + group.MapGet("organizations/{organizationId:objectid}/stack-rollups/stats", async (string organizationId, HttpContext httpContext, IMediator mediator, IMediatorResultMapper resultMapper, string? filter = null, string? time = null, string? offset = null) + => (await mediator.InvokeAsync>(new GetStackRollupStatsByOrganization(organizationId, filter, time, offset, httpContext))).ToHttpResult(resultMapper)) + .RequireAuthorization(AuthorizationRoles.StacksReadPolicy) + .Produces() + .ProducesProblem(StatusCodes.Status400BadRequest) + .ProducesProblem(StatusCodes.Status404NotFound) + .ProducesProblem(StatusCodes.Status426UpgradeRequired) + .WithSummary("Get stack rollup statistics by organization"); + + group.MapGet("projects/{projectId:objectid}/stack-rollups/stats", async (string projectId, HttpContext httpContext, IMediator mediator, IMediatorResultMapper resultMapper, string? filter = null, string? time = null, string? offset = null) + => (await mediator.InvokeAsync>(new GetStackRollupStatsByProject(projectId, filter, time, offset, httpContext))).ToHttpResult(resultMapper)) + .RequireAuthorization(AuthorizationRoles.StacksReadPolicy) + .Produces() + .ProducesProblem(StatusCodes.Status400BadRequest) + .ProducesProblem(StatusCodes.Status404NotFound) + .ProducesProblem(StatusCodes.Status426UpgradeRequired) + .WithSummary("Get stack rollup statistics by project"); + + group.MapGet("stack-rollups", async (HttpContext httpContext, IMediator mediator, IMediatorResultMapper resultMapper, string? filter = null, string? sort = null, string? time = null, string? offset = null, int limit = 10, string? before = null, string? after = null, string? include = null) + => (await mediator.InvokeAsync>>(new GetAllStackRollups(filter, sort, time, offset, limit, before, after, include, httpContext))).ToHttpResult(resultMapper)) + .RequireAuthorization(AuthorizationRoles.StacksReadPolicy) + .Produces>() + .ProducesProblem(StatusCodes.Status400BadRequest) + .ProducesProblem(StatusCodes.Status426UpgradeRequired) + .WithSummary("Get stack rollups"); + + group.MapGet("organizations/{organizationId:objectid}/stack-rollups", async (string organizationId, HttpContext httpContext, IMediator mediator, IMediatorResultMapper resultMapper, string? filter = null, string? sort = null, string? time = null, string? offset = null, int limit = 10, string? before = null, string? after = null, string? include = null) + => (await mediator.InvokeAsync>>(new GetStackRollupsByOrganization(organizationId, filter, sort, time, offset, limit, before, after, include, httpContext))).ToHttpResult(resultMapper)) + .RequireAuthorization(AuthorizationRoles.StacksReadPolicy) + .Produces>() + .ProducesProblem(StatusCodes.Status400BadRequest) + .ProducesProblem(StatusCodes.Status404NotFound) + .ProducesProblem(StatusCodes.Status426UpgradeRequired) + .WithSummary("Get stack rollups by organization"); + + group.MapGet("projects/{projectId:objectid}/stack-rollups", async (string projectId, HttpContext httpContext, IMediator mediator, IMediatorResultMapper resultMapper, string? filter = null, string? sort = null, string? time = null, string? offset = null, int limit = 10, string? before = null, string? after = null, string? include = null) + => (await mediator.InvokeAsync>>(new GetStackRollupsByProject(projectId, filter, sort, time, offset, limit, before, after, include, httpContext))).ToHttpResult(resultMapper)) + .RequireAuthorization(AuthorizationRoles.StacksReadPolicy) + .Produces>() + .ProducesProblem(StatusCodes.Status400BadRequest) + .ProducesProblem(StatusCodes.Status404NotFound) + .ProducesProblem(StatusCodes.Status426UpgradeRequired) + .WithSummary("Get stack rollups by project"); + } } diff --git a/src/Exceptionless.Web/Api/Handlers/EventHandler.cs b/src/Exceptionless.Web/Api/Handlers/EventHandler.cs index 77a270531f..1a8bd7099b 100644 --- a/src/Exceptionless.Web/Api/Handlers/EventHandler.cs +++ b/src/Exceptionless.Web/Api/Handlers/EventHandler.cs @@ -45,10 +45,8 @@ public class EventHandler( ICacheClient cacheClient, ITextSerializer serializer, PersistentEventQueryValidator validator, - EventStackQueryValidator stackModeValidator, AppOptions appOptions, UsageService usageService, - IStackRollupSearchService stackRollupSearchService, TimeProvider timeProvider, LinkGenerator linkGenerator, ILoggerFactory loggerFactory) @@ -79,7 +77,7 @@ public async Task> Handle(GetEventCount message) var ti = TimeRangeParser.GetTimeInfo(message.Time, message.Offset, timeProvider, _allowedDateFields, DefaultDateField, organizations.GetRetentionUtcCutoff(appOptions.MaximumRetentionDays, timeProvider)); var sf = new AppFilter(organizations) { IsUserOrganizationsFilter = true }; - return await CountInternalAsync(sf, ti, httpContext, message.Filter, message.Aggregations, message.Mode); + return await CountInternalAsync(sf, ti, httpContext, message.Filter, message.Aggregations); } public async Task> Handle(GetEventCountByOrganization message) @@ -94,7 +92,7 @@ public async Task> Handle(GetEventCountByOrganization messag var ti = TimeRangeParser.GetTimeInfo(message.Time, message.Offset, timeProvider, _allowedDateFields, DefaultDateField, organization.GetRetentionUtcCutoff(appOptions.MaximumRetentionDays, timeProvider)); var sf = new AppFilter(organization); - return await CountInternalAsync(sf, ti, httpContext, message.Filter, message.Aggregations, message.Mode); + return await CountInternalAsync(sf, ti, httpContext, message.Filter, message.Aggregations); } public async Task> Handle(GetEventCountByProject message) @@ -113,7 +111,7 @@ public async Task> Handle(GetEventCountByProject message) var ti = TimeRangeParser.GetTimeInfo(message.Time, message.Offset, timeProvider, _allowedDateFields, DefaultDateField, organization.GetRetentionUtcCutoff(project, appOptions.MaximumRetentionDays, timeProvider)); var sf = new AppFilter(project, organization); - return await CountInternalAsync(sf, ti, httpContext, message.Filter, message.Aggregations, message.Mode); + return await CountInternalAsync(sf, ti, httpContext, message.Filter, message.Aggregations); } public async Task> Handle(GetEventById message) @@ -689,9 +687,9 @@ public async Task> Handle(DeleteEvents message) #region Private Helpers - private async Task> CountInternalAsync(AppFilter sf, TimeInfo ti, HttpContext httpContext, string? filter = null, string? aggregations = null, string? mode = null) + private async Task> CountInternalAsync(AppFilter sf, TimeInfo ti, HttpContext httpContext, string? filter = null, string? aggregations = null) { - var pr = await GetQueryValidator(mode).ValidateQueryAsync(filter); + var pr = await validator.ValidateQueryAsync(filter); if (!pr.IsValid) return Result.BadRequest(pr.Message ?? "Invalid filter."); @@ -704,9 +702,6 @@ private async Task> CountInternalAsync(AppFilter sf, TimeInf if (systemFilter is not null && ApiFilterPolicy.IsPremiumFeatureQueryBlocked(systemFilter)) return PlanLimitResult(ApiFilterPolicy.PremiumSearchUpgradeMessage); - if (mode == "stack_new") - filter = AddFirstOccurrenceFilter(ti.Range, filter); - var query = new RepositoryQuery() .AppFilter(systemFilter) .DateRange(ti.Range.UtcStart, ti.Range.UtcEnd, ti.Field) @@ -731,6 +726,9 @@ private async Task> CountInternalAsync(AppFilter sf, TimeInf private async Task>> GetInternalAsync(AppFilter sf, TimeInfo ti, HttpContext httpContext, string? filter = null, string? sort = null, string? mode = null, int? page = null, int limit = 10, string? before = null, string? after = null, string? premiumFeatureUpgradeMessage = null, bool includeTotal = false, string? timeExpression = null) { + if (mode is not null && !String.Equals(mode, "summary", StringComparison.OrdinalIgnoreCase)) + return Result.BadRequest("Mode must be 'summary' when specified."); + var currentUser = httpContext.Request.GetUser(); using var _ = _logger.BeginScope(new ExceptionlessState() .Property("Search Filter", new @@ -750,20 +748,13 @@ private async Task>> GetInternalAsync(AppFilter sf, T .SetHttpContext(httpContext) ); - bool isStackMode = IsStackMode(mode); - if (isStackMode && before is not null && after is not null) - return Result.BadRequest("The before and after parameters cannot be used together."); - - if (isStackMode && page.HasValue) - return Result.BadRequest("The page parameter is not supported in stack mode. Use before or after cursor pagination."); - int resolvedPage = Pagination.GetPage(page.GetValueOrDefault(1)); limit = Pagination.GetLimit(limit); int skip = Pagination.GetSkip(resolvedPage, limit); if (skip > Pagination.MaximumSkip) return new PagedResult(Array.Empty(), false); - var pr = await GetQueryValidator(mode).ValidateQueryAsync(filter); + var pr = await validator.ValidateQueryAsync(filter); if (!pr.IsValid) return Result.BadRequest(pr.Message ?? "Invalid filter."); @@ -798,48 +789,11 @@ private async Task>> GetInternalAsync(AppFilter sf, T }; }).ToList(); return new PagedResult(summaries.Cast().ToList(), events.HasMore && !Pagination.NextPageExceedsSkipLimit(page, limit), page, includeTotal ? events.Total : null, events.Hits.FirstOrDefault()?.GetSortToken(serializer), events.Hits.LastOrDefault()?.GetSortToken(serializer)); - case "stack_recent": - case "stack_frequent": - case "stack_new": - case "stack_users": - if (!String.IsNullOrEmpty(sort)) - return Result.BadRequest("Sort is not supported in stack mode."); - - var lookupResult = await stackRollupSearchService.SearchAsync(new StackRollupSearchRequest( - appliedAppFilter, - ti.Range.UtcStart, - ti.Range.UtcEnd, - ti.Offset, - timeExpression, - filter, - mode, - limit, - before, - after, - includeTotal), httpContext.RequestAborted); - - string[] lookupStackIds = lookupResult.Rows.Select(row => row.StackId).ToArray(); - var lookupStacks = (await stackRepository.GetByIdsAsync(lookupStackIds)) - .Select(stack => stack.ApplyOffset(ti.Offset)) - .ToList(); - var lookupSummaries = await GetStackSummariesAsync(lookupStacks, lookupResult.Rows, sf, ti); - - return new PagedResult( - lookupSummaries.Cast().ToList(), - lookupResult.HasMore, - Page: null, - lookupResult.Total, - lookupResult.Before, - lookupResult.After); default: events = await GetEventsInternalAsync(appliedAppFilter, ti, filter, sort, page, limit, before, after, includeTotal); return new PagedResult(events.Documents.Cast().ToList(), events.HasMore && !Pagination.NextPageExceedsSkipLimit(page, limit), page, includeTotal ? events.Total : null, events.Hits.FirstOrDefault()?.GetSortToken(serializer), events.Hits.LastOrDefault()?.GetSortToken(serializer)); } } - catch (InvalidStackRollupCursorException ex) - { - return Result.BadRequest(ex.Message); - } catch (ApplicationException ex) { string message = "An error has occurred: Please check your search filter."; @@ -851,50 +805,6 @@ private async Task>> GetInternalAsync(AppFilter sf, T } } - private static string AddFirstOccurrenceFilter(DateTimeRange timeRange, string? filter) - { - bool inverted = false; - if (filter is not null && filter.StartsWith("@!")) - { - inverted = true; - filter = filter.Substring(2); - } - - var sb = new StringBuilder(); - if (inverted) - sb.Append("@!"); - - sb.Append("first_occurrence:[\""); - sb.Append(timeRange.UtcStart.ToString("O")); - sb.Append("\" TO \""); - sb.Append(timeRange.UtcEnd.ToString("O")); - sb.Append("\"]"); - - if (String.IsNullOrEmpty(filter)) - return sb.ToString(); - - sb.Append(' '); - - bool isGrouped = filter.StartsWith('(') && filter.EndsWith(')'); - - if (isGrouped) - sb.Append(filter); - else - sb.Append('(').Append(filter).Append(')'); - - return sb.ToString(); - } - - private static bool IsStackMode(string? mode) - { - return mode is "stack_recent" or "stack_frequent" or "stack_new" or "stack_users"; - } - - private IAppQueryValidator GetQueryValidator(string? mode) - { - return IsStackMode(mode) ? stackModeValidator : validator; - } - private Task> GetEventsInternalAsync(AppFilter? systemFilter, TimeInfo ti, string? filter, string? sort, int? page, int limit, string? before, string? after, bool includeTotal) { if (String.IsNullOrEmpty(sort)) @@ -912,69 +822,6 @@ private Task> GetEventsInternalAsync(AppFilter? sys : o.SearchBeforeToken(before, serializer).SearchAfterToken(after, serializer).PageLimit(limit).TrackTotalHits(includeTotal)); } - private async Task> GetStackSummariesAsync(List stacks, IReadOnlyCollection rows, AppFilter sf, TimeInfo ti) - { - if (stacks.Count == 0) - return []; - - var stacksById = stacks.ToDictionary(stack => stack.Id, StringComparer.Ordinal); - var projects = await projectRepository.GetByIdsAsync(stacks.Select(stack => stack.ProjectId).Distinct().ToArray(), options => options.Cache()); - var projectNames = projects.ToDictionary(project => project.Id, project => project.Name); - var totalUsers = await GetUserCountByProjectIdsAsync(stacks, sf, ti.Range.UtcStart, ti.Range.UtcEnd); - var summaries = new List(rows.Count); - - foreach (var row in rows) - { - if (!stacksById.TryGetValue(row.StackId, out var stack)) - continue; - - var data = formattingPluginManager.GetStackSummaryData(stack); - summaries.Add(new StackSummaryModel - { - Id = data.Id, - TemplateKey = data.TemplateKey, - Data = data.Data, - ProjectId = stack.ProjectId, - ProjectName = projectNames.GetValueOrDefault(stack.ProjectId), - Tags = stack.Tags?.OfType().Order(StringComparer.OrdinalIgnoreCase).ToArray() ?? [], - Title = stack.Title, - Status = stack.Status, - FirstOccurrence = row.FirstOccurrence, - LastOccurrence = row.LastOccurrence, - Total = row.Total, - Users = row.Users, - TotalUsers = totalUsers.GetOrDefault(stack.ProjectId) - }); - } - - return summaries; - } - - private async Task> GetUserCountByProjectIdsAsync(ICollection stacks, AppFilter sf, DateTime utcStart, DateTime utcEnd) - { - using var scopedCacheClient = new ScopedCacheClient(cacheClient, $"Project:user-count:{utcStart.Floor(TimeSpan.FromMinutes(15)).Ticks}-{utcEnd.Floor(TimeSpan.FromMinutes(15)).Ticks}"); - var projectIds = stacks.Select(s => s.ProjectId).Distinct().ToList(); - var cachedTotals = await scopedCacheClient.GetAllAsync(projectIds); - - var totals = cachedTotals.Where(kvp => kvp.Value.HasValue).ToDictionary(kvp => kvp.Key, kvp => kvp.Value.Value); - if (totals.Count == projectIds.Count) - return totals; - - var systemFilter = new RepositoryQuery().AppFilter(sf).DateRange(utcStart, utcEnd, (PersistentEvent e) => e.Date).Index(utcStart, utcEnd); - var projects = cachedTotals - .Where(kvp => !kvp.Value.HasValue && stacks.Contains(s => s.ProjectId == kvp.Key)) - .Select(kvp => new Project { Id = kvp.Key, OrganizationId = stacks.First(s => s.ProjectId == kvp.Key).OrganizationId }) - .ToList(); - var countResult = await eventRepository.CountAsync(q => q.SystemFilter(systemFilter).FilterExpression(projects.BuildFilter()).EnforceEventStackFilter().AggregationsExpression("terms:(project_id cardinality:user)")); - - var projectTerms = countResult.Aggregations.Terms("terms_project_id")?.Buckets ?? []; - var aggregations = projectTerms.ToDictionary(t => t.Key, t => t.Aggregations.Cardinality("cardinality_user")?.Value.GetValueOrDefault() ?? 0); - await scopedCacheClient.SetAllAsync(aggregations.Where(t => t.Value >= 10).ToDictionary(k => k.Key, v => v.Value), TimeSpan.FromMinutes(5)); - totals.AddRange(aggregations); - - return totals; - } - private async Task GetModelAsync(string id, HttpContext httpContext, bool useCache = true) { if (String.IsNullOrEmpty(id)) diff --git a/src/Exceptionless.Web/Api/Handlers/StackHandler.cs b/src/Exceptionless.Web/Api/Handlers/StackHandler.cs index 0ac1975ef6..0fb87dff40 100644 --- a/src/Exceptionless.Web/Api/Handlers/StackHandler.cs +++ b/src/Exceptionless.Web/Api/Handlers/StackHandler.cs @@ -22,8 +22,10 @@ using Foundatio.Mediator; using Foundatio.Queues; using Foundatio.Repositories; +using Foundatio.Repositories.Elasticsearch.Extensions; using Foundatio.Repositories.Extensions; using Foundatio.Repositories.Models; +using Foundatio.Serializer; using McSherry.SemanticVersioning; namespace Exceptionless.Web.Api.Handlers; @@ -32,14 +34,12 @@ public class StackHandler( IStackRepository stackRepository, IOrganizationRepository organizationRepository, IProjectRepository projectRepository, - IEventRepository eventRepository, IWebHookRepository webHookRepository, WebHookDataPluginManager webHookDataPluginManager, IQueue webHookNotificationQueue, - ICacheClient cacheClient, - FormattingPluginManager formattingPluginManager, SemanticVersionParser semanticVersionParser, StackQueryValidator validator, + ITextSerializer serializer, AppOptions options, TimeProvider timeProvider, ILoggerFactory loggerFactory) @@ -358,7 +358,7 @@ public async Task>> Handle(GetAllStacks message) var ti = TimeRangeParser.GetTimeInfo(message.Time, message.Offset, timeProvider, _allowedDateFields, DefaultDateField, organizations.GetRetentionUtcCutoff(options.MaximumRetentionDays, timeProvider)); var sf = new AppFilter(organizations) { IsUserOrganizationsFilter = true }; - return await GetInternalAsync(sf, ti, httpContext, message.Filter, message.Sort, message.Mode, message.Page, message.Limit); + return await GetInternalAsync(sf, ti, httpContext, message.Filter, message.Sort, message.Limit, message.Before, message.After); } public async Task>> Handle(GetStacksByOrganization message) @@ -373,7 +373,7 @@ public async Task>> Handle(GetStacksByOrganization me var ti = TimeRangeParser.GetTimeInfo(message.Time, message.Offset, timeProvider, _allowedDateFields, DefaultDateField, organization.GetRetentionUtcCutoff(options.MaximumRetentionDays, timeProvider)); var sf = new AppFilter(organization); - return await GetInternalAsync(sf, ti, httpContext, message.Filter, message.Sort, message.Mode, message.Page, message.Limit); + return await GetInternalAsync(sf, ti, httpContext, message.Filter, message.Sort, message.Limit, message.Before, message.After); } public async Task>> Handle(GetStacksByProject message) @@ -392,16 +392,16 @@ public async Task>> Handle(GetStacksByProject message var ti = TimeRangeParser.GetTimeInfo(message.Time, message.Offset, timeProvider, _allowedDateFields, DefaultDateField, organization.GetRetentionUtcCutoff(project, options.MaximumRetentionDays, timeProvider)); var sf = new AppFilter(project, organization); - return await GetInternalAsync(sf, ti, httpContext, message.Filter, message.Sort, message.Mode, message.Page, message.Limit); + return await GetInternalAsync(sf, ti, httpContext, message.Filter, message.Sort, message.Limit, message.Before, message.After); } - private async Task>> GetInternalAsync(AppFilter sf, TimeInfo ti, HttpContext httpContext, string? filter = null, string? sort = null, string? mode = null, int page = 1, int limit = 10) + private async Task>> GetInternalAsync(AppFilter sf, TimeInfo ti, HttpContext httpContext, string? filter = null, string? sort = null, int limit = 10, string? before = null, string? after = null) { - page = Pagination.GetPage(page); + if (before is not null && after is not null) + return Result.BadRequest("The before and after parameters cannot be used together."); + limit = Pagination.GetLimit(limit); - int skip = Pagination.GetSkip(page, limit); - if (skip > Pagination.MaximumSkip) - return new PagedResult(Array.Empty(), false); + sort = String.IsNullOrWhiteSpace(sort) ? "-last" : sort; var pr = await validator.ValidateQueryAsync(filter); if (!pr.IsValid) @@ -414,93 +414,29 @@ private async Task>> GetInternalAsync(AppFilter sf, T try { - var results = await stackRepository.FindAsync(q => q.AppFilter(systemFilter).FilterExpression(filter).SortExpression(sort).DateRange(ti.Range.UtcStart, ti.Range.UtcEnd, ti.Field), o => o.PageNumber(page).PageLimit(limit)); + var results = await stackRepository.FindAsync( + q => q.AppFilter(systemFilter).FilterExpression(filter).SortExpression(sort).DateRange(ti.Range.UtcStart, ti.Range.UtcEnd, ti.Field), + o => o.SearchBeforeToken(before, serializer).SearchAfterToken(after, serializer).PageLimit(limit)); var stacks = results.Documents.Select(s => s.ApplyOffset(ti.Offset)).ToList(); - if (!String.IsNullOrEmpty(mode) && String.Equals(mode, "summary", StringComparison.OrdinalIgnoreCase)) - return new PagedResult((await GetStackSummariesAsync(stacks, sf, ti)).Cast().ToList(), results.HasMore && !Pagination.NextPageExceedsSkipLimit(page, limit), page); - - return new PagedResult(stacks.Cast().ToList(), results.HasMore && !Pagination.NextPageExceedsSkipLimit(page, limit), page); + return new PagedResult( + stacks.Cast().ToList(), + results.HasMore, + Page: null, + Total: null, + results.Hits.FirstOrDefault()?.GetSortToken(serializer), + results.Hits.LastOrDefault()?.GetSortToken(serializer)); } catch (ApplicationException ex) { var currentUser = httpContext.Request.GetUser(); - using (_logger.BeginScope(new ExceptionlessState().Property("Search Filter", new { SystemFilter = sf, UserFilter = filter, Time = ti, Page = page, Limit = limit }).Tag("Search").Identity(currentUser?.EmailAddress).Property("User", currentUser).SetHttpContext(httpContext))) + using (_logger.BeginScope(new ExceptionlessState().Property("Search Filter", new { SystemFilter = sf, UserFilter = filter, Time = ti, Sort = sort, Before = before, After = after, Limit = limit }).Tag("Search").Identity(currentUser?.EmailAddress).Property("User", currentUser).SetHttpContext(httpContext))) _logger.LogError(ex, "An error has occurred. Please check your search filter"); throw; } } - private async Task> GetStackSummariesAsync(ICollection stacks, AppFilter eventSystemFilter, TimeInfo ti) - { - if (stacks.Count == 0) - return new List(); - - var systemFilter = new RepositoryQuery().AppFilter(eventSystemFilter).DateRange(ti.Range.UtcStart, ti.Range.UtcEnd, (PersistentEvent e) => e.Date).Index(ti.Range.UtcStart, ti.Range.UtcEnd); - var stackTerms = await eventRepository.CountAsync(q => q.SystemFilter(systemFilter).Stack(stacks.Select(r => r.Id)).AggregationsExpression($"terms:(stack_id~{stacks.Count} cardinality:user sum:count~1 min:date max:date)")); - var buckets = stackTerms.Aggregations.Terms("terms_stack_id")?.Buckets ?? []; - return await GetStackSummariesAsync(stacks, buckets, eventSystemFilter, ti); - } - - private async Task> GetStackSummariesAsync(ICollection stacks, IReadOnlyCollection> stackTerms, AppFilter sf, TimeInfo ti) - { - if (stacks.Count == 0) - return new List(0); - - var projects = await projectRepository.GetByIdsAsync(stacks.Select(s => s.ProjectId).Distinct().ToArray(), o => o.Cache()); - var projectNames = projects.ToDictionary(p => p.Id, p => p.Name); - var totalUsers = await GetUserCountByProjectIdsAsync(stacks, sf, ti.Range.UtcStart, ti.Range.UtcEnd); - return stacks.Join(stackTerms, s => s.Id, tk => tk.Key, (stack, term) => - { - var data = formattingPluginManager.GetStackSummaryData(stack); - var summary = new StackSummaryModel - { - Id = data.Id, - TemplateKey = data.TemplateKey, - Data = data.Data, - ProjectId = stack.ProjectId, - ProjectName = projectNames.GetValueOrDefault(stack.ProjectId), - Tags = stack.Tags?.OfType().Order(StringComparer.OrdinalIgnoreCase).ToArray() ?? [], - Title = stack.Title, - Status = stack.Status, - FirstOccurrence = term.Aggregations.Min("min_date")?.Value ?? stack.FirstOccurrence, - LastOccurrence = term.Aggregations.Max("max_date")?.Value ?? stack.LastOccurrence, - Total = (long)(term.Aggregations.Sum("sum_count")?.Value ?? term.Total.GetValueOrDefault()), - - Users = term.Aggregations.Cardinality("cardinality_user")?.Value.GetValueOrDefault() ?? 0, - TotalUsers = totalUsers.GetOrDefault(stack.ProjectId) - }; - - return summary; - }).ToList(); - } - - private async Task> GetUserCountByProjectIdsAsync(ICollection stacks, AppFilter sf, DateTime utcStart, DateTime utcEnd) - { - using var scopedCacheClient = new ScopedCacheClient(cacheClient, $"Project:user-count:{utcStart.Floor(TimeSpan.FromMinutes(15)).Ticks}-{utcEnd.Floor(TimeSpan.FromMinutes(15)).Ticks}"); - var projectIds = stacks.Select(s => s.ProjectId).Distinct().ToList(); - var cachedTotals = await scopedCacheClient.GetAllAsync(projectIds); - - var totals = cachedTotals.Where(kvp => kvp.Value.HasValue).ToDictionary(kvp => kvp.Key, kvp => kvp.Value.Value); - if (totals.Count == projectIds.Count) - return totals; - - var systemFilter = new RepositoryQuery().AppFilter(sf).DateRange(utcStart, utcEnd, (PersistentEvent e) => e.Date).Index(utcStart, utcEnd); - var projects = cachedTotals - .Where(kvp => !kvp.Value.HasValue && stacks.Contains(s => s.ProjectId == kvp.Key)) - .Select(kvp => new Project { Id = kvp.Key, OrganizationId = stacks.First(s => s.ProjectId == kvp.Key).OrganizationId }) - .ToList(); - var countResult = await eventRepository.CountAsync(q => q.SystemFilter(systemFilter).FilterExpression(projects.BuildFilter()).AggregationsExpression("terms:(project_id cardinality:user)")); - - var projectTerms = countResult.Aggregations.Terms("terms_project_id")?.Buckets ?? []; - var aggregations = projectTerms.ToDictionary(t => t.Key, t => t.Aggregations.Cardinality("cardinality_user")?.Value.GetValueOrDefault() ?? 0); - await scopedCacheClient.SetAllAsync(aggregations.Where(t => t.Value >= 10).ToDictionary(k => k.Key, v => v.Value), TimeSpan.FromMinutes(5)); - totals.AddRange(aggregations); - - return totals; - } - private async Task GetModelAsync(string id, HttpContext httpContext, bool useCache = true) { if (String.IsNullOrEmpty(id)) diff --git a/src/Exceptionless.Web/Api/Handlers/StackRollupHandler.cs b/src/Exceptionless.Web/Api/Handlers/StackRollupHandler.cs new file mode 100644 index 0000000000..b2c7054369 --- /dev/null +++ b/src/Exceptionless.Web/Api/Handlers/StackRollupHandler.cs @@ -0,0 +1,301 @@ +using Exceptionless.Core; +using Exceptionless.Core.Extensions; +using Exceptionless.Core.Models; +using Exceptionless.Core.Plugins.Formatting; +using Exceptionless.Core.Queries.Validation; +using Exceptionless.Core.Repositories; +using Exceptionless.Core.Repositories.Queries; +using Exceptionless.Core.Services; +using Exceptionless.DateTimeExtensions; +using Exceptionless.Web.Api.Infrastructure; +using Exceptionless.Web.Api.Messages; +using Exceptionless.Web.Api.Results; +using Exceptionless.Web.Extensions; +using Exceptionless.Web.Utility; +using Foundatio.Caching; +using Foundatio.Mediator; +using Foundatio.Repositories; +using Foundatio.Repositories.Extensions; +using Foundatio.Repositories.Models; + +namespace Exceptionless.Web.Api.Handlers; + +public sealed class StackRollupHandler( + IStackRollupSearchService stackRollupSearchService, + IStackRepository stackRepository, + IOrganizationRepository organizationRepository, + IProjectRepository projectRepository, + FormattingPluginManager formattingPluginManager, + ICacheClient cacheClient, + EventStackQueryValidator validator, + AppOptions options, + TimeProvider timeProvider) +{ + private static readonly ICollection _allowedDateFields = ["date"]; + private const string DefaultDateField = "date"; + + public async Task> Handle(GetStackRollupStats message) + { + var organizations = await GetSelectedOrganizationsAsync(message.Context, message.Filter); + if (organizations.All(organization => organization.IsSuspended)) + return new StackRollupStatsResult(0, 0, 0, []); + + var time = TimeRangeParser.GetTimeInfo(message.Time, message.Offset, timeProvider, _allowedDateFields, DefaultDateField, organizations.GetRetentionUtcCutoff(options.MaximumRetentionDays, timeProvider)); + return await GetStatsInternalAsync(new AppFilter(organizations) { IsUserOrganizationsFilter = true }, time, message.Filter, message.Context); + } + + public async Task> Handle(GetStackRollupStatsByOrganization message) + { + var organization = await GetOrganizationAsync(message.OrganizationId, message.Context); + if (organization is null) + return Result.NotFound("Organization not found."); + if (organization.IsSuspended) + return PlanLimitResult("Unable to view stack occurrences for the suspended organization."); + + var time = TimeRangeParser.GetTimeInfo(message.Time, message.Offset, timeProvider, _allowedDateFields, DefaultDateField, organization.GetRetentionUtcCutoff(options.MaximumRetentionDays, timeProvider)); + return await GetStatsInternalAsync(new AppFilter(organization), time, message.Filter, message.Context); + } + + public async Task> Handle(GetStackRollupStatsByProject message) + { + var project = await GetProjectAsync(message.ProjectId, message.Context); + if (project is null) + return Result.NotFound("Project not found."); + var organization = await GetOrganizationAsync(project.OrganizationId, message.Context); + if (organization is null) + return Result.NotFound("Organization not found."); + if (organization.IsSuspended) + return PlanLimitResult("Unable to view stack occurrences for the suspended organization."); + + var time = TimeRangeParser.GetTimeInfo(message.Time, message.Offset, timeProvider, _allowedDateFields, DefaultDateField, organization.GetRetentionUtcCutoff(project, options.MaximumRetentionDays, timeProvider)); + return await GetStatsInternalAsync(new AppFilter(project, organization), time, message.Filter, message.Context); + } + + public async Task>> Handle(GetAllStackRollups message) + { + var organizations = await GetSelectedOrganizationsAsync(message.Context, message.Filter); + if (organizations.All(organization => organization.IsSuspended)) + return new PagedResult([], false); + + var time = TimeRangeParser.GetTimeInfo(message.Time, message.Offset, timeProvider, _allowedDateFields, DefaultDateField, organizations.GetRetentionUtcCutoff(options.MaximumRetentionDays, timeProvider)); + return await GetInternalAsync(new AppFilter(organizations) { IsUserOrganizationsFilter = true }, time, message.Filter, message.Sort, message.Limit, message.Before, message.After, message.Include, message.Context); + } + + public async Task>> Handle(GetStackRollupsByOrganization message) + { + var organization = await GetOrganizationAsync(message.OrganizationId, message.Context); + if (organization is null) + return Result.NotFound("Organization not found."); + if (organization.IsSuspended) + return PlanLimitResult>("Unable to view stack occurrences for the suspended organization."); + + var time = TimeRangeParser.GetTimeInfo(message.Time, message.Offset, timeProvider, _allowedDateFields, DefaultDateField, organization.GetRetentionUtcCutoff(options.MaximumRetentionDays, timeProvider)); + return await GetInternalAsync(new AppFilter(organization), time, message.Filter, message.Sort, message.Limit, message.Before, message.After, message.Include, message.Context); + } + + public async Task>> Handle(GetStackRollupsByProject message) + { + var project = await GetProjectAsync(message.ProjectId, message.Context); + if (project is null) + return Result.NotFound("Project not found."); + + var organization = await GetOrganizationAsync(project.OrganizationId, message.Context); + if (organization is null) + return Result.NotFound("Organization not found."); + if (organization.IsSuspended) + return PlanLimitResult>("Unable to view stack occurrences for the suspended organization."); + + var time = TimeRangeParser.GetTimeInfo(message.Time, message.Offset, timeProvider, _allowedDateFields, DefaultDateField, organization.GetRetentionUtcCutoff(project, options.MaximumRetentionDays, timeProvider)); + return await GetInternalAsync(new AppFilter(project, organization), time, message.Filter, message.Sort, message.Limit, message.Before, message.After, message.Include, message.Context); + } + + private async Task>> GetInternalAsync( + AppFilter appFilter, + TimeInfo time, + string? filter, + string? sort, + int limit, + string? before, + string? after, + string? include, + HttpContext httpContext) + { + if (before is not null && after is not null) + return Result.BadRequest("The before and after parameters cannot be used together."); + + limit = Pagination.GetLimit(limit); + var validation = await validator.ValidateQueryAsync(filter); + if (!validation.IsValid) + return Result.BadRequest(validation.Message ?? "Invalid filter."); + + appFilter.UsesPremiumFeatures = validation.UsesPremiumFeatures; + AppFilter? appliedAppFilter = ApiFilterPolicy.ShouldApplySystemFilter(appFilter, filter, httpContext.Request) ? appFilter : null; + if (appliedAppFilter is not null && ApiFilterPolicy.IsPremiumFeatureQueryBlocked(appliedAppFilter)) + return PlanLimitResult>(ApiFilterPolicy.PremiumSearchUpgradeMessage); + + try + { + var result = await stackRollupSearchService.SearchAsync(new StackRollupSearchRequest( + appliedAppFilter, + time.Range.UtcStart, + time.Range.UtcEnd, + time.Offset, + httpContext.Request.Query["time"], + filter, + sort, + limit, + before, + after, + ShouldInclude(include, "total")), httpContext.RequestAborted); + + string[] stackIds = result.Rows.Select(row => row.StackId).ToArray(); + var stacks = (await stackRepository.GetByIdsAsync(stackIds)) + .Select(stack => stack.ApplyOffset(time.Offset)) + .ToList(); + var summaries = await GetStackSummariesAsync(stacks, result.Rows, appFilter, time); + + return new PagedResult(summaries.Cast().ToList(), result.HasMore, null, result.Total, result.Before, result.After); + } + catch (InvalidStackRollupCursorException ex) + { + return Result.BadRequest(ex.Message); + } + catch (ArgumentOutOfRangeException ex) when (ex.ParamName == "sort") + { + return Result.BadRequest("Sort must be one of total, users, first_occurrence, or last_occurrence, optionally prefixed with '-'."); + } + } + + private async Task> GetStatsInternalAsync(AppFilter appFilter, TimeInfo time, string? filter, HttpContext httpContext) + { + var filterValidation = await validator.ValidateQueryAsync(filter); + if (!filterValidation.IsValid) + return Result.BadRequest(filterValidation.Message ?? "Invalid filter."); + + appFilter.UsesPremiumFeatures = filterValidation.UsesPremiumFeatures; + AppFilter? appliedAppFilter = ApiFilterPolicy.ShouldApplySystemFilter(appFilter, filter, httpContext.Request) ? appFilter : null; + if (appliedAppFilter is not null && ApiFilterPolicy.IsPremiumFeatureQueryBlocked(appliedAppFilter)) + return PlanLimitResult(ApiFilterPolicy.PremiumSearchUpgradeMessage); + + return await stackRollupSearchService.GetStatsAsync(new StackRollupStatsRequest( + appliedAppFilter, + time.Range.UtcStart, + time.Range.UtcEnd, + time.Offset, + filter), httpContext.RequestAborted); + } + + private async Task> GetStackSummariesAsync(List stacks, IReadOnlyCollection rows, AppFilter appFilter, TimeInfo time) + { + if (stacks.Count == 0) + return []; + + var stacksById = stacks.ToDictionary(stack => stack.Id, StringComparer.Ordinal); + var projects = await projectRepository.GetByIdsAsync(stacks.Select(stack => stack.ProjectId).Distinct().ToArray(), query => query.Cache()); + var projectNames = projects.ToDictionary(project => project.Id, project => project.Name); + var totalUsers = await GetUserCountByProjectIdsAsync(stacks, appFilter, time.Range.UtcStart, time.Range.UtcEnd); + var summaries = new List(rows.Count); + + foreach (var row in rows) + { + if (!stacksById.TryGetValue(row.StackId, out var stack)) + continue; + + var data = formattingPluginManager.GetStackSummaryData(stack); + summaries.Add(new StackSummaryModel + { + Id = data.Id, + TemplateKey = data.TemplateKey, + Data = data.Data, + ProjectId = stack.ProjectId, + ProjectName = projectNames.GetValueOrDefault(stack.ProjectId), + Tags = stack.Tags?.OfType().Order(StringComparer.OrdinalIgnoreCase).ToArray() ?? [], + Title = stack.Title, + Status = stack.Status, + FirstOccurrence = row.FirstOccurrence, + LastOccurrence = row.LastOccurrence, + Total = row.Total, + Users = row.Users, + TotalUsers = totalUsers.GetOrDefault(stack.ProjectId) + }); + } + + return summaries; + } + + private async Task> GetUserCountByProjectIdsAsync(ICollection stacks, AppFilter appFilter, DateTime utcStart, DateTime utcEnd) + { + using var scopedCacheClient = new ScopedCacheClient(cacheClient, $"Project:user-count:{utcStart.Floor(TimeSpan.FromMinutes(15)).Ticks}-{utcEnd.Floor(TimeSpan.FromMinutes(15)).Ticks}"); + var projectIds = stacks.Select(stack => stack.ProjectId).Distinct().ToList(); + var cachedTotals = await scopedCacheClient.GetAllAsync(projectIds); + var totals = cachedTotals.Where(item => item.Value.HasValue).ToDictionary(item => item.Key, item => item.Value.Value); + if (totals.Count == projectIds.Count) + return totals; + + var projects = cachedTotals + .Where(item => !item.Value.HasValue && stacks.Contains(stack => stack.ProjectId == item.Key)) + .Select(item => new Project { Id = item.Key, OrganizationId = stacks.First(stack => stack.ProjectId == item.Key).OrganizationId }) + .ToList(); + var aggregations = (await stackRollupSearchService.GetProjectUserCountsAsync(new StackRollupProjectUsersRequest( + appFilter, + utcStart, + utcEnd, + projects.Select(project => project.Id).ToArray()))) + .ToDictionary(item => item.Key, item => (double)item.Value); + await scopedCacheClient.SetAllAsync(aggregations.Where(item => item.Value >= 10).ToDictionary(item => item.Key, item => item.Value), TimeSpan.FromMinutes(5)); + totals.AddRange(aggregations); + return totals; + } + + private async Task> GetSelectedOrganizationsAsync(HttpContext httpContext, string? filter) + { + var organizationIds = httpContext.Request.GetAssociatedOrganizationIds(); + if (organizationIds.Count == 0) + return []; + + if (!String.IsNullOrEmpty(filter)) + { + var scope = GetFilterScopeVisitor.Run(filter); + if (scope.IsScopable) + { + Organization? organization = null; + if (scope.OrganizationId is not null) + organization = await organizationRepository.GetByIdAsync(scope.OrganizationId, query => query.Cache()); + else if (scope.ProjectId is not null) + { + var project = await projectRepository.GetByIdAsync(scope.ProjectId, query => query.Cache()); + if (project is not null) + organization = await organizationRepository.GetByIdAsync(project.OrganizationId, query => query.Cache()); + } + else if (scope.StackId is not null) + { + var stack = await stackRepository.GetByIdAsync(scope.StackId, query => query.Cache()); + if (stack is not null) + organization = await organizationRepository.GetByIdAsync(stack.OrganizationId, query => query.Cache()); + } + + if (organization is not null) + return organizationIds.Contains(organization.Id) || httpContext.Request.IsGlobalAdmin() ? [organization] : []; + } + } + + return await organizationRepository.GetByIdsAsync(organizationIds.ToArray(), query => query.Cache()); + } + + private Task GetOrganizationAsync(string organizationId, HttpContext httpContext) + => String.IsNullOrEmpty(organizationId) || !httpContext.Request.CanAccessOrganization(organizationId) + ? Task.FromResult(null) + : organizationRepository.GetByIdAsync(organizationId, query => query.Cache()); + + private async Task GetProjectAsync(string projectId, HttpContext httpContext) + { + var project = await projectRepository.GetByIdAsync(projectId, query => query.Cache()); + return project is null || !httpContext.Request.CanAccessOrganization(project.OrganizationId) ? null : project; + } + + private static bool ShouldInclude(string? include, string value) + => include?.Split(',', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries).Contains(value, StringComparer.OrdinalIgnoreCase) == true; + + private static Result PlanLimitResult(string message) + => Result.Invalid(ValidationError.Create(ApiValidationErrorIdentifiers.PlanLimit, message)); +} diff --git a/src/Exceptionless.Web/Api/Messages/EventMessages.cs b/src/Exceptionless.Web/Api/Messages/EventMessages.cs index 6c1ac3bb95..d5a6999f14 100644 --- a/src/Exceptionless.Web/Api/Messages/EventMessages.cs +++ b/src/Exceptionless.Web/Api/Messages/EventMessages.cs @@ -5,9 +5,9 @@ namespace Exceptionless.Web.Api.Messages; // Count messages -public record GetEventCount(string? Filter, string? Aggregations, string? Time, string? Offset, string? Mode, HttpContext Context); -public record GetEventCountByOrganization(string OrganizationId, string? Filter, string? Aggregations, string? Time, string? Offset, string? Mode, HttpContext Context); -public record GetEventCountByProject(string ProjectId, string? Filter, string? Aggregations, string? Time, string? Offset, string? Mode, HttpContext Context); +public record GetEventCount(string? Filter, string? Aggregations, string? Time, string? Offset, HttpContext Context); +public record GetEventCountByOrganization(string OrganizationId, string? Filter, string? Aggregations, string? Time, string? Offset, HttpContext Context); +public record GetEventCountByProject(string ProjectId, string? Filter, string? Aggregations, string? Time, string? Offset, HttpContext Context); // Get events public record GetEventById(string Id, string? ExpectedStackId, string? Time, string? Offset, HttpContext Context); diff --git a/src/Exceptionless.Web/Api/Messages/StackMessages.cs b/src/Exceptionless.Web/Api/Messages/StackMessages.cs index 26fba9556c..7b0d0c319b 100644 --- a/src/Exceptionless.Web/Api/Messages/StackMessages.cs +++ b/src/Exceptionless.Web/Api/Messages/StackMessages.cs @@ -16,6 +16,12 @@ public record MarkStacksNotCritical(string Ids, HttpContext Context); public record ChangeStacksStatus(string Ids, StackStatus Status, HttpContext Context); public record PromoteStack(string Id, HttpContext Context); public record DeleteStacks(string Ids, HttpContext Context); -public record GetAllStacks(string? Filter, string? Sort, string? Time, string? Offset, string? Mode, int Page, int Limit, HttpContext Context); -public record GetStacksByOrganization(string OrganizationId, string? Filter, string? Sort, string? Time, string? Offset, string? Mode, int Page, int Limit, HttpContext Context); -public record GetStacksByProject(string ProjectId, string? Filter, string? Sort, string? Time, string? Offset, string? Mode, int Page, int Limit, HttpContext Context); +public record GetAllStacks(string? Filter, string? Sort, string? Time, string? Offset, int Limit, string? Before, string? After, HttpContext Context); +public record GetStacksByOrganization(string OrganizationId, string? Filter, string? Sort, string? Time, string? Offset, int Limit, string? Before, string? After, HttpContext Context); +public record GetStacksByProject(string ProjectId, string? Filter, string? Sort, string? Time, string? Offset, int Limit, string? Before, string? After, HttpContext Context); +public record GetAllStackRollups(string? Filter, string? Sort, string? Time, string? Offset, int Limit, string? Before, string? After, string? Include, HttpContext Context); +public record GetStackRollupsByOrganization(string OrganizationId, string? Filter, string? Sort, string? Time, string? Offset, int Limit, string? Before, string? After, string? Include, HttpContext Context); +public record GetStackRollupsByProject(string ProjectId, string? Filter, string? Sort, string? Time, string? Offset, int Limit, string? Before, string? After, string? Include, HttpContext Context); +public record GetStackRollupStats(string? Filter, string? Time, string? Offset, HttpContext Context); +public record GetStackRollupStatsByOrganization(string OrganizationId, string? Filter, string? Time, string? Offset, HttpContext Context); +public record GetStackRollupStatsByProject(string ProjectId, string? Filter, string? Time, string? Offset, HttpContext Context); diff --git a/src/Exceptionless.Web/ClientApp.angular/app/frequent-controller.js b/src/Exceptionless.Web/ClientApp.angular/app/frequent-controller.js index b0e2225202..f73a5799b0 100644 --- a/src/Exceptionless.Web/ClientApp.angular/app/frequent-controller.js +++ b/src/Exceptionless.Web/ClientApp.angular/app/frequent-controller.js @@ -212,10 +212,10 @@ vm.mostFrequent = { header: "Most Frequent", - get: eventService.getAll, + get: stackService.getRollups, options: { limit: 15, - mode: "stack_frequent", + sort: "-total", }, source: vm._source + ".Events", }; diff --git a/src/Exceptionless.Web/ClientApp.angular/app/new-controller.js b/src/Exceptionless.Web/ClientApp.angular/app/new-controller.js index e1dbad0269..f6d3142bfc 100644 --- a/src/Exceptionless.Web/ClientApp.angular/app/new-controller.js +++ b/src/Exceptionless.Web/ClientApp.angular/app/new-controller.js @@ -108,6 +108,23 @@ return organizationService.getAll().then(onSuccess); } + function getNewStacks(options) { + return stackService.getRollups(options, function (mergedOptions) { + var range = filterService.getTimeRange(); + if (!range.start && !range.end) { + return mergedOptions; + } + + var start = (range.start || moment(filterService.getOldestPossibleEventDate())).utc().format(); + var end = (range.end || moment()).utc().format(); + var firstOccurrenceFilter = 'first_occurrence:["' + start + '" TO "' + end + '"]'; + mergedOptions.filter = mergedOptions.filter + ? firstOccurrenceFilter + " (" + mergedOptions.filter + ")" + : firstOccurrenceFilter; + return mergedOptions; + }); + } + this.$onInit = function $onInit() { vm._organizations = []; vm._source = "app.New"; @@ -212,10 +229,10 @@ vm.newest = { header: "New Stacks", - get: eventService.getAll, + get: getNewStacks, options: { limit: 15, - mode: "stack_new", + sort: "-first_occurrence", }, source: vm._source + ".Events", }; diff --git a/src/Exceptionless.Web/ClientApp.angular/app/users-controller.js b/src/Exceptionless.Web/ClientApp.angular/app/users-controller.js index f6ed12eaf3..72a00a992f 100644 --- a/src/Exceptionless.Web/ClientApp.angular/app/users-controller.js +++ b/src/Exceptionless.Web/ClientApp.angular/app/users-controller.js @@ -212,10 +212,10 @@ vm.mostUsers = { header: "Most Users", - get: eventService.getAll, + get: stackService.getRollups, options: { limit: 15, - mode: "stack_users", + sort: "-users", }, source: vm._source + ".Events", }; diff --git a/src/Exceptionless.Web/ClientApp.angular/components/stack/stack-service.js b/src/Exceptionless.Web/ClientApp.angular/components/stack/stack-service.js index ce22340668..8bf67af7c2 100644 --- a/src/Exceptionless.Web/ClientApp.angular/components/stack/stack-service.js +++ b/src/Exceptionless.Web/ClientApp.angular/components/stack/stack-service.js @@ -33,58 +33,24 @@ return Restangular.one("stacks", id).get(); } - function getFrequent(options) { - var mergedOptions = filterService.apply(options); - var organization = filterService.getOrganizationId(); - if (organization) { - return Restangular.one("organizations", organization) - .one("stacks") - .all("frequent") - .getList(mergedOptions); - } - - var project = filterService.getProjectId(); - if (project) { - return Restangular.one("projects", project).one("stacks").all("frequent").getList(mergedOptions); - } - - return Restangular.one("stacks").all("frequent").getList(mergedOptions); - } - - function getUsers(options) { - var mergedOptions = filterService.apply(options); - var organization = filterService.getOrganizationId(); - if (organization) { - return Restangular.one("organizations", organization) - .one("stacks") - .all("users") - .getList(mergedOptions); - } - - var project = filterService.getProjectId(); - if (project) { - return Restangular.one("projects", project).one("stacks").all("users").getList(mergedOptions); - } - - return Restangular.one("stacks").all("users").getList(mergedOptions); - } - - function getNew(options) { - var mergedOptions = filterService.apply(options); + function getRollups(options, optionsCallback) { + optionsCallback = angular.isFunction(optionsCallback) + ? optionsCallback + : function (o) { + return o; + }; + var mergedOptions = optionsCallback(filterService.apply(options)); var organization = filterService.getOrganizationId(); if (organization) { - return Restangular.one("organizations", organization) - .one("stacks") - .all("new") - .getList(mergedOptions); + return Restangular.one("organizations", organization).all("stack-rollups").getList(mergedOptions); } var project = filterService.getProjectId(); if (project) { - return Restangular.one("projects", project).one("stacks").all("new").getList(mergedOptions); + return Restangular.one("projects", project).all("stack-rollups").getList(mergedOptions); } - return Restangular.one("stacks").all("new").getList(mergedOptions); + return Restangular.all("stack-rollups").getList(mergedOptions); } function markCritical(id) { @@ -122,9 +88,7 @@ changeStatus: changeStatus, getAll: getAll, getById: getById, - getFrequent: getFrequent, - getUsers: getUsers, - getNew: getNew, + getRollups: getRollups, markCritical: markCritical, markNotCritical: markNotCritical, markFixed: markFixed, diff --git a/src/Exceptionless.Web/ClientApp/e2e/tests/list-query-cache.e2e.ts b/src/Exceptionless.Web/ClientApp/e2e/tests/list-query-cache.e2e.ts index 797ae09030..707cef1825 100644 --- a/src/Exceptionless.Web/ClientApp/e2e/tests/list-query-cache.e2e.ts +++ b/src/Exceptionless.Web/ClientApp/e2e/tests/list-query-cache.e2e.ts @@ -83,7 +83,7 @@ function isApiRequest(request: Request): boolean { } function isListRequest(request: Request): boolean { - return /^\/api\/v2\/organizations\/[^/]+\/events(?:\/count)?$/.test(new URL(request.url()).pathname); + return /^\/api\/v2\/organizations\/[^/]+\/(?:events(?:\/count)?|stack-rollups(?:\/stats)?)$/.test(new URL(request.url()).pathname); } async function navigateToList(page: Page, name: 'Events' | 'Stacks'): Promise { @@ -131,9 +131,9 @@ function recordListRequest(counts: RequestCounts, request: Request): void { return; } - const isStats = url.pathname.endsWith('/count'); + const isStats = url.pathname.endsWith('/count') || url.pathname.endsWith('/stats'); const mode = url.searchParams.get('mode'); - if (mode === 'stack_frequent') { + if (url.pathname.includes('/stack-rollups')) { counts[isStats ? 'stackStats' : 'stackList']++; } else if (isStats || mode === 'summary') { counts[isStats ? 'eventStats' : 'eventList']++; diff --git a/src/Exceptionless.Web/ClientApp/e2e/tests/stack-effects-chaos.e2e.ts b/src/Exceptionless.Web/ClientApp/e2e/tests/stack-effects-chaos.e2e.ts index 3f8a565737..068d48bcfe 100644 --- a/src/Exceptionless.Web/ClientApp/e2e/tests/stack-effects-chaos.e2e.ts +++ b/src/Exceptionless.Web/ClientApp/e2e/tests/stack-effects-chaos.e2e.ts @@ -131,9 +131,11 @@ test('stack effects stay bounded through background, paging, and navigation chao await measureAction(diagnostics, 'paging', async () => { for (let index = 0; index < 4; index++) { await page.getByRole('button', { name: 'Go to next page' }).click(); - await expect(page).toHaveURL(/(?:\?|&)page=2(?:&|$)/); + await expect(page).toHaveURL(/(?:\?|&)after=[^&]+(?:&|$)/); + await expect(page).not.toHaveURL(/(?:\?|&)page=/); await page.getByRole('button', { name: 'Go to previous page' }).click(); - await expect(page).not.toHaveURL(/(?:\?|&)page=2(?:&|$)/); + await expect(page).not.toHaveURL(/(?:\?|&)(?:before|after)=/); + await expect(page).not.toHaveURL(/(?:\?|&)page=/); } }); expect(actionSample(diagnostics, 'paging').listRequests).toBe(1); @@ -268,7 +270,7 @@ function createChaosEvent(appUrl: string, run: string, index: number): { event: function isStackListRequest(request: Request, organizationId: string): boolean { const url = new URL(request.url()); - return url.pathname === `/api/v2/organizations/${organizationId}/events` && url.searchParams.get('mode') === 'stack_frequent'; + return url.pathname === `/api/v2/organizations/${organizationId}/stack-rollups`; } function isStackListResponse(response: Response, organizationId: string): boolean { @@ -322,7 +324,7 @@ function recordRequest(diagnostics: RuntimeDiagnostics, request: Request, organi } const url = new URL(request.url()); - if (url.pathname === `/api/v2/organizations/${organizationId}/events/count` && url.searchParams.get('mode') === 'stack_frequent') { + if (url.pathname === `/api/v2/organizations/${organizationId}/stack-rollups/stats`) { diagnostics.countRequests++; } } diff --git a/src/Exceptionless.Web/ClientApp/e2e/tests/stack-triage.e2e.ts b/src/Exceptionless.Web/ClientApp/e2e/tests/stack-triage.e2e.ts index efbebab3d9..477694bd76 100644 --- a/src/Exceptionless.Web/ClientApp/e2e/tests/stack-triage.e2e.ts +++ b/src/Exceptionless.Web/ClientApp/e2e/tests/stack-triage.e2e.ts @@ -1,5 +1,40 @@ import { expect, test } from '../fixtures/e2e-test'; import { ExceptionlessE2EJourney } from '../support/exceptionless-journey'; +import { createRepresentativeEvent } from '../support/synthetic-event'; + +test('project stack management uses cursor pagination without page numbers @signup', async ({ e2eApi, e2eScenario, page }) => { + const events = Array.from({ length: 6 }, (_, index) => { + const referenceId = `pw-stack-management-${e2eScenario.run}-${index}`; + const event = createRepresentativeEvent({ + appUrl: e2eApi.environment.appUrl, + message: `Project stack management ${e2eScenario.run} ${index}`, + referenceId, + runId: e2eScenario.run + }); + const simpleError = (event.data as Record)['@simple_error'] as Record; + simpleError.type = `ProjectStackManagementException${index}`; + simpleError.stack_trace = `Error: ${referenceId}\n at stack-management-${index}.ts:${index + 1}:1`; + return { event, referenceId }; + }); + + await Promise.all(events.map(({ event }) => e2eApi.submitEvent(e2eScenario.projectId, e2eScenario.projectToken, event))); + await Promise.all(events.map(({ referenceId }) => e2eApi.pollForEventByReference(e2eScenario.userToken, e2eScenario.projectId, referenceId))); + + const listResponse = page.waitForResponse((response) => new URL(response.url()).pathname === `/api/v2/projects/${e2eScenario.projectId}/stacks`); + await page.goto(`/next/project/${e2eScenario.projectId}/stacks?filter=status%3Aopen&limit=5`); + expect((await listResponse).ok()).toBe(true); + await expect(page.getByText('Manage project stacks, including restoring ignored or discarded stacks')).toBeVisible(); + await expect(page.locator('tbody tr:visible')).toHaveCount(5); + + await page.getByRole('button', { name: 'Go to next page' }).click(); + await expect(page).toHaveURL(/(?:\?|&)after=[^&]+(?:&|$)/); + await expect(page).not.toHaveURL(/(?:\?|&)page=/); + await expect(page.locator('tbody tr:visible').first()).toBeVisible(); + + await page.getByRole('button', { name: 'Go to previous page' }).click(); + await expect(page).not.toHaveURL(/(?:\?|&)(?:before|after|page)=/); + await expect(page.locator('tbody tr:visible')).toHaveCount(5); +}); test('new user can mark an open stack fixed from event details @signup', async ({ e2eApi, e2eScenario, page }) => { const journey = ExceptionlessE2EJourney.fromScenario(page, e2eApi, e2eScenario); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/api.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/events/api.svelte.ts index 16bcce87b3..11f273fc31 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/api.svelte.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/api.svelte.ts @@ -163,7 +163,7 @@ export interface GetEventsByReferenceRequest { }; } -export type GetEventsMode = 'stack_frequent' | 'stack_new' | 'stack_recent' | 'stack_users' | 'summary' | null; +export type GetEventsMode = 'summary' | null; export interface GetEventsParams { after?: string; @@ -183,7 +183,6 @@ export interface GetOrganizationCountRequest { params?: { aggregations?: string; filter?: string; - mode?: GetEventsMode; offset?: string; time?: string; }; @@ -216,7 +215,6 @@ export interface GetProjectCountRequest { params?: { aggregations?: string; filter?: string; - mode?: 'stack_new'; offset?: string; time?: string; }; @@ -246,7 +244,6 @@ export interface GetStackCountRequest { params?: { aggregations?: string; filter?: string; - mode?: 'stack_new'; offset?: string; time?: string; }; diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/events-data-table.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/events-data-table.svelte index 03564d2ad5..ccdcf65329 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/events-data-table.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/events-data-table.svelte @@ -1,11 +1,11 @@ -