diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 8241985197..a3d57c109d 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -134,7 +134,6 @@ jobs: - name: Resolve Elasticsearch image id: elasticsearch_image env: - PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} PR_HEAD_REPOSITORY: ${{ github.event.pull_request.head.repo.full_name }} PREVIEW_BASE_SHA: ${{ github.event.client_payload.base_sha }} PUSH_BEFORE_SHA: ${{ github.event.before }} @@ -145,7 +144,7 @@ jobs: build_locally=false if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]] && - ! git diff --quiet "$PR_BASE_SHA"...HEAD -- build/docker/elasticsearch/9.x .github/workflows/elasticsearch-docker-9.yml; then + ! git diff --quiet origin/main...HEAD -- build/docker/elasticsearch/9.x .github/workflows/elasticsearch-docker-9.yml; then image_changed=true elif [[ "$GITHUB_EVENT_NAME" == "push" && "$GITHUB_REF" == "refs/heads/main" ]] && ! git diff --quiet "$PUSH_BEFORE_SHA"..HEAD -- build/docker/elasticsearch/9.x .github/workflows/elasticsearch-docker-9.yml; then @@ -362,8 +361,8 @@ jobs: - name: Wait for Aspire Resources run: | for attempt in {1..60}; do - if curl -fksS https://web-ex.dev.localhost:7131/api/v2/about > /dev/null && - curl -fksS https://web-ex.dev.localhost:7131/next/login > /dev/null; then + if curl -fksS --connect-timeout 5 --max-time 10 https://web-ex.dev.localhost:7131/api/v2/about > /dev/null && + curl -fksS --connect-timeout 5 --max-time 10 https://web-ex.dev.localhost:7131/next/login > /dev/null; then break fi @@ -382,8 +381,8 @@ jobs: - name: Verify E2E Endpoints run: | - curl -fksS https://web-ex.dev.localhost:7131/api/v2/about > /dev/null - curl -fksS https://web-ex.dev.localhost:7131/next/login > /dev/null + curl -fksS --connect-timeout 5 --max-time 10 https://web-ex.dev.localhost:7131/api/v2/about > /dev/null + curl -fksS --connect-timeout 5 --max-time 10 https://web-ex.dev.localhost:7131/next/login > /dev/null - name: Run Playwright E2E Tests working-directory: src/Exceptionless.Web/ClientApp diff --git a/src/Exceptionless.AppHost/Exceptionless.AppHost.csproj b/src/Exceptionless.AppHost/Exceptionless.AppHost.csproj index 16cc44ac96..d2a4d8d9ff 100644 --- a/src/Exceptionless.AppHost/Exceptionless.AppHost.csproj +++ b/src/Exceptionless.AppHost/Exceptionless.AppHost.csproj @@ -12,7 +12,6 @@ - diff --git a/src/Exceptionless.AppHost/Extensions/ElasticsearchExtensions.cs b/src/Exceptionless.AppHost/Extensions/ElasticsearchExtensions.cs index 4022e89583..9a7762d4dc 100644 --- a/src/Exceptionless.AppHost/Extensions/ElasticsearchExtensions.cs +++ b/src/Exceptionless.AppHost/Extensions/ElasticsearchExtensions.cs @@ -1,4 +1,4 @@ -using Elastic.Clients.Elasticsearch; +using System.Text.Json; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Diagnostics.HealthChecks; @@ -136,19 +136,25 @@ public async Task CheckHealthAsync(HealthCheckContext context if (string.IsNullOrEmpty(connectionString)) return new HealthCheckResult(context.Registration.FailureStatus, "Connection string not available."); - using var settings = new ElasticsearchClientSettings(new Uri(connectionString)); - var client = new ElasticsearchClient(settings); - var response = await client.Cluster.HealthAsync( - request => request.WaitForStatus(Elastic.Clients.Elasticsearch.HealthStatus.Yellow), - cancellationToken); - bool isReady = response.IsValidResponse - && !response.TimedOut - && response.Status is Elastic.Clients.Elasticsearch.HealthStatus.Yellow or Elastic.Clients.Elasticsearch.HealthStatus.Green; - if (isReady) - return HealthCheckResult.Healthy(); - - return new HealthCheckResult( - context.Registration.FailureStatus, - $"Elasticsearch cluster health check failed. Timed out: {response.TimedOut}; status: {response.Status}. {response.DebugInformation}"); + try + { + using var client = new HttpClient { BaseAddress = new Uri(connectionString), Timeout = TimeSpan.FromSeconds(10) }; + using var response = await client.GetAsync("_cluster/health?wait_for_status=yellow&timeout=5s", cancellationToken); + if (!response.IsSuccessStatusCode) + return new HealthCheckResult(context.Registration.FailureStatus, $"Elasticsearch cluster health returned HTTP {(int)response.StatusCode}."); + + await using var responseStream = await response.Content.ReadAsStreamAsync(cancellationToken); + using var document = await JsonDocument.ParseAsync(responseStream, cancellationToken: cancellationToken); + bool timedOut = document.RootElement.TryGetProperty("timed_out", out var timedOutElement) && timedOutElement.GetBoolean(); + string? status = document.RootElement.TryGetProperty("status", out var statusElement) ? statusElement.GetString() : null; + if (!timedOut && status is "yellow" or "green") + return HealthCheckResult.Healthy(); + + return new HealthCheckResult(context.Registration.FailureStatus, $"Elasticsearch cluster health check failed. Timed out: {timedOut}; status: {status ?? "unknown"}."); + } + catch (Exception ex) when (ex is HttpRequestException or JsonException or TaskCanceledException) + { + return new HealthCheckResult(context.Registration.FailureStatus, "Elasticsearch cluster health check request failed.", ex); + } } } diff --git a/src/Exceptionless.AppHost/appsettings.Development.json b/src/Exceptionless.AppHost/appsettings.Development.json index 0c208ae918..832d220e44 100644 --- a/src/Exceptionless.AppHost/appsettings.Development.json +++ b/src/Exceptionless.AppHost/appsettings.Development.json @@ -1,4 +1,11 @@ { + "Elasticsearch": { + "Port": 9215, + "ContainerName": "Exceptionless-Elasticsearch-9-5-Lookup", + "DataVolume": "exceptionless.elasticsearch-9-5-lookup.data.v1", + "KibanaPort": 5615, + "KibanaContainerName": "Exceptionless-Kibana-9-5-Lookup" + }, "Logging": { "LogLevel": { "Default": "Information", diff --git a/src/Exceptionless.Core/Bootstrapper.cs b/src/Exceptionless.Core/Bootstrapper.cs index 805a2f76d4..812476998c 100644 --- a/src/Exceptionless.Core/Bootstrapper.cs +++ b/src/Exceptionless.Core/Bootstrapper.cs @@ -77,6 +77,7 @@ public static void RegisterServices(IServiceCollection services, AppOptions appO services.AddSingleton(); services.AddSingleton(s => s.GetRequiredService().Client); + services.AddSingleton(); services.AddSingleton(s => s.GetRequiredService()); services.AddStartupAction(); diff --git a/src/Exceptionless.Core/Repositories/Configuration/Indexes/StackIndex.cs b/src/Exceptionless.Core/Repositories/Configuration/Indexes/StackIndex.cs index 02b15bdd89..76c84023ba 100644 --- a/src/Exceptionless.Core/Repositories/Configuration/Indexes/StackIndex.cs +++ b/src/Exceptionless.Core/Repositories/Configuration/Indexes/StackIndex.cs @@ -17,7 +17,7 @@ public sealed class StackIndex : VersionedIndex private readonly ExceptionlessElasticConfiguration _configuration; - public StackIndex(ExceptionlessElasticConfiguration configuration) : base(configuration, configuration.Options.ScopePrefix + "stacks", 1) + public StackIndex(ExceptionlessElasticConfiguration configuration) : base(configuration, configuration.Options.ScopePrefix + "stacks", 2) { _configuration = configuration; } @@ -27,11 +27,28 @@ public override void ConfigureIndex(CreateIndexRequestDescriptor idx) base.ConfigureIndex(idx); idx.Settings(s => s .Analysis(a => BuildAnalysis(a)) - .NumberOfShards(_configuration.Options.NumberOfShards) + .Mode("lookup") + .NumberOfShards(1) .NumberOfReplicas(_configuration.Options.NumberOfReplicas) .Priority(5)); } + protected override Task UpdateIndexAsync(string name, Action? descriptor = null) + { + if (descriptor is not null) + return base.UpdateIndexAsync(name, descriptor); + + // index.mode is a final creation-only setting. Foundatio derives updates from + // ConfigureIndex, so keep the mutable settings explicit for existing indexes. + return base.UpdateIndexAsync(name, update => update + .Reopen(true) + .Settings(new IndexSettings + { + NumberOfReplicas = _configuration.Options.NumberOfReplicas, + Priority = 5 + })); + } + public override void ConfigureIndexMapping(TypeMappingDescriptor map) { map diff --git a/src/Exceptionless.Core/Repositories/Queries/Visitors/EventStackFilterQueryVisitor.cs b/src/Exceptionless.Core/Repositories/Queries/Visitors/EventStackFilterQueryVisitor.cs index 7d06961364..a3d5e4299a 100644 --- a/src/Exceptionless.Core/Repositories/Queries/Visitors/EventStackFilterQueryVisitor.cs +++ b/src/Exceptionless.Core/Repositories/Queries/Visitors/EventStackFilterQueryVisitor.cs @@ -65,7 +65,7 @@ public EventStackFilter() _stackQueryVisitor.AddVisitor(new RemoveFieldsQueryVisitor(f => !stackFields.Contains(f))); _stackQueryVisitor.AddVisitor(new CleanupQueryVisitor()); // handles stack special fields and changing event field names to their stack equivalent - _stackQueryVisitor.AddVisitor(new StackFilterQueryVisitor()); + _stackQueryVisitor.AddVisitor(new StackFilterQueryVisitor(_stackOnlyFields.Union(_stackOnlySpecialFields))); _stackQueryVisitor.AddVisitor(new CleanupQueryVisitor()); _invertedStackQueryVisitor = new ChainedQueryVisitor(); @@ -73,7 +73,7 @@ public EventStackFilter() _invertedStackQueryVisitor.AddVisitor(new RemoveFieldsQueryVisitor(f => !stackFields.Contains(f))); _invertedStackQueryVisitor.AddVisitor(new CleanupQueryVisitor()); // handles stack special fields and changing event field names to their stack equivalent - _invertedStackQueryVisitor.AddVisitor(new StackFilterQueryVisitor()); + _invertedStackQueryVisitor.AddVisitor(new StackFilterQueryVisitor(_stackOnlyFields.Union(_stackOnlySpecialFields))); _invertedStackQueryVisitor.AddVisitor(new CleanupQueryVisitor()); // inverts the filter _invertedStackQueryVisitor.AddVisitor(new InvertQueryVisitor(_stackNonInvertedFields)); @@ -109,13 +109,21 @@ public EventStackFilter() InvertedFilter = invertedResult?.ToString(), HasStatus = context.GetBoolean(nameof(StackFilter.HasStatus)), HasStackIds = context.GetBoolean(nameof(StackFilter.HasStackIds)), - HasStatusOpen = context.GetBoolean(nameof(StackFilter.HasStatusOpen)) + HasStatusOpen = context.GetBoolean(nameof(StackFilter.HasStatusOpen)), + HasStackOnlyCriteria = context.GetBoolean(nameof(StackFilter.HasStackOnlyCriteria)) }; } } public class StackFilterQueryVisitor : ChainableQueryVisitor { + private readonly ISet _stackOnlyFields; + + public StackFilterQueryVisitor(IEnumerable? stackOnlyFields = null) + { + _stackOnlyFields = new HashSet(stackOnlyFields ?? [], StringComparer.OrdinalIgnoreCase); + } + public override Task VisitAsync(TermNode node, IQueryVisitorContext context) { IQueryNode result = node; @@ -127,6 +135,9 @@ public class StackFilterQueryVisitor : ChainableQueryVisitor return Task.FromResult(null); } + if (_stackOnlyFields.Contains(node.Field)) + context.SetValue(nameof(StackFilter.HasStackOnlyCriteria), true); + // process special stack fields switch (node.Field?.ToLowerInvariant()) { @@ -216,4 +227,5 @@ public record StackFilter public required bool HasStatus { get; set; } public required bool HasStatusOpen { get; set; } public required bool HasStackIds { get; set; } + public required bool HasStackOnlyCriteria { get; set; } } diff --git a/src/Exceptionless.Core/Services/StackRollupSearchService.cs b/src/Exceptionless.Core/Services/StackRollupSearchService.cs new file mode 100644 index 0000000000..c2e67b0ab7 --- /dev/null +++ b/src/Exceptionless.Core/Services/StackRollupSearchService.cs @@ -0,0 +1,1084 @@ +using System.Diagnostics; +using System.Globalization; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.Esql; +using Elastic.Clients.Elasticsearch.QueryDsl; +using Exceptionless.Core.Configuration; +using Exceptionless.Core.Models; +using Exceptionless.Core.Models.Data; +using Exceptionless.Core.Repositories; +using Exceptionless.Core.Repositories.Configuration; +using Exceptionless.Core.Repositories.Queries; +using Foundatio.Repositories; +using Foundatio.Repositories.Elasticsearch.Queries.Builders; +using Foundatio.Repositories.Models; +using Foundatio.Repositories.Options; +using Microsoft.Extensions.Logging; + +namespace Exceptionless.Core.Services; + +public interface IStackRollupSearchService +{ + Task RequiresLookupJoinAsync(string? filter, CancellationToken cancellationToken = default); + Task SearchEventsAsync(EventLookupSearchRequest request, CancellationToken cancellationToken = default); + Task CountEventsAsync(EventLookupCountRequest request, CancellationToken cancellationToken = default); + Task SearchAsync(StackRollupSearchRequest request, CancellationToken cancellationToken = default); + Task GetStatsAsync(StackRollupStatsRequest request, CancellationToken cancellationToken = default); + Task> GetProjectUserCountsAsync(StackRollupProjectUsersRequest request, CancellationToken cancellationToken = default); +} + +public sealed record EventLookupSearchRequest( + AppFilter? AppFilter, + DateTime UtcStart, + DateTime UtcEnd, + string? TimeExpression, + string? Filter, + string? Sort, + int Limit, + string? Before, + string? After, + bool IncludeTotal); + +public sealed record EventLookupSearchResult( + IReadOnlyCollection EventIds, + bool HasMore, + long? Total, + string? Before, + string? After); + +public sealed record EventLookupCountRequest( + AppFilter? AppFilter, + DateTime UtcStart, + DateTime UtcEnd, + TimeSpan Offset, + string? Filter, + string? Aggregations, + int BucketCount = 50); + +public sealed record StackRollupSearchRequest( + AppFilter? AppFilter, + DateTime UtcStart, + DateTime UtcEnd, + TimeSpan Offset, + string? TimeExpression, + string? Filter, + string? Sort, + int Limit, + string? Before, + string? After, + bool IncludeTotal); + +public sealed record StackRollupSearchResult( + IReadOnlyCollection Rows, + bool HasMore, + long? Total, + string? Before, + string? After); + +public sealed record StackRollupStatsRequest( + AppFilter? AppFilter, + DateTime UtcStart, + DateTime UtcEnd, + TimeSpan Offset, + string? Filter, + int BucketCount = 50); + +public sealed record StackRollupStatsResult( + long TotalEvents, + long TotalStacks, + long NewStacks, + IReadOnlyCollection Buckets, + long Documents = 0); + +public sealed record StackRollupStatsBucket(DateTime Date, long Events, long Stacks, long Documents = 0); + +public sealed record StackRollupProjectUsersRequest( + AppFilter AppFilter, + DateTime UtcStart, + DateTime UtcEnd, + IReadOnlyCollection ProjectIds); + +public sealed record StackRollupRow( + string StackId, + long Total, + long Users, + DateTime FirstOccurrence, + DateTime LastOccurrence); + +public sealed class InvalidStackRollupCursorException(string message) : Exception(message); +public sealed class InvalidEventLookupCursorException(string message) : Exception(message); + +public sealed class StackRollupSearchService : IStackRollupSearchService +{ + private const int CursorVersion = 1; + private static readonly TimeSpan ReadinessCacheDuration = TimeSpan.FromMinutes(1); + private readonly ElasticsearchClient _client; + private readonly ExceptionlessElasticConfiguration _configuration; + private readonly TimeProvider _timeProvider; + private readonly JsonSerializerOptions _serializerOptions; + private readonly EventStackFilter _eventStackFilter = new(); + private readonly ILogger _logger; + private readonly SemaphoreSlim _readinessLock = new(1, 1); + private StackRollupReadiness? _readiness; + private DateTimeOffset _readinessExpiresUtc; + + public StackRollupSearchService( + ElasticsearchClient client, + ExceptionlessElasticConfiguration configuration, + TimeProvider timeProvider, + JsonSerializerOptions serializerOptions, + ILoggerFactory loggerFactory) + { + _client = client; + _configuration = configuration; + _timeProvider = timeProvider; + _serializerOptions = serializerOptions; + _logger = loggerFactory.CreateLogger(); + } + + public async Task RequiresLookupJoinAsync(string? filter, CancellationToken cancellationToken = default) + { + if (String.IsNullOrWhiteSpace(filter)) + return false; + + var stackFilter = await _eventStackFilter.GetStackFilterAsync(StripAlternateInversion(filter)); + return stackFilter?.HasStackOnlyCriteria == true; + } + + public async Task SearchEventsAsync(EventLookupSearchRequest request, CancellationToken cancellationToken = default) + { + var sort = GetEventSort(request.Sort); + await EnsureReadyAsync(cancellationToken); + + string fingerprint = CreateEventFingerprint(request, sort.Value); + EventLookupCursor? cursor = DecodeEventCursor(request.Before ?? request.After, sort.Value, fingerprint); + DateTime utcStart = cursor is null ? request.UtcStart : new DateTime(cursor.UtcStart, DateTimeKind.Utc); + DateTime utcEnd = cursor is null ? request.UtcEnd : new DateTime(cursor.UtcEnd, DateTimeKind.Utc); + string normalizedFilter = StripAlternateInversion(request.Filter); + string? eventFilter = await _eventStackFilter.GetEventFilterAsync(normalizedFilter); + string? stackFilter = (await _eventStackFilter.GetStackFilterAsync(normalizedFilter))?.Filter; + Query? sourceFilter = await BuildSourceFilterAsync(request.AppFilter, utcStart, utcEnd, eventFilter); + var parameters = new List>>(); + string query = BuildEventQuery(request, sort, cursor, stackFilter, parameters); + var rows = await ExecuteEventRowsAsync(query, sourceFilter, parameters, cancellationToken); + long? total = request.IncludeTotal ? rows.FirstOrDefault()?.TotalEvents : null; + if (request.IncludeTotal && total is null) + total = await ExecuteJoinedEventTotalAsync(stackFilter, sourceFilter, cancellationToken); + + bool isBefore = request.Before is not null; + bool hasExtra = rows.Count > request.Limit; + if (hasExtra) + rows.RemoveAt(rows.Count - 1); + if (isBefore) + rows.Reverse(); + + bool hasPrevious = rows.Count > 0 && (isBefore ? hasExtra : request.After is not null); + bool hasNext = rows.Count > 0 && (isBefore || hasExtra); + return new EventLookupSearchResult( + rows.Select(row => row.EventId).ToArray(), + hasNext, + total, + hasPrevious ? EncodeEventCursor(rows[0], sort, utcStart, utcEnd, fingerprint) : null, + hasNext ? EncodeEventCursor(rows[^1], sort, utcStart, utcEnd, fingerprint) : null); + } + + public async Task CountEventsAsync(EventLookupCountRequest request, CancellationToken cancellationToken = default) + { + await EnsureReadyAsync(cancellationToken); + string normalizedFilter = StripAlternateInversion(request.Filter); + string? eventFilter = await _eventStackFilter.GetEventFilterAsync(normalizedFilter); + string? stackFilter = (await _eventStackFilter.GetStackFilterAsync(normalizedFilter))?.Filter; + Query? sourceFilter = await BuildSourceFilterAsync(request.AppFilter, request.UtcStart, request.UtcEnd, eventFilter); + + if (String.IsNullOrWhiteSpace(request.Aggregations)) + return new CountResult(await ExecuteJoinedEventTotalAsync(stackFilter, sourceFilter, cancellationToken)); + + if (IsTermsTagsAggregation(request.Aggregations)) + return await CountEventTagsAsync(stackFilter, sourceFilter, cancellationToken); + + if (!IsDashboardAggregation(request.Aggregations)) + throw new NotSupportedException("Stack filters currently support the event dashboard aggregations or terms:tags."); + + var stats = await GetStatsAsync(new StackRollupStatsRequest( + request.AppFilter, + request.UtcStart, + request.UtcEnd, + request.Offset, + request.Filter, + request.BucketCount), cancellationToken); + return ToCountResult(stats); + } + + public async Task SearchAsync(StackRollupSearchRequest request, CancellationToken cancellationToken = default) + { + var sort = GetSort(request.Sort); + await EnsureReadyAsync(cancellationToken); + + string fingerprint = CreateFingerprint(request); + StackRollupCursor? cursor = DecodeCursor(request.Before ?? request.After, sort.Value, fingerprint); + DateTime utcStart = cursor is null ? request.UtcStart : new DateTime(cursor.UtcStart, DateTimeKind.Utc); + DateTime utcEnd = cursor is null ? request.UtcEnd : new DateTime(cursor.UtcEnd, DateTimeKind.Utc); + string normalizedFilter = StripAlternateInversion(request.Filter); + string? eventFilter = await _eventStackFilter.GetEventFilterAsync(normalizedFilter); + string? stackFilter = (await _eventStackFilter.GetStackFilterAsync(normalizedFilter))?.Filter; + Query? sourceFilter = await BuildSourceFilterAsync(request.AppFilter, utcStart, utcEnd, eventFilter); + + var stopwatch = Stopwatch.StartNew(); + _logger.LogDebug( + "Executing ES|QL stack rollup sort {Sort}, direction {Direction}, range {UtcStart:o} to {UtcEnd:o}, event filter {HasEventFilter}, stack filter {HasStackFilter}", + sort.Value, + request.Before is not null ? "before" : request.After is not null ? "after" : "initial", + utcStart, + utcEnd, + !String.IsNullOrWhiteSpace(eventFilter), + !String.IsNullOrWhiteSpace(stackFilter)); + + var parameters = new List>>(); + string query = BuildQuery(request, sort, cursor, stackFilter, parameters, countOnly: false); + var rows = await ExecuteRowsAsync(query, sourceFilter, parameters, cancellationToken); + long? total = request.IncludeTotal ? rows.FirstOrDefault()?.TotalStacks : null; + + if (request.IncludeTotal && total is null) + { + parameters.Clear(); + string countQuery = BuildQuery(request, sort, cursor: null, stackFilter, parameters, countOnly: true); + total = await ExecuteTotalAsync(countQuery, sourceFilter, parameters, cancellationToken); + } + + bool isBefore = request.Before is not null; + bool hasExtra = rows.Count > request.Limit; + if (hasExtra) + rows.RemoveAt(rows.Count - 1); + + if (isBefore) + rows.Reverse(); + + bool hasPrevious = rows.Count > 0 && (isBefore ? hasExtra : request.After is not null); + bool hasNext = rows.Count > 0 && (isBefore || hasExtra); + string? before = hasPrevious ? EncodeCursor(rows[0], sort, utcStart, utcEnd, fingerprint) : null; + string? after = hasNext ? EncodeCursor(rows[^1], sort, utcStart, utcEnd, fingerprint) : null; + + _logger.LogDebug( + "Completed ES|QL stack rollup sort {Sort}, direction {Direction}, rows {RowCount}, has more {HasMore}, duration {DurationMs}ms", + sort.Value, + isBefore ? "before" : request.After is not null ? "after" : "initial", + rows.Count, + hasNext, + stopwatch.Elapsed.TotalMilliseconds); + + return new StackRollupSearchResult( + rows.Select(ToPublicRow).ToArray(), + hasNext, + total, + before, + after); + } + + public async Task GetStatsAsync(StackRollupStatsRequest request, CancellationToken cancellationToken = default) + { + await EnsureReadyAsync(cancellationToken); + string normalizedFilter = StripAlternateInversion(request.Filter); + string? eventFilter = await _eventStackFilter.GetEventFilterAsync(normalizedFilter); + string? stackFilter = (await _eventStackFilter.GetStackFilterAsync(normalizedFilter))?.Filter; + Query? sourceFilter = await BuildSourceFilterAsync(request.AppFilter, request.UtcStart, request.UtcEnd, eventFilter); + var parameters = new List>>(); + string query = BuildStatsQuery(request, stackFilter, parameters); + + using var response = await _client.Esql.QueryAsync(new EsqlQueryRequest(query) + { + AllowPartialResults = false, + Columnar = false, + Filter = sourceFilter, + Format = EsqlFormat.Json, + Params = new Union>, ICollection>>>(parameters) + }, cancellationToken); + + if (!response.IsValidResponse) + { + _logger.LogWarning("Stack rollup stats lookup join failed with Elasticsearch status {StatusCode}", response.ApiCallDetails?.HttpStatusCode); + throw new ApplicationException("The stack rollup stats query failed."); + } + + using var document = await JsonDocument.ParseAsync(response.Body, cancellationToken: cancellationToken); + return ReadStats(document.RootElement); + } + + public async Task> GetProjectUserCountsAsync(StackRollupProjectUsersRequest request, CancellationToken cancellationToken = default) + { + if (request.ProjectIds.Count == 0) + return new Dictionary(); + + await EnsureReadyAsync(cancellationToken); + Query? sourceFilter = await BuildSourceFilterAsync(request.AppFilter, request.UtcStart, request.UtcEnd, eventFilter: null); + string eventIndex = ValidateIndexName(_configuration.Events.Name); + string stackIndex = ValidateIndexName(_configuration.Stacks.Name); + string userField = EscapeIdentifier(EventIndexExtensions.DataPath(Event.KnownDataKeys.UserInfo, user => user.Identity) + ".keyword"); + var parameters = new List>>(); + AddParameter(parameters, "project_ids", request.ProjectIds.Select(FieldValue.String).ToArray()); + string query = new StringBuilder() + .Append("FROM ").Append(eventIndex) + .Append(" | KEEP stack_id, project_id, ").Append(userField) + .Append(" | RENAME project_id AS event_project_id, ").Append(userField).Append(" AS event_user") + .Append(" | LOOKUP JOIN ").Append(stackIndex).Append(" ON stack_id == id AND is_deleted == false") + .Append(" | WHERE id IS NOT NULL AND event_project_id IN (?project_ids)") + .Append(" | STATS users = COUNT_DISTINCT(event_user) BY project_id = event_project_id") + .Append(" | KEEP project_id, users") + .ToString(); + + using var response = await _client.Esql.QueryAsync(new EsqlQueryRequest(query) + { + AllowPartialResults = false, + Columnar = false, + Filter = sourceFilter, + Format = EsqlFormat.Json, + Params = new Union>, ICollection>>>(parameters) + }, cancellationToken); + + if (!response.IsValidResponse) + { + _logger.LogWarning("Stack rollup project user lookup join failed with Elasticsearch status {StatusCode}", response.ApiCallDetails?.HttpStatusCode); + throw new ApplicationException("The stack rollup project user query failed."); + } + + using var document = await JsonDocument.ParseAsync(response.Body, cancellationToken: cancellationToken); + return ReadProjectUserCounts(document.RootElement); + } + + private async Task EnsureReadyAsync(CancellationToken cancellationToken) + { + var readiness = await GetReadinessAsync(cancellationToken); + if (readiness.IsReady) + return; + + _logger.LogError("Stack rollup lookup join prerequisite failed: {Reason}", readiness.Reason); + throw new InvalidOperationException($"The stack rollup lookup join prerequisite failed: {readiness.Reason}."); + } + + private async Task BuildSourceFilterAsync(AppFilter? appFilter, DateTime utcStart, DateTime utcEnd, string? eventFilter) + { + var query = new RepositoryQuery() + .AppFilter(appFilter) + .DateRange(utcStart, utcEnd, (PersistentEvent e) => e.Date) + .FilterExpression(eventFilter); + + var options = new CommandOptions() + .TimeProvider(_timeProvider) + .ElasticIndex(_configuration.Events) + .DocumentType(typeof(PersistentEvent)); + var context = new QueryBuilderContext(query, options); + await _configuration.Events.QueryBuilder.BuildAsync(context); + return context.Filter; + } + + private string BuildEventQuery( + EventLookupSearchRequest request, + EventLookupSort sort, + EventLookupCursor? cursor, + string? stackFilter, + ICollection>> parameters) + { + string eventIndex = ValidateIndexName(_configuration.Events.Name); + string stackIndex = ValidateIndexName(_configuration.Stacks.Name); + var query = new StringBuilder() + .Append("FROM ").Append(eventIndex).Append(" METADATA _id") + .Append(" | KEEP _id, stack_id, date") + .Append(" | LOOKUP JOIN ").Append(stackIndex) + .Append(" ON stack_id == id AND is_deleted == false"); + + if (!String.IsNullOrWhiteSpace(stackFilter)) + { + query.Append(" AND QSTR(?stack_filter, {\"default_operator\": \"AND\"})"); + AddParameter(parameters, "stack_filter", FieldValue.String(stackFilter)); + } + + query.Append(" | WHERE id IS NOT NULL"); + if (request.IncludeTotal) + query.Append(" | INLINE STATS total_events = COUNT(*)"); + + bool isBefore = request.Before is not null; + if (cursor is not null) + { + string primaryComparison = isBefore + ? sort.Ascending ? "<" : ">" + : sort.Ascending ? ">" : "<"; + string idComparison = isBefore ? "<" : ">"; + query + .Append(" | WHERE date ").Append(primaryComparison).Append(" TO_DATETIME(?cursor_date)") + .Append(" OR (date == TO_DATETIME(?cursor_date) AND _id ").Append(idComparison).Append(" ?cursor_event_id)"); + AddParameter(parameters, "cursor_date", FieldValue.String(new DateTime(cursor.Date, DateTimeKind.Utc).ToString("O", CultureInfo.InvariantCulture))); + AddParameter(parameters, "cursor_event_id", FieldValue.String(cursor.EventId)); + } + + bool queryAscending = isBefore ? !sort.Ascending : sort.Ascending; + query + .Append(" | SORT date ").Append(queryAscending ? "ASC" : "DESC") + .Append(", _id ").Append(isBefore ? "DESC" : "ASC") + .Append(" | LIMIT ").Append(request.Limit + 1) + .Append(" | KEEP _id, date"); + if (request.IncludeTotal) + query.Append(", total_events"); + + return query.ToString(); + } + + private async Task> ExecuteEventRowsAsync( + string query, + Query? sourceFilter, + ICollection>> parameters, + CancellationToken cancellationToken) + { + using var response = await _client.Esql.QueryAsync(new EsqlQueryRequest(query) + { + AllowPartialResults = false, + Columnar = false, + Filter = sourceFilter, + Format = EsqlFormat.Json, + Params = new Union>, ICollection>>>(parameters) + }, cancellationToken); + + if (!response.IsValidResponse) + { + _logger.LogWarning("Event lookup join failed with Elasticsearch status {StatusCode}", response.ApiCallDetails?.HttpStatusCode); + throw new ApplicationException("The event lookup join query failed."); + } + + using var document = await JsonDocument.ParseAsync(response.Body, cancellationToken: cancellationToken); + if (!document.RootElement.TryGetProperty("columns", out var columns) || !document.RootElement.TryGetProperty("values", out var values)) + throw new JsonException("The ES|QL event lookup response did not contain columns and values."); + + var columnIndexes = GetColumnIndexes(columns); + int idIndex = columnIndexes["_id"]; + int dateIndex = columnIndexes["date"]; + int? totalIndex = columnIndexes.TryGetValue("total_events", out int index) ? index : null; + return values.EnumerateArray() + .Select(value => new EventLookupEsqlRow( + value[idIndex].GetString() ?? throw new JsonException("An event lookup row did not contain an event id."), + value[dateIndex].GetDateTime().ToUniversalTime(), + totalIndex.HasValue ? value[totalIndex.Value].GetInt64() : null)) + .ToList(); + } + + private Task ExecuteJoinedEventTotalAsync(string? stackFilter, Query? sourceFilter, CancellationToken cancellationToken) + { + string eventIndex = ValidateIndexName(_configuration.Events.Name); + string stackIndex = ValidateIndexName(_configuration.Stacks.Name); + var parameters = new List>>(); + var query = new StringBuilder() + .Append("FROM ").Append(eventIndex) + .Append(" | KEEP stack_id") + .Append(" | LOOKUP JOIN ").Append(stackIndex) + .Append(" ON stack_id == id AND is_deleted == false"); + if (!String.IsNullOrWhiteSpace(stackFilter)) + { + query.Append(" AND QSTR(?stack_filter, {\"default_operator\": \"AND\"})"); + AddParameter(parameters, "stack_filter", FieldValue.String(stackFilter)); + } + query.Append(" | WHERE id IS NOT NULL | STATS total_events = COUNT(*) | KEEP total_events"); + return ExecuteTotalAsync(query.ToString(), sourceFilter, parameters, cancellationToken); + } + + private async Task CountEventTagsAsync(string? stackFilter, Query? sourceFilter, CancellationToken cancellationToken) + { + string eventIndex = ValidateIndexName(_configuration.Events.Name); + string stackIndex = ValidateIndexName(_configuration.Stacks.Name); + var parameters = new List>>(); + var query = new StringBuilder() + .Append("FROM ").Append(eventIndex) + .Append(" | KEEP stack_id, tags") + .Append(" | LOOKUP JOIN ").Append(stackIndex) + .Append(" ON stack_id == id AND is_deleted == false"); + if (!String.IsNullOrWhiteSpace(stackFilter)) + { + query.Append(" AND QSTR(?stack_filter, {\"default_operator\": \"AND\"})"); + AddParameter(parameters, "stack_filter", FieldValue.String(stackFilter)); + } + query + .Append(" | WHERE id IS NOT NULL") + .Append(" | MV_EXPAND tags") + .Append(" | STATS tag_total = COUNT(*) BY tag = tags") + .Append(" | SORT tag_total DESC, tag ASC | LIMIT 1000 | KEEP tag, tag_total"); + + using var response = await _client.Esql.QueryAsync(new EsqlQueryRequest(query.ToString()) + { + AllowPartialResults = false, + Columnar = false, + Filter = sourceFilter, + Format = EsqlFormat.Json, + Params = new Union>, ICollection>>>(parameters) + }, cancellationToken); + if (!response.IsValidResponse) + throw new ApplicationException("The event tag lookup join query failed."); + + using var document = await JsonDocument.ParseAsync(response.Body, cancellationToken: cancellationToken); + if (!document.RootElement.TryGetProperty("columns", out var columns) || !document.RootElement.TryGetProperty("values", out var values)) + throw new JsonException("The ES|QL event tag response did not contain columns and values."); + var columnIndexes = GetColumnIndexes(columns); + int tagIndex = columnIndexes["tag"]; + int tagTotalIndex = columnIndexes["tag_total"]; + var buckets = new List(); + foreach (var value in values.EnumerateArray()) + { + buckets.Add(new KeyedBucket(null) + { + Key = value[tagIndex].GetString() ?? String.Empty, + Total = value[tagTotalIndex].GetInt64(), + Data = new Dictionary { ["@type"] = "string" } + }); + } + + long total = await ExecuteJoinedEventTotalAsync(stackFilter, sourceFilter, cancellationToken); + return new CountResult(total, new Dictionary + { + ["terms_tags"] = new BucketAggregate + { + Items = buckets, + Data = new Dictionary { ["@type"] = "bucket" } + } + }); + } + + private string BuildQuery( + StackRollupSearchRequest request, + StackRollupSort sort, + StackRollupCursor? cursor, + string? stackFilter, + ICollection>> parameters, + bool countOnly) + { + string eventIndex = ValidateIndexName(_configuration.Events.Name); + string stackIndex = ValidateIndexName(_configuration.Stacks.Name); + string userField = EscapeIdentifier(EventIndexExtensions.DataPath(Event.KnownDataKeys.UserInfo, user => user.Identity) + ".keyword"); + var query = new StringBuilder() + .Append("FROM ").Append(eventIndex) + .Append(" | KEEP stack_id, count, date, ").Append(userField) + .Append(" | RENAME ").Append(userField).Append(" AS event_user") + .Append(" | LOOKUP JOIN ").Append(stackIndex) + .Append(" ON stack_id == id AND is_deleted == false"); + + if (!String.IsNullOrWhiteSpace(stackFilter)) + { + query.Append(" AND QSTR(?stack_filter, {\"default_operator\": \"AND\"})"); + AddParameter(parameters, "stack_filter", FieldValue.String(stackFilter)); + } + + query + .Append(" | WHERE id IS NOT NULL") + .Append(" | STATS event_total = SUM(COALESCE(count, 1)), event_users = COUNT_DISTINCT(event_user), event_first = MIN(date), event_last = MAX(date) BY stack_id"); + + if (countOnly) + return query.Append(" | STATS total_stacks = COUNT(*) | KEEP total_stacks").ToString(); + + if (request.IncludeTotal) + query.Append(" | INLINE STATS total_stacks = COUNT(*)"); + + bool isBefore = request.Before is not null; + if (cursor is not null) + { + string primaryComparison = isBefore + ? sort.Ascending ? "<" : ">" + : sort.Ascending ? ">" : "<"; + string idComparison = isBefore ? "<" : ">"; + string metricParameter = sort.IsDate ? "TO_DATETIME(?cursor_metric)" : "?cursor_metric"; + query + .Append(" | WHERE ").Append(sort.Metric).Append(' ').Append(primaryComparison).Append(' ').Append(metricParameter) + .Append(" OR (").Append(sort.Metric).Append(" == ").Append(metricParameter) + .Append(" AND stack_id ").Append(idComparison).Append(" ?cursor_stack_id)"); + AddParameter(parameters, "cursor_metric", sort.IsDate + ? FieldValue.String(new DateTime(cursor.Metric, DateTimeKind.Utc).ToString("O", CultureInfo.InvariantCulture)) + : FieldValue.Long(cursor.Metric)); + AddParameter(parameters, "cursor_stack_id", FieldValue.String(cursor.StackId)); + } + + bool queryAscending = isBefore ? !sort.Ascending : sort.Ascending; + string primarySort = queryAscending ? "ASC" : "DESC"; + string idSort = isBefore ? "DESC" : "ASC"; + query + .Append(" | SORT ").Append(sort.Metric).Append(' ').Append(primarySort).Append(", stack_id ").Append(idSort) + .Append(" | LIMIT ").Append(request.Limit + 1) + .Append(" | KEEP stack_id, event_total, event_users, event_first, event_last"); + + if (request.IncludeTotal) + query.Append(", total_stacks"); + + return query.ToString(); + } + + private string BuildStatsQuery( + StackRollupStatsRequest request, + string? stackFilter, + ICollection>> parameters) + { + string eventIndex = ValidateIndexName(_configuration.Events.Name); + string stackIndex = ValidateIndexName(_configuration.Stacks.Name); + int bucketCount = Math.Clamp(request.BucketCount, 1, 100); + string timeZone = FormatTimeZone(request.Offset); + string utcStart = request.UtcStart.ToString("O", CultureInfo.InvariantCulture); + string utcEnd = request.UtcEnd.ToString("O", CultureInfo.InvariantCulture); + var query = new StringBuilder() + .Append("SET time_zone = \"").Append(timeZone).Append("\"; FROM ").Append(eventIndex) + .Append(" | KEEP stack_id, count, date, is_first_occurrence") + .Append(" | LOOKUP JOIN ").Append(stackIndex) + .Append(" ON stack_id == id AND is_deleted == false"); + + if (!String.IsNullOrWhiteSpace(stackFilter)) + { + query.Append(" AND QSTR(?stack_filter, {\"default_operator\": \"AND\"})"); + AddParameter(parameters, "stack_filter", FieldValue.String(stackFilter)); + } + + return query + .Append(" | WHERE id IS NOT NULL") + .Append(" | INLINE STATS total_documents = COUNT(*), total_events = SUM(COALESCE(count, 1)), total_stacks = COUNT_DISTINCT(stack_id, 40000), new_stacks = SUM(CASE(is_first_occurrence, 1, 0))") + .Append(" | STATS documents = COUNT(*), events = SUM(COALESCE(count, 1)), stacks = COUNT_DISTINCT(stack_id), total_documents = MAX(total_documents), total_events = MAX(total_events), total_stacks = MAX(total_stacks), new_stacks = MAX(new_stacks)") + .Append(" BY bucket = BUCKET(date, ").Append(bucketCount).Append(", \"").Append(utcStart).Append("\", \"").Append(utcEnd).Append("\")") + .Append(" | SORT bucket | LIMIT ").Append(bucketCount + 2) + .Append(" | KEEP bucket, documents, events, stacks, total_documents, total_events, total_stacks, new_stacks") + .ToString(); + } + + private async Task> ExecuteRowsAsync( + string query, + Query? sourceFilter, + ICollection>> parameters, + CancellationToken cancellationToken) + { + using var response = await _client.Esql.QueryAsync(new EsqlQueryRequest(query) + { + AllowPartialResults = false, + Columnar = false, + Filter = sourceFilter, + Format = EsqlFormat.Json, + Params = new Union>, ICollection>>>(parameters) + }, cancellationToken); + + if (!response.IsValidResponse) + { + _logger.LogWarning("Stack rollup lookup join failed with Elasticsearch status {StatusCode}", response.ApiCallDetails?.HttpStatusCode); + throw new ApplicationException("The experimental stack rollup query failed."); + } + + using var document = await JsonDocument.ParseAsync(response.Body, cancellationToken: cancellationToken); + return ReadRows(document.RootElement); + } + + private async Task ExecuteTotalAsync( + string query, + Query? sourceFilter, + ICollection>> parameters, + CancellationToken cancellationToken) + { + using var response = await _client.Esql.QueryAsync(new EsqlQueryRequest(query) + { + AllowPartialResults = false, + Columnar = false, + Filter = sourceFilter, + Format = EsqlFormat.Json, + Params = new Union>, ICollection>>>(parameters) + }, cancellationToken); + + if (!response.IsValidResponse) + throw new ApplicationException("The experimental stack rollup count query failed."); + + using var document = await JsonDocument.ParseAsync(response.Body, cancellationToken: cancellationToken); + if (!document.RootElement.TryGetProperty("values", out var values) || values.GetArrayLength() == 0) + return 0; + + return values[0][0].GetInt64(); + } + + private static List ReadRows(JsonElement root) + { + if (!root.TryGetProperty("columns", out var columns) || !root.TryGetProperty("values", out var values)) + throw new JsonException("The ES|QL stack rollup response did not contain columns and values."); + + var columnIndexes = columns.EnumerateArray() + .Select((column, index) => (Name: column.GetProperty("name").GetString(), Index: index)) + .Where(column => column.Name is not null) + .ToDictionary(column => column.Name!, column => column.Index, StringComparer.Ordinal); + + int stackIdIndex = columnIndexes["stack_id"]; + int totalIndex = columnIndexes["event_total"]; + int usersIndex = columnIndexes["event_users"]; + int firstIndex = columnIndexes["event_first"]; + int lastIndex = columnIndexes["event_last"]; + int? totalStacksIndex = columnIndexes.TryGetValue("total_stacks", out int index) ? index : null; + var rows = new List(); + foreach (var value in values.EnumerateArray()) + { + rows.Add(new StackRollupEsqlRow( + value[stackIdIndex].GetString() ?? throw new JsonException("A stack rollup row did not contain a stack id."), + value[totalIndex].GetInt64(), + value[usersIndex].GetInt64(), + value[firstIndex].GetDateTime().ToUniversalTime(), + value[lastIndex].GetDateTime().ToUniversalTime(), + totalStacksIndex.HasValue ? value[totalStacksIndex.Value].GetInt64() : null)); + } + + return rows; + } + + private static StackRollupStatsResult ReadStats(JsonElement root) + { + if (!root.TryGetProperty("columns", out var columns) || !root.TryGetProperty("values", out var values)) + throw new JsonException("The ES|QL stack rollup stats response did not contain columns and values."); + if (values.GetArrayLength() == 0) + return new StackRollupStatsResult(0, 0, 0, []); + + var columnIndexes = GetColumnIndexes(columns); + int bucketIndex = columnIndexes["bucket"]; + int documentsIndex = columnIndexes["documents"]; + int eventsIndex = columnIndexes["events"]; + int stacksIndex = columnIndexes["stacks"]; + int totalDocumentsIndex = columnIndexes["total_documents"]; + int totalEventsIndex = columnIndexes["total_events"]; + int totalStacksIndex = columnIndexes["total_stacks"]; + int newStacksIndex = columnIndexes["new_stacks"]; + var buckets = new List(values.GetArrayLength()); + long totalEvents = 0; + long totalStacks = 0; + long newStacks = 0; + long totalDocuments = 0; + foreach (var value in values.EnumerateArray()) + { + totalDocuments = value[totalDocumentsIndex].GetInt64(); + totalEvents = value[totalEventsIndex].GetInt64(); + totalStacks = value[totalStacksIndex].GetInt64(); + newStacks = value[newStacksIndex].GetInt64(); + buckets.Add(new StackRollupStatsBucket( + value[bucketIndex].GetDateTimeOffset().UtcDateTime, + value[eventsIndex].GetInt64(), + value[stacksIndex].GetInt64(), + value[documentsIndex].GetInt64())); + } + + return new StackRollupStatsResult(totalEvents, totalStacks, newStacks, buckets, totalDocuments); + } + + private static IReadOnlyDictionary ReadProjectUserCounts(JsonElement root) + { + if (!root.TryGetProperty("columns", out var columns) || !root.TryGetProperty("values", out var values)) + throw new JsonException("The ES|QL project user response did not contain columns and values."); + + var columnIndexes = GetColumnIndexes(columns); + int projectIndex = columnIndexes["project_id"]; + int usersIndex = columnIndexes["users"]; + var result = new Dictionary(StringComparer.Ordinal); + foreach (var value in values.EnumerateArray()) + { + string projectId = value[projectIndex].GetString() ?? throw new JsonException("A project user row did not contain a project id."); + result[projectId] = value[usersIndex].GetInt64(); + } + + return result; + } + + private static IReadOnlyDictionary GetColumnIndexes(JsonElement columns) + => columns.EnumerateArray() + .Select((column, index) => (Name: column.GetProperty("name").GetString(), Index: index)) + .Where(column => column.Name is not null) + .ToDictionary(column => column.Name!, column => column.Index, StringComparer.Ordinal); + + private static bool IsDashboardAggregation(string aggregations) + { + string value = aggregations.Replace(" ", String.Empty, StringComparison.Ordinal).ToLowerInvariant(); + return value.Contains("date:(date", StringComparison.Ordinal) + && value.Contains("cardinality:stack", StringComparison.Ordinal) + && value.Contains("sum:count~1", StringComparison.Ordinal) + && value.Contains("terms:(first@include:true)", StringComparison.Ordinal); + } + + private static bool IsTermsTagsAggregation(string aggregations) + => String.Equals(aggregations.Replace(" ", String.Empty, StringComparison.Ordinal), "terms:tags", StringComparison.OrdinalIgnoreCase); + + private static CountResult ToCountResult(StackRollupStatsResult stats) + { + static ValueAggregate Metric(double value) => new() + { + Value = value, + Data = new Dictionary { ["@type"] = "value" } + }; + + var dateBuckets = stats.Buckets.Select(bucket => (IBucket)new DateHistogramBucket(bucket.Date, new Dictionary + { + ["cardinality_stack"] = Metric(bucket.Stacks), + ["sum_count"] = Metric(bucket.Events) + }) + { + Key = new DateTimeOffset(bucket.Date).ToUnixTimeMilliseconds(), + KeyAsString = bucket.Date.ToString("O", CultureInfo.InvariantCulture), + Total = bucket.Documents, + Data = new Dictionary { ["@type"] = "datehistogram" } + }).ToArray(); + + var firstBuckets = new IBucket[] + { + new KeyedBucket(null) + { + Key = true, + Total = stats.NewStacks, + Data = new Dictionary { ["@type"] = "bool" } + } + }; + + return new CountResult(stats.Documents, new Dictionary + { + ["date_date"] = new BucketAggregate + { + Items = dateBuckets, + Data = new Dictionary { ["@type"] = "bucket" } + }, + ["cardinality_stack"] = Metric(stats.TotalStacks), + ["terms_first"] = new BucketAggregate + { + Items = firstBuckets, + Data = new Dictionary { ["@type"] = "bucket" } + }, + ["sum_count"] = Metric(stats.TotalEvents) + }); + } + + private async Task GetReadinessAsync(CancellationToken cancellationToken) + { + DateTimeOffset now = _timeProvider.GetUtcNow(); + if (_readiness is not null && now < _readinessExpiresUtc) + return _readiness; + + await _readinessLock.WaitAsync(cancellationToken); + try + { + now = _timeProvider.GetUtcNow(); + if (_readiness is not null && now < _readinessExpiresUtc) + return _readiness; + + _readiness = await CheckReadinessAsync(cancellationToken); + _readinessExpiresUtc = now.Add(ReadinessCacheDuration); + return _readiness; + } + finally + { + _readinessLock.Release(); + } + } + + private async Task CheckReadinessAsync(CancellationToken cancellationToken) + { + var info = await _client.InfoAsync(cancellationToken); + if (!info.IsValidResponse || !System.Version.TryParse(info.Version.Number, out var version) || version < new System.Version(9, 5)) + return new StackRollupReadiness(false, "elasticsearch-version"); + + var settings = await _client.Indices.GetSettingsAsync((Indices)_configuration.Stacks.Name, cancellationToken); + if (!settings.IsValidResponse || settings.Settings.Count != 1) + return new StackRollupReadiness(false, "stack-alias-target"); + + var indexSettings = settings.Settings.Single().Value.Settings?.Index; + if (indexSettings is null || !String.Equals(indexSettings.Mode, "lookup", StringComparison.OrdinalIgnoreCase)) + return new StackRollupReadiness(false, "stack-index-mode"); + + int shards = indexSettings.NumberOfShards is null + ? 0 + : indexSettings.NumberOfShards.Match(value => value, value => Int32.TryParse(value, out int parsed) ? parsed : 0); + return shards == 1 + ? new StackRollupReadiness(true, "ready") + : new StackRollupReadiness(false, "stack-primary-shards"); + } + + private string EncodeCursor(StackRollupEsqlRow row, StackRollupSort sort, DateTime utcStart, DateTime utcEnd, string fingerprint) + { + long metric = sort.Metric switch + { + "event_total" => row.Total, + "event_users" => row.Users, + "event_first" => row.FirstOccurrence.Ticks, + "event_last" => row.LastOccurrence.Ticks, + _ => throw new InvalidOperationException("Unsupported stack rollup metric.") + }; + var cursor = new StackRollupCursor(CursorVersion, sort.Value, metric, row.StackId, utcStart.Ticks, utcEnd.Ticks, fingerprint); + byte[] bytes = JsonSerializer.SerializeToUtf8Bytes(cursor, _serializerOptions); + return Convert.ToBase64String(bytes).TrimEnd('=').Replace('+', '-').Replace('/', '_'); + } + + private string EncodeEventCursor(EventLookupEsqlRow row, EventLookupSort sort, DateTime utcStart, DateTime utcEnd, string fingerprint) + { + var cursor = new EventLookupCursor(CursorVersion, sort.Value, row.Date.Ticks, row.EventId, utcStart.Ticks, utcEnd.Ticks, fingerprint); + byte[] bytes = JsonSerializer.SerializeToUtf8Bytes(cursor, _serializerOptions); + return Convert.ToBase64String(bytes).TrimEnd('=').Replace('+', '-').Replace('/', '_'); + } + + private EventLookupCursor? DecodeEventCursor(string? token, string sort, string fingerprint) + { + if (String.IsNullOrWhiteSpace(token)) + return null; + + try + { + string base64 = token.Replace('-', '+').Replace('_', '/'); + base64 = base64.PadRight(base64.Length + (4 - base64.Length % 4) % 4, '='); + var cursor = JsonSerializer.Deserialize(Convert.FromBase64String(base64), _serializerOptions); + if (cursor is null + || cursor.Version != CursorVersion + || !String.Equals(cursor.Sort, sort, StringComparison.Ordinal) + || !String.Equals(cursor.Fingerprint, fingerprint, StringComparison.Ordinal) + || String.IsNullOrWhiteSpace(cursor.EventId) + || cursor.Date < DateTime.MinValue.Ticks + || cursor.Date > DateTime.MaxValue.Ticks + || cursor.UtcStart < DateTime.MinValue.Ticks + || cursor.UtcStart > DateTime.MaxValue.Ticks + || cursor.UtcEnd < DateTime.MinValue.Ticks + || cursor.UtcEnd > DateTime.MaxValue.Ticks + || cursor.UtcStart > cursor.UtcEnd) + { + throw new InvalidEventLookupCursorException("The event pagination cursor is not valid for this query."); + } + + return cursor; + } + catch (InvalidEventLookupCursorException) + { + throw; + } + catch (Exception ex) when (ex is FormatException or JsonException or OverflowException) + { + throw new InvalidEventLookupCursorException("The event pagination cursor is malformed."); + } + } + + private StackRollupCursor? DecodeCursor(string? token, string sort, string fingerprint) + { + if (String.IsNullOrWhiteSpace(token)) + return null; + + try + { + string base64 = token.Replace('-', '+').Replace('_', '/'); + base64 = base64.PadRight(base64.Length + (4 - base64.Length % 4) % 4, '='); + var cursor = JsonSerializer.Deserialize(Convert.FromBase64String(base64), _serializerOptions); + if (cursor is null + || cursor.Version != CursorVersion + || !String.Equals(cursor.Sort, sort, StringComparison.Ordinal) + || !String.Equals(cursor.Fingerprint, fingerprint, StringComparison.Ordinal) + || String.IsNullOrWhiteSpace(cursor.StackId) + || cursor.UtcStart < DateTime.MinValue.Ticks + || cursor.UtcStart > DateTime.MaxValue.Ticks + || cursor.UtcEnd < DateTime.MinValue.Ticks + || cursor.UtcEnd > DateTime.MaxValue.Ticks + || cursor.UtcStart > cursor.UtcEnd + || GetSort(sort).IsDate && (cursor.Metric < DateTime.MinValue.Ticks || cursor.Metric > DateTime.MaxValue.Ticks)) + { + throw new InvalidStackRollupCursorException("The stack pagination cursor is not valid for this query."); + } + + return cursor; + } + catch (InvalidStackRollupCursorException) + { + throw; + } + catch (Exception ex) when (ex is FormatException or JsonException or OverflowException) + { + throw new InvalidStackRollupCursorException("The stack pagination cursor is malformed."); + } + } + + private static string CreateFingerprint(StackRollupSearchRequest request) + { + string organizations = String.Join(',', request.AppFilter?.Organizations.Select(organization => organization.Id).Order(StringComparer.Ordinal) ?? Enumerable.Empty()); + string projects = String.Join(',', request.AppFilter?.Projects?.Select(project => project.Id).Order(StringComparer.Ordinal) ?? Enumerable.Empty()); + string value = String.Join('\n', [ + GetSort(request.Sort).Value, + request.Filter ?? String.Empty, + request.TimeExpression ?? String.Empty, + request.Offset.Ticks.ToString(CultureInfo.InvariantCulture), + organizations, + projects, + request.AppFilter?.Stack?.Id ?? String.Empty + ]); + return Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(value))); + } + + private static string CreateEventFingerprint(EventLookupSearchRequest request, string sort) + { + string organizations = String.Join(',', request.AppFilter?.Organizations.Select(organization => organization.Id).Order(StringComparer.Ordinal) ?? Enumerable.Empty()); + string projects = String.Join(',', request.AppFilter?.Projects?.Select(project => project.Id).Order(StringComparer.Ordinal) ?? Enumerable.Empty()); + string value = String.Join('\n', [ + sort, + request.Filter ?? String.Empty, + request.TimeExpression ?? String.Empty, + organizations, + projects, + request.AppFilter?.Stack?.Id ?? String.Empty + ]); + return Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(value))); + } + + private static StackRollupRow ToPublicRow(StackRollupEsqlRow row) => new( + row.StackId, + row.Total, + row.Users, + row.FirstOccurrence, + row.LastOccurrence); + + private static StackRollupSort GetSort(string? sort) => (String.IsNullOrWhiteSpace(sort) ? "-total" : sort.Trim()) switch + { + "total" => new StackRollupSort("total", "event_total", false, true), + "-total" => new StackRollupSort("-total", "event_total", false, false), + "users" => new StackRollupSort("users", "event_users", false, true), + "-users" => new StackRollupSort("-users", "event_users", false, false), + "first_occurrence" => new StackRollupSort("first_occurrence", "event_first", true, true), + "-first_occurrence" => new StackRollupSort("-first_occurrence", "event_first", true, false), + "last_occurrence" => new StackRollupSort("last_occurrence", "event_last", true, true), + "-last_occurrence" => new StackRollupSort("-last_occurrence", "event_last", true, false), + _ => throw new ArgumentOutOfRangeException(nameof(sort), sort, "Unsupported stack rollup sort.") + }; + + private static EventLookupSort GetEventSort(string? sort) => (String.IsNullOrWhiteSpace(sort) ? "-date" : sort.Trim()) switch + { + "date" => new EventLookupSort("date", true), + "-date" => new EventLookupSort("-date", false), + _ => throw new ArgumentOutOfRangeException(nameof(sort), sort, "Unsupported event lookup sort.") + }; + + private static string StripAlternateInversion(string? filter) => filter?.StartsWith("@!", StringComparison.Ordinal) == true ? filter[2..] : filter ?? String.Empty; + + private static string ValidateIndexName(string index) + { + if (String.IsNullOrWhiteSpace(index) || index.Any(character => !Char.IsAsciiLetterOrDigit(character) && character is not '-' and not '_' and not '.')) + throw new InvalidOperationException("The configured Elasticsearch index alias cannot be used in ES|QL."); + + return index; + } + + private static string EscapeIdentifier(string field) => $"`{field.Replace("`", "``", StringComparison.Ordinal)}`"; + + private static string FormatTimeZone(TimeSpan offset) + { + if (offset < TimeSpan.FromHours(-14) || offset > TimeSpan.FromHours(14)) + throw new ArgumentOutOfRangeException(nameof(offset), offset, "The stack rollup time zone offset must be between -14:00 and +14:00."); + + string sign = offset < TimeSpan.Zero ? "-" : "+"; + var absolute = offset.Duration(); + return $"{sign}{(int)absolute.TotalHours:00}:{absolute.Minutes:00}"; + } + + private static void AddParameter(ICollection>> parameters, string name, FieldValue value) + => parameters.Add(new KeyValuePair>(name, [value])); + + private static void AddParameter(ICollection>> parameters, string name, ICollection values) + => parameters.Add(new KeyValuePair>(name, values)); + + private sealed record StackRollupSort(string Value, string Metric, bool IsDate, bool Ascending); + private sealed record EventLookupSort(string Value, bool Ascending); + private sealed record StackRollupReadiness(bool IsReady, string Reason); + private sealed record StackRollupCursor(int Version, string Sort, long Metric, string StackId, long UtcStart, long UtcEnd, string Fingerprint); + private sealed record EventLookupCursor(int Version, string Sort, long Date, string EventId, long UtcStart, long UtcEnd, string Fingerprint); + private sealed record EventLookupEsqlRow(string EventId, DateTime Date, long? TotalEvents); + private sealed record StackRollupEsqlRow( + string StackId, + long Total, + long Users, + DateTime FirstOccurrence, + DateTime LastOccurrence, + long? TotalStacks); +} diff --git a/src/Exceptionless.Web/Api/Endpoints/EventEndpoints.cs b/src/Exceptionless.Web/Api/Endpoints/EventEndpoints.cs index 05ad674605..ca1e687064 100644 --- a/src/Exceptionless.Web/Api/Endpoints/EventEndpoints.cs +++ b/src/Exceptionless.Web/Api/Endpoints/EventEndpoints.cs @@ -39,7 +39,7 @@ public static IEndpointRouteBuilder MapEventEndpoints(this IEndpointRouteBuilder ["aggregations"] = "A list of values you want returned. Example: avg:value cardinality:value sum:users max:value min:value", ["time"] = "The time filter that limits the data being returned to a specific date range.", ["offset"] = "The time offset in minutes that controls what data is returned based on the time filter. This is used for time zone support.", - ["mode"] = "If no mode is set then the whole event object will be returned. If the mode is set to summary than a lightweight object will be returned.", + ["mode"] = "Set to stack to calculate the stack-list dashboard metrics with a lookup join.", }, ResponseDescriptions = new() { ["400"] = "Invalid filter.", @@ -61,7 +61,7 @@ public static IEndpointRouteBuilder MapEventEndpoints(this IEndpointRouteBuilder ["aggregations"] = "A list of values you want returned. Example: avg:value cardinality:value sum:users max:value min:value", ["time"] = "The time filter that limits the data being returned to a specific date range.", ["offset"] = "The time offset in minutes that controls what data is returned based on the time filter. This is used for time zone support.", - ["mode"] = "If no mode is set then the whole event object will be returned. If the mode is set to summary than a lightweight object will be returned.", + ["mode"] = "Set to stack to calculate the stack-list dashboard metrics with a lookup join.", }, ResponseDescriptions = new() { ["400"] = "Invalid filter.", @@ -83,7 +83,7 @@ public static IEndpointRouteBuilder MapEventEndpoints(this IEndpointRouteBuilder ["aggregations"] = "A list of values you want returned. Example: avg:value cardinality:value sum:users max:value min:value", ["time"] = "The time filter that limits the data being returned to a specific date range.", ["offset"] = "The time offset in minutes that controls what data is returned based on the time filter. This is used for time zone support.", - ["mode"] = "If mode is set to stack_new, then additional filters will be added.", + ["mode"] = "Set to stack to calculate the stack-list dashboard metrics with a lookup join.", }, ResponseDescriptions = new() { ["400"] = "Invalid filter.", @@ -129,7 +129,7 @@ public static IEndpointRouteBuilder MapEventEndpoints(this IEndpointRouteBuilder ["sort"] = "Controls the sort order that the data is returned in. In this example -date returns the results descending by date.", ["time"] = "The time filter that limits the data being returned to a specific date range.", ["offset"] = "The time offset in minutes that controls what data is returned based on the time filter. This is used for time zone support.", - ["mode"] = "If no mode is set then the whole event object will be returned. If the mode is set to summary than a lightweight object will be returned.", + ["mode"] = "If no mode is set, the whole event object is returned. Summary returns a lightweight event object; stack returns event metrics grouped into stack summaries.", ["page"] = "The page parameter is used for pagination. This value must be greater than 0.", ["limit"] = "A limit on the number of objects to be returned. Limit can range between 1 and 100 items.", ["before"] = "The before parameter is a cursor used for pagination and defines your place in the list of results.", @@ -158,7 +158,7 @@ public static IEndpointRouteBuilder MapEventEndpoints(this IEndpointRouteBuilder ["sort"] = "Controls the sort order that the data is returned in. In this example -date returns the results descending by date.", ["time"] = "The time filter that limits the data being returned to a specific date range.", ["offset"] = "The time offset in minutes that controls what data is returned based on the time filter. This is used for time zone support.", - ["mode"] = "If no mode is set then the whole event object will be returned. If the mode is set to summary than a lightweight object will be returned.", + ["mode"] = "If no mode is set, the whole event object is returned. Summary returns a lightweight event object; stack returns event metrics grouped into stack summaries.", ["page"] = "The page parameter is used for pagination. This value must be greater than 0.", ["limit"] = "A limit on the number of objects to be returned. Limit can range between 1 and 100 items.", ["before"] = "The before parameter is a cursor used for pagination and defines your place in the list of results.", @@ -188,7 +188,7 @@ public static IEndpointRouteBuilder MapEventEndpoints(this IEndpointRouteBuilder ["sort"] = "Controls the sort order that the data is returned in. In this example -date returns the results descending by date.", ["time"] = "The time filter that limits the data being returned to a specific date range.", ["offset"] = "The time offset in minutes that controls what data is returned based on the time filter. This is used for time zone support.", - ["mode"] = "If no mode is set then the whole event object will be returned. If the mode is set to summary than a lightweight object will be returned.", + ["mode"] = "If no mode is set, the whole event object is returned. Summary returns a lightweight event object; stack returns event metrics grouped into stack summaries.", ["page"] = "The page parameter is used for pagination. This value must be greater than 0.", ["limit"] = "A limit on the number of objects to be returned. Limit can range between 1 and 100 items.", ["before"] = "The before parameter is a cursor used for pagination and defines your place in the list of results.", @@ -218,7 +218,7 @@ public static IEndpointRouteBuilder MapEventEndpoints(this IEndpointRouteBuilder ["sort"] = "Controls the sort order that the data is returned in. In this example -date returns the results descending by date.", ["time"] = "The time filter that limits the data being returned to a specific date range.", ["offset"] = "The time offset in minutes that controls what data is returned based on the time filter. This is used for time zone support.", - ["mode"] = "If no mode is set then the whole event object will be returned. If the mode is set to summary than a lightweight object will be returned.", + ["mode"] = "If no mode is set, the whole event object is returned. Summary returns a lightweight event object; stack returns event metrics grouped into stack summaries.", ["page"] = "The page parameter is used for pagination. This value must be greater than 0.", ["limit"] = "A limit on the number of objects to be returned. Limit can range between 1 and 100 items.", ["before"] = "The before parameter is a cursor used for pagination and defines your place in the list of results.", @@ -244,7 +244,7 @@ public static IEndpointRouteBuilder MapEventEndpoints(this IEndpointRouteBuilder ParameterDescriptions = new() { ["referenceId"] = "An identifier used that references an event instance.", ["offset"] = "The time offset in minutes that controls what data is returned based on the time filter. This is used for time zone support.", - ["mode"] = "If no mode is set then the whole event object will be returned. If the mode is set to summary than a lightweight object will be returned.", + ["mode"] = "If no mode is set, the whole event object is returned. Summary returns a lightweight event object; stack returns event metrics grouped into stack summaries.", ["page"] = "The page parameter is used for pagination. This value must be greater than 0.", ["limit"] = "A limit on the number of objects to be returned. Limit can range between 1 and 100 items.", ["before"] = "The before parameter is a cursor used for pagination and defines your place in the list of results.", @@ -271,7 +271,7 @@ public static IEndpointRouteBuilder MapEventEndpoints(this IEndpointRouteBuilder ["referenceId"] = "An identifier used that references an event instance.", ["projectId"] = "The identifier of the project.", ["offset"] = "The time offset in minutes that controls what data is returned based on the time filter. This is used for time zone support.", - ["mode"] = "If no mode is set then the whole event object will be returned. If the mode is set to summary than a lightweight object will be returned.", + ["mode"] = "If no mode is set, the whole event object is returned. Summary returns a lightweight event object; stack returns event metrics grouped into stack summaries.", ["page"] = "The page parameter is used for pagination. This value must be greater than 0.", ["limit"] = "A limit on the number of objects to be returned. Limit can range between 1 and 100 items.", ["before"] = "The before parameter is a cursor used for pagination and defines your place in the list of results.", @@ -300,7 +300,7 @@ public static IEndpointRouteBuilder MapEventEndpoints(this IEndpointRouteBuilder ["sort"] = "Controls the sort order that the data is returned in. In this example -date returns the results descending by date.", ["time"] = "The time filter that limits the data being returned to a specific date range.", ["offset"] = "The time offset in minutes that controls what data is returned based on the time filter. This is used for time zone support.", - ["mode"] = "If no mode is set then the whole event object will be returned. If the mode is set to summary than a lightweight object will be returned.", + ["mode"] = "If no mode is set, the whole event object is returned. Summary returns a lightweight event object; stack returns event metrics grouped into stack summaries.", ["page"] = "The page parameter is used for pagination. This value must be greater than 0.", ["limit"] = "A limit on the number of objects to be returned. Limit can range between 1 and 100 items.", ["before"] = "The before parameter is a cursor used for pagination and defines your place in the list of results.", @@ -330,7 +330,7 @@ public static IEndpointRouteBuilder MapEventEndpoints(this IEndpointRouteBuilder ["sort"] = "Controls the sort order that the data is returned in. In this example -date returns the results descending by date.", ["time"] = "The time filter that limits the data being returned to a specific date range.", ["offset"] = "The time offset in minutes that controls what data is returned based on the time filter. This is used for time zone support.", - ["mode"] = "If no mode is set then the whole event object will be returned. If the mode is set to summary than a lightweight object will be returned.", + ["mode"] = "If no mode is set, the whole event object is returned. Summary returns a lightweight event object; stack returns event metrics grouped into stack summaries.", ["page"] = "The page parameter is used for pagination. This value must be greater than 0.", ["limit"] = "A limit on the number of objects to be returned. Limit can range between 1 and 100 items.", ["before"] = "The before parameter is a cursor used for pagination and defines your place in the list of results.", @@ -358,7 +358,7 @@ public static IEndpointRouteBuilder MapEventEndpoints(this IEndpointRouteBuilder ["sort"] = "Controls the sort order that the data is returned in. In this example -date returns the results descending by date.", ["time"] = "The time filter that limits the data being returned to a specific date range.", ["offset"] = "The time offset in minutes that controls what data is returned based on the time filter. This is used for time zone support.", - ["mode"] = "If no mode is set then the whole event object will be returned. If the mode is set to summary than a lightweight object will be returned.", + ["mode"] = "If no mode is set, the whole event object is returned. Summary returns a lightweight event object; stack returns event metrics grouped into stack summaries.", ["page"] = "The page parameter is used for pagination. This value must be greater than 0.", ["limit"] = "A limit on the number of objects to be returned. Limit can range between 1 and 100 items.", ["before"] = "The before parameter is a cursor used for pagination and defines your place in the list of results.", @@ -387,7 +387,7 @@ public static IEndpointRouteBuilder MapEventEndpoints(this IEndpointRouteBuilder ["sort"] = "Controls the sort order that the data is returned in. In this example -date returns the results descending by date.", ["time"] = "The time filter that limits the data being returned to a specific date range.", ["offset"] = "The time offset in minutes that controls what data is returned based on the time filter. This is used for time zone support.", - ["mode"] = "If no mode is set then the whole event object will be returned. If the mode is set to summary than a lightweight object will be returned.", + ["mode"] = "If no mode is set, the whole event object is returned. Summary returns a lightweight event object; stack returns event metrics grouped into stack summaries.", ["page"] = "The page parameter is used for pagination. This value must be greater than 0.", ["limit"] = "A limit on the number of objects to be returned. Limit can range between 1 and 100 items.", ["before"] = "The before parameter is a cursor used for pagination and defines your place in the list of results.", @@ -417,7 +417,7 @@ public static IEndpointRouteBuilder MapEventEndpoints(this IEndpointRouteBuilder ["sort"] = "Controls the sort order that the data is returned in. In this example -date returns the results descending by date.", ["time"] = "The time filter that limits the data being returned to a specific date range.", ["offset"] = "The time offset in minutes that controls what data is returned based on the time filter. This is used for time zone support.", - ["mode"] = "If no mode is set then the whole event object will be returned. If the mode is set to summary than a lightweight object will be returned.", + ["mode"] = "If no mode is set, the whole event object is returned. Summary returns a lightweight event object; stack returns event metrics grouped into stack summaries.", ["page"] = "The page parameter is used for pagination. This value must be greater than 0.", ["limit"] = "A limit on the number of objects to be returned. Limit can range between 1 and 100 items.", ["before"] = "The before parameter is a cursor used for pagination and defines your place in the list of results.", diff --git a/src/Exceptionless.Web/Api/Endpoints/StackEndpoints.cs b/src/Exceptionless.Web/Api/Endpoints/StackEndpoints.cs index c88d1e757a..6f7449415e 100644 --- a/src/Exceptionless.Web/Api/Endpoints/StackEndpoints.cs +++ b/src/Exceptionless.Web/Api/Endpoints/StackEndpoints.cs @@ -2,6 +2,7 @@ using Exceptionless.Core.Authorization; using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; +using Exceptionless.Core.Services; using Exceptionless.Web.Api.Filters; using Exceptionless.Web.Api.Infrastructure; using Exceptionless.Web.Api.Messages; @@ -248,8 +249,8 @@ public static IEndpointRouteBuilder MapStackEndpoints(this IEndpointRouteBuilder }); // Get all - group.MapGet("stacks", async (HttpContext httpContext, IMediator mediator, IMediatorResultMapper resultMapper, string? filter = null, string? sort = null, string? time = null, string? offset = null, string? mode = null, int page = 1, int limit = 10) - => (await mediator.InvokeAsync>>(new GetAllStacks(filter, sort, time, offset, mode, page, limit, httpContext))).ToHttpResult(resultMapper)) + group.MapGet("stacks", async (HttpContext httpContext, IMediator mediator, IMediatorResultMapper resultMapper, string? filter = null, string? sort = null, string? time = null, string? offset = null, int limit = 10, string? before = null, string? after = null) + => (await mediator.InvokeAsync>>(new GetAllStacks(filter, sort, time, offset, limit, before, after, httpContext))).ToHttpResult(resultMapper)) .RequireAuthorization(AuthorizationRoles.StacksReadPolicy) .Produces>() .ProducesProblem(StatusCodes.Status400BadRequest) @@ -261,9 +262,9 @@ public static IEndpointRouteBuilder MapStackEndpoints(this IEndpointRouteBuilder ["sort"] = "Controls the sort order that the data is returned in. In this example -date returns the results descending by date.", ["time"] = "The time filter that limits the data being returned to a specific date range.", ["offset"] = "The time offset in minutes that controls what data is returned based on the time filter. This is used for time zone support.", - ["mode"] = "If no mode is set then the whole stack object will be returned. If the mode is set to summary than a lightweight object will be returned.", - ["page"] = "The page parameter is used for pagination. This value must be greater than 0.", ["limit"] = "A limit on the number of objects to be returned. Limit can range between 1 and 100 items.", + ["before"] = "A cursor that returns the previous page for this exact filter and sort.", + ["after"] = "A cursor that returns the next page for this exact filter and sort.", }, ResponseDescriptions = new() { ["400"] = "Invalid filter.", @@ -272,8 +273,8 @@ public static IEndpointRouteBuilder MapStackEndpoints(this IEndpointRouteBuilder }); // Get by organization - group.MapGet("organizations/{organizationId:objectid}/stacks", async (string organizationId, HttpContext httpContext, IMediator mediator, IMediatorResultMapper resultMapper, string? filter = null, string? sort = null, string? time = null, string? offset = null, string? mode = null, int page = 1, int limit = 10) - => (await mediator.InvokeAsync>>(new GetStacksByOrganization(organizationId, filter, sort, time, offset, mode, page, limit, httpContext))).ToHttpResult(resultMapper)) + group.MapGet("organizations/{organizationId:objectid}/stacks", async (string organizationId, HttpContext httpContext, IMediator mediator, IMediatorResultMapper resultMapper, string? filter = null, string? sort = null, string? time = null, string? offset = null, int limit = 10, string? before = null, string? after = null) + => (await mediator.InvokeAsync>>(new GetStacksByOrganization(organizationId, filter, sort, time, offset, limit, before, after, httpContext))).ToHttpResult(resultMapper)) .RequireAuthorization(AuthorizationRoles.StacksReadPolicy) .Produces>() .ProducesProblem(StatusCodes.Status400BadRequest) @@ -287,9 +288,9 @@ public static IEndpointRouteBuilder MapStackEndpoints(this IEndpointRouteBuilder ["sort"] = "Controls the sort order that the data is returned in. In this example -date returns the results descending by date.", ["time"] = "The time filter that limits the data being returned to a specific date range.", ["offset"] = "The time offset in minutes that controls what data is returned based on the time filter. This is used for time zone support.", - ["mode"] = "If no mode is set then the whole stack object will be returned. If the mode is set to summary than a lightweight object will be returned.", - ["page"] = "The page parameter is used for pagination. This value must be greater than 0.", ["limit"] = "A limit on the number of objects to be returned. Limit can range between 1 and 100 items.", + ["before"] = "A cursor that returns the previous page for this exact filter and sort.", + ["after"] = "A cursor that returns the next page for this exact filter and sort.", }, ResponseDescriptions = new() { ["400"] = "Invalid filter.", @@ -299,8 +300,8 @@ public static IEndpointRouteBuilder MapStackEndpoints(this IEndpointRouteBuilder }); // Get by project - group.MapGet("projects/{projectId:objectid}/stacks", async (string projectId, HttpContext httpContext, IMediator mediator, IMediatorResultMapper resultMapper, string? filter = null, string? sort = null, string? time = null, string? offset = null, string? mode = null, int page = 1, int limit = 10) - => (await mediator.InvokeAsync>>(new GetStacksByProject(projectId, filter, sort, time, offset, mode, page, limit, httpContext))).ToHttpResult(resultMapper)) + group.MapGet("projects/{projectId:objectid}/stacks", async (string projectId, HttpContext httpContext, IMediator mediator, IMediatorResultMapper resultMapper, string? filter = null, string? sort = null, string? time = null, string? offset = null, int limit = 10, string? before = null, string? after = null) + => (await mediator.InvokeAsync>>(new GetStacksByProject(projectId, filter, sort, time, offset, limit, before, after, httpContext))).ToHttpResult(resultMapper)) .RequireAuthorization(AuthorizationRoles.StacksReadPolicy) .Produces>() .ProducesProblem(StatusCodes.Status400BadRequest) @@ -314,9 +315,9 @@ public static IEndpointRouteBuilder MapStackEndpoints(this IEndpointRouteBuilder ["sort"] = "Controls the sort order that the data is returned in. In this example -date returns the results descending by date.", ["time"] = "The time filter that limits the data being returned to a specific date range.", ["offset"] = "The time offset in minutes that controls what data is returned based on the time filter. This is used for time zone support.", - ["mode"] = "If no mode is set then the whole stack object will be returned. If the mode is set to summary than a lightweight object will be returned.", - ["page"] = "The page parameter is used for pagination. This value must be greater than 0.", ["limit"] = "A limit on the number of objects to be returned. Limit can range between 1 and 100 items.", + ["before"] = "A cursor that returns the previous page for this exact filter and sort.", + ["after"] = "A cursor that returns the next page for this exact filter and sort.", }, ResponseDescriptions = new() { ["400"] = "Invalid filter.", diff --git a/src/Exceptionless.Web/Api/Handlers/EventHandler.cs b/src/Exceptionless.Web/Api/Handlers/EventHandler.cs index c1bdb6d5c1..cd51cb2882 100644 --- a/src/Exceptionless.Web/Api/Handlers/EventHandler.cs +++ b/src/Exceptionless.Web/Api/Handlers/EventHandler.cs @@ -38,6 +38,7 @@ public class EventHandler( IOrganizationRepository organizationRepository, IProjectRepository projectRepository, IStackRepository stackRepository, + IStackRollupSearchService stackRollupSearchService, EventPostService eventPostService, IQueue eventUserDescriptionQueue, MiniValidationValidator miniValidationValidator, @@ -45,7 +46,7 @@ public class EventHandler( ICacheClient cacheClient, ITextSerializer serializer, PersistentEventQueryValidator validator, - EventStackQueryValidator stackModeValidator, + EventStackQueryValidator stackValidator, AppOptions appOptions, UsageService usageService, TimeProvider timeProvider, @@ -166,7 +167,7 @@ public async Task>> Handle(GetAllEvents message) var ti = TimeRangeParser.GetTimeInfo(message.Time, message.Offset, timeProvider, _allowedDateFields, DefaultDateField, organizations.GetRetentionUtcCutoff(appOptions.MaximumRetentionDays, timeProvider)); var sf = new AppFilter(organizations) { IsUserOrganizationsFilter = true }; - return await GetInternalAsync(sf, ti, httpContext, message.Filter, message.Sort, message.Mode, message.Page, message.Limit, message.Before, message.After, includeTotal: ShouldIncludeTotal(message.Include)); + return await GetInternalAsync(sf, ti, httpContext, message.Filter, message.Sort, message.Mode, message.Page, message.Limit, message.Before, message.After, includeTotal: ShouldIncludeTotal(message.Include), timeExpression: message.Time); } public async Task>> Handle(GetEventsByOrganization message) @@ -181,7 +182,7 @@ public async Task>> Handle(GetEventsByOrganization me var ti = TimeRangeParser.GetTimeInfo(message.Time, message.Offset, timeProvider, _allowedDateFields, DefaultDateField, organization.GetRetentionUtcCutoff(appOptions.MaximumRetentionDays, timeProvider)); var sf = new AppFilter(organization); - return await GetInternalAsync(sf, ti, httpContext, message.Filter, message.Sort, message.Mode, message.Page, message.Limit, message.Before, message.After, includeTotal: ShouldIncludeTotal(message.Include)); + return await GetInternalAsync(sf, ti, httpContext, message.Filter, message.Sort, message.Mode, message.Page, message.Limit, message.Before, message.After, includeTotal: ShouldIncludeTotal(message.Include), timeExpression: message.Time); } public async Task>> Handle(GetEventsByProject message) @@ -200,7 +201,7 @@ public async Task>> Handle(GetEventsByProject message var ti = TimeRangeParser.GetTimeInfo(message.Time, message.Offset, timeProvider, _allowedDateFields, DefaultDateField, organization.GetRetentionUtcCutoff(project, appOptions.MaximumRetentionDays, timeProvider)); var sf = new AppFilter(project, organization); - return await GetInternalAsync(sf, ti, httpContext, message.Filter, message.Sort, message.Mode, message.Page, message.Limit, message.Before, message.After, includeTotal: ShouldIncludeTotal(message.Include)); + return await GetInternalAsync(sf, ti, httpContext, message.Filter, message.Sort, message.Mode, message.Page, message.Limit, message.Before, message.After, includeTotal: ShouldIncludeTotal(message.Include), timeExpression: message.Time); } public async Task>> Handle(GetEventsByStack message) @@ -219,7 +220,7 @@ public async Task>> Handle(GetEventsByStack message) var ti = TimeRangeParser.GetTimeInfo(message.Time, message.Offset, timeProvider, _allowedDateFields, DefaultDateField, organization.GetRetentionUtcCutoff(stack, appOptions.MaximumRetentionDays, timeProvider)); var sf = new AppFilter(stack, organization); - return await GetInternalAsync(sf, ti, httpContext, message.Filter, message.Sort, message.Mode, message.Page, message.Limit, message.Before, message.After, includeTotal: ShouldIncludeTotal(message.Include)); + return await GetInternalAsync(sf, ti, httpContext, message.Filter, message.Sort, message.Mode, message.Page, message.Limit, message.Before, message.After, includeTotal: ShouldIncludeTotal(message.Include), timeExpression: message.Time); } public async Task>> Handle(GetEventsByReferenceId message) @@ -262,7 +263,7 @@ public async Task>> Handle(GetEventsBySessionId messa var ti = TimeRangeParser.GetTimeInfo(message.Time, message.Offset, timeProvider, _allowedDateFields, DefaultDateField, organizations.GetRetentionUtcCutoff(appOptions.MaximumRetentionDays, timeProvider)); var sf = new AppFilter(organizations) { IsUserOrganizationsFilter = true }; - return await GetInternalAsync(sf, ti, httpContext, $"(reference:{message.SessionId} OR ref.session:{message.SessionId}) {message.Filter}", message.Sort, message.Mode, message.Page, message.Limit, message.Before, message.After, premiumFeatureUpgradeMessage: ApiFilterPolicy.PremiumSessionUpgradeMessage, includeTotal: ShouldIncludeTotal(message.Include)); + return await GetInternalAsync(sf, ti, httpContext, $"(reference:{message.SessionId} OR ref.session:{message.SessionId}) {message.Filter}", message.Sort, message.Mode, message.Page, message.Limit, message.Before, message.After, premiumFeatureUpgradeMessage: ApiFilterPolicy.PremiumSessionUpgradeMessage, includeTotal: ShouldIncludeTotal(message.Include), timeExpression: message.Time); } public async Task>> Handle(GetEventsBySessionIdAndProject message) @@ -281,7 +282,7 @@ public async Task>> Handle(GetEventsBySessionIdAndPro var ti = TimeRangeParser.GetTimeInfo(message.Time, message.Offset, timeProvider, _allowedDateFields, DefaultDateField, organization.GetRetentionUtcCutoff(project, appOptions.MaximumRetentionDays, timeProvider)); var sf = new AppFilter(project, organization); - return await GetInternalAsync(sf, ti, httpContext, $"(reference:{message.SessionId} OR ref.session:{message.SessionId}) {message.Filter}", message.Sort, message.Mode, message.Page, message.Limit, message.Before, message.After, premiumFeatureUpgradeMessage: ApiFilterPolicy.PremiumSessionUpgradeMessage, includeTotal: ShouldIncludeTotal(message.Include)); + return await GetInternalAsync(sf, ti, httpContext, $"(reference:{message.SessionId} OR ref.session:{message.SessionId}) {message.Filter}", message.Sort, message.Mode, message.Page, message.Limit, message.Before, message.After, premiumFeatureUpgradeMessage: ApiFilterPolicy.PremiumSessionUpgradeMessage, includeTotal: ShouldIncludeTotal(message.Include), timeExpression: message.Time); } public async Task>> Handle(GetSessions message) @@ -293,7 +294,7 @@ public async Task>> Handle(GetSessions message) var ti = TimeRangeParser.GetTimeInfo(message.Time, message.Offset, timeProvider, _allowedDateFields, DefaultDateField, organizations.GetRetentionUtcCutoff(appOptions.MaximumRetentionDays, timeProvider)); var sf = new AppFilter(organizations) { IsUserOrganizationsFilter = true }; - return await GetInternalAsync(sf, ti, httpContext, $"type:{Event.KnownTypes.Session} {message.Filter}", message.Sort, message.Mode, message.Page, message.Limit, message.Before, message.After, premiumFeatureUpgradeMessage: ApiFilterPolicy.PremiumSessionUpgradeMessage, includeTotal: ShouldIncludeTotal(message.Include)); + return await GetInternalAsync(sf, ti, httpContext, $"type:{Event.KnownTypes.Session} {message.Filter}", message.Sort, message.Mode, message.Page, message.Limit, message.Before, message.After, premiumFeatureUpgradeMessage: ApiFilterPolicy.PremiumSessionUpgradeMessage, includeTotal: ShouldIncludeTotal(message.Include), timeExpression: message.Time); } public async Task>> Handle(GetSessionsByOrganization message) @@ -308,7 +309,7 @@ public async Task>> Handle(GetSessionsByOrganization var ti = TimeRangeParser.GetTimeInfo(message.Time, message.Offset, timeProvider, _allowedDateFields, DefaultDateField, organization.GetRetentionUtcCutoff(appOptions.MaximumRetentionDays, timeProvider)); var sf = new AppFilter(organization); - return await GetInternalAsync(sf, ti, httpContext, $"type:{Event.KnownTypes.Session} {message.Filter}", message.Sort, message.Mode, message.Page, message.Limit, message.Before, message.After, premiumFeatureUpgradeMessage: ApiFilterPolicy.PremiumSessionUpgradeMessage, includeTotal: ShouldIncludeTotal(message.Include)); + return await GetInternalAsync(sf, ti, httpContext, $"type:{Event.KnownTypes.Session} {message.Filter}", message.Sort, message.Mode, message.Page, message.Limit, message.Before, message.After, premiumFeatureUpgradeMessage: ApiFilterPolicy.PremiumSessionUpgradeMessage, includeTotal: ShouldIncludeTotal(message.Include), timeExpression: message.Time); } public async Task>> Handle(GetSessionsByProject message) @@ -327,7 +328,7 @@ public async Task>> Handle(GetSessionsByProject messa var ti = TimeRangeParser.GetTimeInfo(message.Time, message.Offset, timeProvider, _allowedDateFields, DefaultDateField, organization.GetRetentionUtcCutoff(project, appOptions.MaximumRetentionDays, timeProvider)); var sf = new AppFilter(project, organization); - return await GetInternalAsync(sf, ti, httpContext, $"type:{Event.KnownTypes.Session} {message.Filter}", message.Sort, message.Mode, message.Page, message.Limit, message.Before, message.After, premiumFeatureUpgradeMessage: ApiFilterPolicy.PremiumSessionUpgradeMessage, includeTotal: ShouldIncludeTotal(message.Include)); + return await GetInternalAsync(sf, ti, httpContext, $"type:{Event.KnownTypes.Session} {message.Filter}", message.Sort, message.Mode, message.Page, message.Limit, message.Before, message.After, premiumFeatureUpgradeMessage: ApiFilterPolicy.PremiumSessionUpgradeMessage, includeTotal: ShouldIncludeTotal(message.Include), timeExpression: message.Time); } public async Task Handle(SetEventUserDescription message) @@ -690,7 +691,13 @@ public async Task> Handle(DeleteEvents message) private async Task> CountInternalAsync(AppFilter sf, TimeInfo ti, HttpContext httpContext, string? filter = null, string? aggregations = null, string? mode = null) { - var pr = await GetQueryValidator(mode).ValidateQueryAsync(filter); + bool isStackMode = String.Equals(mode, "stack", StringComparison.OrdinalIgnoreCase); + if (mode is not null && !isStackMode) + return Result.BadRequest("Mode must be 'stack' when specified."); + + var pr = isStackMode + ? await stackValidator.ValidateQueryAsync(filter) + : await validator.ValidateQueryAsync(filter); if (!pr.IsValid) return Result.BadRequest(pr.Message ?? "Invalid filter."); @@ -698,14 +705,11 @@ private async Task> CountInternalAsync(AppFilter sf, TimeInf if (!far.IsValid) return Result.BadRequest(far.Message ?? "Invalid aggregations."); - sf.UsesPremiumFeatures = pr.UsesPremiumFeatures || far.UsesPremiumFeatures; + sf.UsesPremiumFeatures = pr.UsesPremiumFeatures || !isStackMode && far.UsesPremiumFeatures; AppFilter? systemFilter = ApiFilterPolicy.ShouldApplySystemFilter(sf, filter, httpContext.Request) ? sf : null; if (systemFilter is not null && ApiFilterPolicy.IsPremiumFeatureQueryBlocked(systemFilter)) return PlanLimitResult(ApiFilterPolicy.PremiumSearchUpgradeMessage); - if (mode == "stack_new") - filter = AddFirstOccurrenceFilter(ti.Range, filter); - var query = new RepositoryQuery() .AppFilter(systemFilter) .DateRange(ti.Range.UtcStart, ti.Range.UtcEnd, ti.Field) @@ -714,7 +718,24 @@ private async Task> CountInternalAsync(AppFilter sf, TimeInf CountResult result; try { - result = await eventRepository.CountAsync(q => q.SystemFilter(query).FilterExpression(filter).EnforceEventStackFilter().AggregationsExpression(aggregations)); + if (isStackMode || await stackRollupSearchService.RequiresLookupJoinAsync(filter, httpContext.RequestAborted)) + { + result = await stackRollupSearchService.CountEventsAsync(new EventLookupCountRequest( + systemFilter, + ti.Range.UtcStart, + ti.Range.UtcEnd, + ti.Offset, + filter, + aggregations), httpContext.RequestAborted); + } + else + { + result = await eventRepository.CountAsync(q => q.SystemFilter(query).FilterExpression(filter).EnforceEventStackFilter().AggregationsExpression(aggregations)); + } + } + catch (NotSupportedException ex) + { + return Result.BadRequest(ex.Message); } catch (Exception ex) { @@ -728,8 +749,21 @@ private async Task> CountInternalAsync(AppFilter sf, TimeInf return result; } - private async Task>> GetInternalAsync(AppFilter sf, TimeInfo ti, HttpContext httpContext, string? filter = null, string? sort = null, string? mode = null, int? page = null, int limit = 10, string? before = null, string? after = null, string? premiumFeatureUpgradeMessage = null, bool includeTotal = false) + private async Task>> GetInternalAsync(AppFilter sf, TimeInfo ti, HttpContext httpContext, string? filter = null, string? sort = null, string? mode = null, int? page = null, int limit = 10, string? before = null, string? after = null, string? premiumFeatureUpgradeMessage = null, bool includeTotal = false, string? timeExpression = null) { + if (mode is not null + && !String.Equals(mode, "summary", StringComparison.OrdinalIgnoreCase) + && !String.Equals(mode, "stack", StringComparison.OrdinalIgnoreCase)) + return Result.BadRequest("Mode must be 'summary' or 'stack' when specified."); + + if (String.Equals(mode, "stack", StringComparison.OrdinalIgnoreCase)) + { + if (page.HasValue) + return Result.BadRequest("Stack mode uses before and after cursors; page is not supported."); + + return await GetStackModeEventsInternalAsync(sf, ti, httpContext, filter, sort, limit, before, after, includeTotal, timeExpression); + } + var currentUser = httpContext.Request.GetUser(); using var _ = _logger.BeginScope(new ExceptionlessState() .Property("Search Filter", new @@ -755,7 +789,7 @@ private async Task>> GetInternalAsync(AppFilter sf, T if (skip > Pagination.MaximumSkip) return new PagedResult(Array.Empty(), false); - var pr = await GetQueryValidator(mode).ValidateQueryAsync(filter); + var pr = await validator.ValidateQueryAsync(filter); if (!pr.IsValid) return Result.BadRequest(pr.Message ?? "Invalid filter."); @@ -766,6 +800,48 @@ private async Task>> GetInternalAsync(AppFilter sf, T try { + if (await stackRollupSearchService.RequiresLookupJoinAsync(filter, httpContext.RequestAborted)) + { + if (page.HasValue) + return Result.BadRequest("Event queries with stack filters use before and after cursors; page is not supported."); + if (before is not null && after is not null) + return Result.BadRequest("The before and after parameters cannot be used together."); + + EventLookupSearchResult lookup; + try + { + lookup = await stackRollupSearchService.SearchEventsAsync(new EventLookupSearchRequest( + appliedAppFilter, + ti.Range.UtcStart, + ti.Range.UtcEnd, + timeExpression, + filter, + sort, + limit, + before, + after, + includeTotal), httpContext.RequestAborted); + } + catch (InvalidEventLookupCursorException ex) + { + return Result.BadRequest(ex.Message); + } + catch (ArgumentOutOfRangeException ex) when (ex.ParamName == "sort") + { + return Result.BadRequest("Event queries with stack filters can only be sorted by date or -date."); + } + + var byId = (await eventRepository.GetByIdsAsync(lookup.EventIds.ToArray())).ToDictionary(eventItem => eventItem.Id, StringComparer.Ordinal); + var documents = lookup.EventIds.Where(byId.ContainsKey).Select(id => byId[id]).ToList(); + if (String.Equals(mode, "summary", StringComparison.OrdinalIgnoreCase)) + { + var summaries = await GetEventSummariesAsync(documents); + return new PagedResult(summaries.Cast().ToList(), lookup.HasMore, null, lookup.Total, lookup.Before, lookup.After); + } + + return new PagedResult(documents.Cast().ToList(), lookup.HasMore, null, lookup.Total, lookup.Before, lookup.After); + } + FindResults events; switch (mode) { @@ -790,54 +866,6 @@ private async Task>> GetInternalAsync(AppFilter sf, T }; }).ToList(); return new PagedResult(summaries.Cast().ToList(), events.HasMore && !Pagination.NextPageExceedsSkipLimit(page, limit), page, includeTotal ? events.Total : null, events.Hits.FirstOrDefault()?.GetSortToken(serializer), events.Hits.LastOrDefault()?.GetSortToken(serializer)); - case "stack_recent": - case "stack_frequent": - case "stack_new": - case "stack_users": - if (!String.IsNullOrEmpty(sort)) - return Result.BadRequest("Sort is not supported in stack mode."); - - var systemFilter = new RepositoryQuery() - .AppFilter(appliedAppFilter) - .EnforceEventStackFilter() - .DateRange(ti.Range.UtcStart, ti.Range.UtcEnd, (PersistentEvent e) => e.Date) - .Index(ti.Range.UtcStart, ti.Range.UtcEnd); - - string? stackAggregations = mode switch - { - "stack_recent" => "cardinality:user sum:count~1 min:date -max:date", - "stack_frequent" => "cardinality:user -sum:count~1 min:date max:date", - "stack_new" => "cardinality:user sum:count~1 -min:date max:date", - "stack_users" => "-cardinality:user sum:count~1 min:date max:date", - _ => null - }; - - if (mode == "stack_new") - filter = AddFirstOccurrenceFilter(ti.Range, filter); - - string aggregationExpression = includeTotal - ? $"cardinality:stack_id terms:(stack_id~{Pagination.GetSkip(resolvedPage + 1, limit) + 1} {stackAggregations})" - : $"terms:(stack_id~{Pagination.GetSkip(resolvedPage + 1, limit) + 1} {stackAggregations})"; - - var countResponse = await eventRepository.CountAsync(q => q - .SystemFilter(systemFilter) - .FilterExpression(filter) - .EnforceEventStackFilter() - .AggregationsExpression(aggregationExpression), - o => o.TrackTotalHits(false)); - - var stackTerms = countResponse.Aggregations.Terms("terms_stack_id"); - if (stackTerms is null || stackTerms.Buckets.Count == 0) - return new PagedResult(Array.Empty(), false); - - string[] stackIds = stackTerms.Buckets.Skip(skip).Take(limit + 1).Select(t => t.Key).ToArray(); - var stacks = (await stackRepository.GetByIdsAsync(stackIds)).Select(s => s.ApplyOffset(ti.Offset)).ToList(); - - var stackSummaries = await GetStackSummariesAsync(stacks, stackTerms.Buckets, sf, ti); - - double? totalStackCount = countResponse.Aggregations.Cardinality("cardinality_stack_id")?.Value; - long? total = includeTotal && totalStackCount.HasValue ? Convert.ToInt64(totalStackCount.Value) : null; - return new PagedResult(stackSummaries.Take(limit).Cast().ToList(), stackSummaries.Count > limit && !Pagination.NextPageExceedsSkipLimit(resolvedPage, limit), resolvedPage, total); default: events = await GetEventsInternalAsync(appliedAppFilter, ti, filter, sort, page, limit, before, after, includeTotal); return new PagedResult(events.Documents.Cast().ToList(), events.HasMore && !Pagination.NextPageExceedsSkipLimit(page, limit), page, includeTotal ? events.Total : null, events.Hits.FirstOrDefault()?.GetSortToken(serializer), events.Hits.LastOrDefault()?.GetSortToken(serializer)); @@ -854,50 +882,6 @@ private async Task>> GetInternalAsync(AppFilter sf, T } } - private static string AddFirstOccurrenceFilter(DateTimeRange timeRange, string? filter) - { - bool inverted = false; - if (filter is not null && filter.StartsWith("@!")) - { - inverted = true; - filter = filter.Substring(2); - } - - var sb = new StringBuilder(); - if (inverted) - sb.Append("@!"); - - sb.Append("first_occurrence:[\""); - sb.Append(timeRange.UtcStart.ToString("O")); - sb.Append("\" TO \""); - sb.Append(timeRange.UtcEnd.ToString("O")); - sb.Append("\"]"); - - if (String.IsNullOrEmpty(filter)) - return sb.ToString(); - - sb.Append(' '); - - bool isGrouped = filter.StartsWith('(') && filter.EndsWith(')'); - - if (isGrouped) - sb.Append(filter); - else - sb.Append('(').Append(filter).Append(')'); - - return sb.ToString(); - } - - private static bool IsStackMode(string? mode) - { - return mode is "stack_recent" or "stack_frequent" or "stack_new" or "stack_users"; - } - - private IAppQueryValidator GetQueryValidator(string? mode) - { - return IsStackMode(mode) ? stackModeValidator : validator; - } - private Task> GetEventsInternalAsync(AppFilter? systemFilter, TimeInfo ti, string? filter, string? sort, int? page, int limit, string? before, string? after, bool includeTotal) { if (String.IsNullOrEmpty(sort)) @@ -915,18 +899,102 @@ private Task> GetEventsInternalAsync(AppFilter? sys : o.SearchBeforeToken(before, serializer).SearchAfterToken(after, serializer).PageLimit(limit).TrackTotalHits(includeTotal)); } - private async Task> GetStackSummariesAsync(List stacks, IReadOnlyCollection> stackTerms, AppFilter sf, TimeInfo ti) + private async Task> GetEventSummariesAsync(IReadOnlyCollection events) + { + var projects = await projectRepository.GetByIdsAsync(events.Select(eventItem => eventItem.ProjectId).Distinct().ToArray(), query => query.Cache()); + var projectNames = projects.ToDictionary(project => project.Id, project => project.Name); + return events.Select(eventItem => + { + var summaryData = formattingPluginManager.GetEventSummaryData(eventItem); + return new EventSummaryModel + { + Id = summaryData.Id, + TemplateKey = summaryData.TemplateKey, + Date = eventItem.Date, + ProjectId = eventItem.ProjectId, + ProjectName = projectNames.GetValueOrDefault(eventItem.ProjectId), + Tags = eventItem.Tags?.OfType().Order(StringComparer.OrdinalIgnoreCase).ToArray() ?? [], + Type = eventItem.Type, + Version = eventItem.GetVersion(), + Data = summaryData.Data + }; + }).ToList(); + } + + private async Task>> GetStackModeEventsInternalAsync( + AppFilter appFilter, + TimeInfo time, + HttpContext httpContext, + string? filter, + string? sort, + int limit, + string? before, + string? after, + bool includeTotal, + string? timeExpression) + { + if (before is not null && after is not null) + return Result.BadRequest("The before and after parameters cannot be used together."); + + limit = Pagination.GetLimit(limit); + var validation = await stackValidator.ValidateQueryAsync(filter); + if (!validation.IsValid) + return Result.BadRequest(validation.Message ?? "Invalid filter."); + + appFilter.UsesPremiumFeatures = validation.UsesPremiumFeatures; + AppFilter? appliedAppFilter = ApiFilterPolicy.ShouldApplySystemFilter(appFilter, filter, httpContext.Request) ? appFilter : null; + if (appliedAppFilter is not null && ApiFilterPolicy.IsPremiumFeatureQueryBlocked(appliedAppFilter)) + return PlanLimitResult>(ApiFilterPolicy.PremiumSearchUpgradeMessage); + + try + { + var result = await stackRollupSearchService.SearchAsync(new StackRollupSearchRequest( + appliedAppFilter, + time.Range.UtcStart, + time.Range.UtcEnd, + time.Offset, + timeExpression, + filter, + sort, + limit, + before, + after, + includeTotal), httpContext.RequestAborted); + + string[] stackIds = result.Rows.Select(row => row.StackId).ToArray(); + var stacks = (await stackRepository.GetByIdsAsync(stackIds)) + .Select(stack => stack.ApplyOffset(time.Offset)) + .ToList(); + var summaries = await GetStackSummariesAsync(stacks, result.Rows, appFilter, time); + return new PagedResult(summaries.Cast().ToList(), result.HasMore, null, result.Total, result.Before, result.After); + } + catch (InvalidStackRollupCursorException ex) + { + return Result.BadRequest(ex.Message); + } + catch (ArgumentOutOfRangeException ex) when (ex.ParamName == "sort") + { + return Result.BadRequest("Stack mode sort must be one of total, users, first_occurrence, or last_occurrence, optionally prefixed with '-'."); + } + } + + private async Task> GetStackSummariesAsync(List stacks, IReadOnlyCollection rows, AppFilter appFilter, TimeInfo time) { if (stacks.Count == 0) - return new List(0); + return []; - var projects = await projectRepository.GetByIdsAsync(stacks.Select(s => s.ProjectId).Distinct().ToArray(), o => o.Cache()); - var projectNames = projects.ToDictionary(p => p.Id, p => p.Name); - var totalUsers = await GetUserCountByProjectIdsAsync(stacks, sf, ti.Range.UtcStart, ti.Range.UtcEnd); - return stacks.Join(stackTerms, s => s.Id, tk => tk.Key, (stack, term) => + var stacksById = stacks.ToDictionary(stack => stack.Id, StringComparer.Ordinal); + var projects = await projectRepository.GetByIdsAsync(stacks.Select(stack => stack.ProjectId).Distinct().ToArray(), query => query.Cache()); + var projectNames = projects.ToDictionary(project => project.Id, project => project.Name); + var totalUsers = await GetUserCountByProjectIdsAsync(stacks, appFilter, time.Range.UtcStart, time.Range.UtcEnd); + var summaries = new List(rows.Count); + foreach (var row in rows) { + if (!stacksById.TryGetValue(row.StackId, out var stack)) + continue; + var data = formattingPluginManager.GetStackSummaryData(stack); - var summary = new StackSummaryModel + summaries.Add(new StackSummaryModel { Id = data.Id, TemplateKey = data.TemplateKey, @@ -936,40 +1004,38 @@ private async Task> GetStackSummariesAsync(List().Order(StringComparer.OrdinalIgnoreCase).ToArray() ?? [], Title = stack.Title, Status = stack.Status, - FirstOccurrence = term.Aggregations.Min("min_date")?.Value ?? stack.FirstOccurrence, - LastOccurrence = term.Aggregations.Max("max_date")?.Value ?? stack.LastOccurrence, - Total = (long)(term.Aggregations.Sum("sum_count")?.Value ?? term.Total.GetValueOrDefault()), - - Users = term.Aggregations.Cardinality("cardinality_user")?.Value.GetValueOrDefault() ?? 0, + FirstOccurrence = row.FirstOccurrence, + LastOccurrence = row.LastOccurrence, + Total = row.Total, + Users = row.Users, TotalUsers = totalUsers.GetOrDefault(stack.ProjectId) - }; + }); + } - return summary; - }).ToList(); + return summaries; } - private async Task> GetUserCountByProjectIdsAsync(ICollection stacks, AppFilter sf, DateTime utcStart, DateTime utcEnd) + private async Task> GetUserCountByProjectIdsAsync(ICollection stacks, AppFilter appFilter, DateTime utcStart, DateTime utcEnd) { using var scopedCacheClient = new ScopedCacheClient(cacheClient, $"Project:user-count:{utcStart.Floor(TimeSpan.FromMinutes(15)).Ticks}-{utcEnd.Floor(TimeSpan.FromMinutes(15)).Ticks}"); - var projectIds = stacks.Select(s => s.ProjectId).Distinct().ToList(); + var projectIds = stacks.Select(stack => stack.ProjectId).Distinct().ToList(); var cachedTotals = await scopedCacheClient.GetAllAsync(projectIds); - - var totals = cachedTotals.Where(kvp => kvp.Value.HasValue).ToDictionary(kvp => kvp.Key, kvp => kvp.Value.Value); + var totals = cachedTotals.Where(item => item.Value.HasValue).ToDictionary(item => item.Key, item => item.Value.Value); if (totals.Count == projectIds.Count) return totals; - var systemFilter = new RepositoryQuery().AppFilter(sf).DateRange(utcStart, utcEnd, (PersistentEvent e) => e.Date).Index(utcStart, utcEnd); var projects = cachedTotals - .Where(kvp => !kvp.Value.HasValue && stacks.Contains(s => s.ProjectId == kvp.Key)) - .Select(kvp => new Project { Id = kvp.Key, OrganizationId = stacks.First(s => s.ProjectId == kvp.Key).OrganizationId }) + .Where(item => !item.Value.HasValue && stacks.Contains(stack => stack.ProjectId == item.Key)) + .Select(item => new Project { Id = item.Key, OrganizationId = stacks.First(stack => stack.ProjectId == item.Key).OrganizationId }) .ToList(); - var countResult = await eventRepository.CountAsync(q => q.SystemFilter(systemFilter).FilterExpression(projects.BuildFilter()).EnforceEventStackFilter().AggregationsExpression("terms:(project_id cardinality:user)")); - - var projectTerms = countResult.Aggregations.Terms("terms_project_id")?.Buckets ?? []; - var aggregations = projectTerms.ToDictionary(t => t.Key, t => t.Aggregations.Cardinality("cardinality_user")?.Value.GetValueOrDefault() ?? 0); - await scopedCacheClient.SetAllAsync(aggregations.Where(t => t.Value >= 10).ToDictionary(k => k.Key, v => v.Value), TimeSpan.FromMinutes(5)); + var aggregations = (await stackRollupSearchService.GetProjectUserCountsAsync(new StackRollupProjectUsersRequest( + appFilter, + utcStart, + utcEnd, + projects.Select(project => project.Id).ToArray()))) + .ToDictionary(item => item.Key, item => (double)item.Value); + await scopedCacheClient.SetAllAsync(aggregations.Where(item => item.Value >= 10).ToDictionary(item => item.Key, item => item.Value), TimeSpan.FromMinutes(5)); totals.AddRange(aggregations); - return totals; } diff --git a/src/Exceptionless.Web/Api/Handlers/StackHandler.cs b/src/Exceptionless.Web/Api/Handlers/StackHandler.cs index 0ac1975ef6..0fb87dff40 100644 --- a/src/Exceptionless.Web/Api/Handlers/StackHandler.cs +++ b/src/Exceptionless.Web/Api/Handlers/StackHandler.cs @@ -22,8 +22,10 @@ using Foundatio.Mediator; using Foundatio.Queues; using Foundatio.Repositories; +using Foundatio.Repositories.Elasticsearch.Extensions; using Foundatio.Repositories.Extensions; using Foundatio.Repositories.Models; +using Foundatio.Serializer; using McSherry.SemanticVersioning; namespace Exceptionless.Web.Api.Handlers; @@ -32,14 +34,12 @@ public class StackHandler( IStackRepository stackRepository, IOrganizationRepository organizationRepository, IProjectRepository projectRepository, - IEventRepository eventRepository, IWebHookRepository webHookRepository, WebHookDataPluginManager webHookDataPluginManager, IQueue webHookNotificationQueue, - ICacheClient cacheClient, - FormattingPluginManager formattingPluginManager, SemanticVersionParser semanticVersionParser, StackQueryValidator validator, + ITextSerializer serializer, AppOptions options, TimeProvider timeProvider, ILoggerFactory loggerFactory) @@ -358,7 +358,7 @@ public async Task>> Handle(GetAllStacks message) var ti = TimeRangeParser.GetTimeInfo(message.Time, message.Offset, timeProvider, _allowedDateFields, DefaultDateField, organizations.GetRetentionUtcCutoff(options.MaximumRetentionDays, timeProvider)); var sf = new AppFilter(organizations) { IsUserOrganizationsFilter = true }; - return await GetInternalAsync(sf, ti, httpContext, message.Filter, message.Sort, message.Mode, message.Page, message.Limit); + return await GetInternalAsync(sf, ti, httpContext, message.Filter, message.Sort, message.Limit, message.Before, message.After); } public async Task>> Handle(GetStacksByOrganization message) @@ -373,7 +373,7 @@ public async Task>> Handle(GetStacksByOrganization me var ti = TimeRangeParser.GetTimeInfo(message.Time, message.Offset, timeProvider, _allowedDateFields, DefaultDateField, organization.GetRetentionUtcCutoff(options.MaximumRetentionDays, timeProvider)); var sf = new AppFilter(organization); - return await GetInternalAsync(sf, ti, httpContext, message.Filter, message.Sort, message.Mode, message.Page, message.Limit); + return await GetInternalAsync(sf, ti, httpContext, message.Filter, message.Sort, message.Limit, message.Before, message.After); } public async Task>> Handle(GetStacksByProject message) @@ -392,16 +392,16 @@ public async Task>> Handle(GetStacksByProject message var ti = TimeRangeParser.GetTimeInfo(message.Time, message.Offset, timeProvider, _allowedDateFields, DefaultDateField, organization.GetRetentionUtcCutoff(project, options.MaximumRetentionDays, timeProvider)); var sf = new AppFilter(project, organization); - return await GetInternalAsync(sf, ti, httpContext, message.Filter, message.Sort, message.Mode, message.Page, message.Limit); + return await GetInternalAsync(sf, ti, httpContext, message.Filter, message.Sort, message.Limit, message.Before, message.After); } - private async Task>> GetInternalAsync(AppFilter sf, TimeInfo ti, HttpContext httpContext, string? filter = null, string? sort = null, string? mode = null, int page = 1, int limit = 10) + private async Task>> GetInternalAsync(AppFilter sf, TimeInfo ti, HttpContext httpContext, string? filter = null, string? sort = null, int limit = 10, string? before = null, string? after = null) { - page = Pagination.GetPage(page); + if (before is not null && after is not null) + return Result.BadRequest("The before and after parameters cannot be used together."); + limit = Pagination.GetLimit(limit); - int skip = Pagination.GetSkip(page, limit); - if (skip > Pagination.MaximumSkip) - return new PagedResult(Array.Empty(), false); + sort = String.IsNullOrWhiteSpace(sort) ? "-last" : sort; var pr = await validator.ValidateQueryAsync(filter); if (!pr.IsValid) @@ -414,93 +414,29 @@ private async Task>> GetInternalAsync(AppFilter sf, T try { - var results = await stackRepository.FindAsync(q => q.AppFilter(systemFilter).FilterExpression(filter).SortExpression(sort).DateRange(ti.Range.UtcStart, ti.Range.UtcEnd, ti.Field), o => o.PageNumber(page).PageLimit(limit)); + var results = await stackRepository.FindAsync( + q => q.AppFilter(systemFilter).FilterExpression(filter).SortExpression(sort).DateRange(ti.Range.UtcStart, ti.Range.UtcEnd, ti.Field), + o => o.SearchBeforeToken(before, serializer).SearchAfterToken(after, serializer).PageLimit(limit)); var stacks = results.Documents.Select(s => s.ApplyOffset(ti.Offset)).ToList(); - if (!String.IsNullOrEmpty(mode) && String.Equals(mode, "summary", StringComparison.OrdinalIgnoreCase)) - return new PagedResult((await GetStackSummariesAsync(stacks, sf, ti)).Cast().ToList(), results.HasMore && !Pagination.NextPageExceedsSkipLimit(page, limit), page); - - return new PagedResult(stacks.Cast().ToList(), results.HasMore && !Pagination.NextPageExceedsSkipLimit(page, limit), page); + return new PagedResult( + stacks.Cast().ToList(), + results.HasMore, + Page: null, + Total: null, + results.Hits.FirstOrDefault()?.GetSortToken(serializer), + results.Hits.LastOrDefault()?.GetSortToken(serializer)); } catch (ApplicationException ex) { var currentUser = httpContext.Request.GetUser(); - using (_logger.BeginScope(new ExceptionlessState().Property("Search Filter", new { SystemFilter = sf, UserFilter = filter, Time = ti, Page = page, Limit = limit }).Tag("Search").Identity(currentUser?.EmailAddress).Property("User", currentUser).SetHttpContext(httpContext))) + using (_logger.BeginScope(new ExceptionlessState().Property("Search Filter", new { SystemFilter = sf, UserFilter = filter, Time = ti, Sort = sort, Before = before, After = after, Limit = limit }).Tag("Search").Identity(currentUser?.EmailAddress).Property("User", currentUser).SetHttpContext(httpContext))) _logger.LogError(ex, "An error has occurred. Please check your search filter"); throw; } } - private async Task> GetStackSummariesAsync(ICollection stacks, AppFilter eventSystemFilter, TimeInfo ti) - { - if (stacks.Count == 0) - return new List(); - - var systemFilter = new RepositoryQuery().AppFilter(eventSystemFilter).DateRange(ti.Range.UtcStart, ti.Range.UtcEnd, (PersistentEvent e) => e.Date).Index(ti.Range.UtcStart, ti.Range.UtcEnd); - var stackTerms = await eventRepository.CountAsync(q => q.SystemFilter(systemFilter).Stack(stacks.Select(r => r.Id)).AggregationsExpression($"terms:(stack_id~{stacks.Count} cardinality:user sum:count~1 min:date max:date)")); - var buckets = stackTerms.Aggregations.Terms("terms_stack_id")?.Buckets ?? []; - return await GetStackSummariesAsync(stacks, buckets, eventSystemFilter, ti); - } - - private async Task> GetStackSummariesAsync(ICollection stacks, IReadOnlyCollection> stackTerms, AppFilter sf, TimeInfo ti) - { - if (stacks.Count == 0) - return new List(0); - - var projects = await projectRepository.GetByIdsAsync(stacks.Select(s => s.ProjectId).Distinct().ToArray(), o => o.Cache()); - var projectNames = projects.ToDictionary(p => p.Id, p => p.Name); - var totalUsers = await GetUserCountByProjectIdsAsync(stacks, sf, ti.Range.UtcStart, ti.Range.UtcEnd); - return stacks.Join(stackTerms, s => s.Id, tk => tk.Key, (stack, term) => - { - var data = formattingPluginManager.GetStackSummaryData(stack); - var summary = new StackSummaryModel - { - Id = data.Id, - TemplateKey = data.TemplateKey, - Data = data.Data, - ProjectId = stack.ProjectId, - ProjectName = projectNames.GetValueOrDefault(stack.ProjectId), - Tags = stack.Tags?.OfType().Order(StringComparer.OrdinalIgnoreCase).ToArray() ?? [], - Title = stack.Title, - Status = stack.Status, - FirstOccurrence = term.Aggregations.Min("min_date")?.Value ?? stack.FirstOccurrence, - LastOccurrence = term.Aggregations.Max("max_date")?.Value ?? stack.LastOccurrence, - Total = (long)(term.Aggregations.Sum("sum_count")?.Value ?? term.Total.GetValueOrDefault()), - - Users = term.Aggregations.Cardinality("cardinality_user")?.Value.GetValueOrDefault() ?? 0, - TotalUsers = totalUsers.GetOrDefault(stack.ProjectId) - }; - - return summary; - }).ToList(); - } - - private async Task> GetUserCountByProjectIdsAsync(ICollection stacks, AppFilter sf, DateTime utcStart, DateTime utcEnd) - { - using var scopedCacheClient = new ScopedCacheClient(cacheClient, $"Project:user-count:{utcStart.Floor(TimeSpan.FromMinutes(15)).Ticks}-{utcEnd.Floor(TimeSpan.FromMinutes(15)).Ticks}"); - var projectIds = stacks.Select(s => s.ProjectId).Distinct().ToList(); - var cachedTotals = await scopedCacheClient.GetAllAsync(projectIds); - - var totals = cachedTotals.Where(kvp => kvp.Value.HasValue).ToDictionary(kvp => kvp.Key, kvp => kvp.Value.Value); - if (totals.Count == projectIds.Count) - return totals; - - var systemFilter = new RepositoryQuery().AppFilter(sf).DateRange(utcStart, utcEnd, (PersistentEvent e) => e.Date).Index(utcStart, utcEnd); - var projects = cachedTotals - .Where(kvp => !kvp.Value.HasValue && stacks.Contains(s => s.ProjectId == kvp.Key)) - .Select(kvp => new Project { Id = kvp.Key, OrganizationId = stacks.First(s => s.ProjectId == kvp.Key).OrganizationId }) - .ToList(); - var countResult = await eventRepository.CountAsync(q => q.SystemFilter(systemFilter).FilterExpression(projects.BuildFilter()).AggregationsExpression("terms:(project_id cardinality:user)")); - - var projectTerms = countResult.Aggregations.Terms("terms_project_id")?.Buckets ?? []; - var aggregations = projectTerms.ToDictionary(t => t.Key, t => t.Aggregations.Cardinality("cardinality_user")?.Value.GetValueOrDefault() ?? 0); - await scopedCacheClient.SetAllAsync(aggregations.Where(t => t.Value >= 10).ToDictionary(k => k.Key, v => v.Value), TimeSpan.FromMinutes(5)); - totals.AddRange(aggregations); - - return totals; - } - private async Task GetModelAsync(string id, HttpContext httpContext, bool useCache = true) { if (String.IsNullOrEmpty(id)) diff --git a/src/Exceptionless.Web/Api/Messages/StackMessages.cs b/src/Exceptionless.Web/Api/Messages/StackMessages.cs index 26fba9556c..78b02faae0 100644 --- a/src/Exceptionless.Web/Api/Messages/StackMessages.cs +++ b/src/Exceptionless.Web/Api/Messages/StackMessages.cs @@ -16,6 +16,6 @@ public record MarkStacksNotCritical(string Ids, HttpContext Context); public record ChangeStacksStatus(string Ids, StackStatus Status, HttpContext Context); public record PromoteStack(string Id, HttpContext Context); public record DeleteStacks(string Ids, HttpContext Context); -public record GetAllStacks(string? Filter, string? Sort, string? Time, string? Offset, string? Mode, int Page, int Limit, HttpContext Context); -public record GetStacksByOrganization(string OrganizationId, string? Filter, string? Sort, string? Time, string? Offset, string? Mode, int Page, int Limit, HttpContext Context); -public record GetStacksByProject(string ProjectId, string? Filter, string? Sort, string? Time, string? Offset, string? Mode, int Page, int Limit, HttpContext Context); +public record GetAllStacks(string? Filter, string? Sort, string? Time, string? Offset, int Limit, string? Before, string? After, HttpContext Context); +public record GetStacksByOrganization(string OrganizationId, string? Filter, string? Sort, string? Time, string? Offset, int Limit, string? Before, string? After, HttpContext Context); +public record GetStacksByProject(string ProjectId, string? Filter, string? Sort, string? Time, string? Offset, int Limit, string? Before, string? After, HttpContext Context); diff --git a/src/Exceptionless.Web/ClientApp.angular/app/frequent-controller.js b/src/Exceptionless.Web/ClientApp.angular/app/frequent-controller.js index b0e2225202..f73a5799b0 100644 --- a/src/Exceptionless.Web/ClientApp.angular/app/frequent-controller.js +++ b/src/Exceptionless.Web/ClientApp.angular/app/frequent-controller.js @@ -212,10 +212,10 @@ vm.mostFrequent = { header: "Most Frequent", - get: eventService.getAll, + get: stackService.getRollups, options: { limit: 15, - mode: "stack_frequent", + sort: "-total", }, source: vm._source + ".Events", }; diff --git a/src/Exceptionless.Web/ClientApp.angular/app/new-controller.js b/src/Exceptionless.Web/ClientApp.angular/app/new-controller.js index e1dbad0269..f6d3142bfc 100644 --- a/src/Exceptionless.Web/ClientApp.angular/app/new-controller.js +++ b/src/Exceptionless.Web/ClientApp.angular/app/new-controller.js @@ -108,6 +108,23 @@ return organizationService.getAll().then(onSuccess); } + function getNewStacks(options) { + return stackService.getRollups(options, function (mergedOptions) { + var range = filterService.getTimeRange(); + if (!range.start && !range.end) { + return mergedOptions; + } + + var start = (range.start || moment(filterService.getOldestPossibleEventDate())).utc().format(); + var end = (range.end || moment()).utc().format(); + var firstOccurrenceFilter = 'first_occurrence:["' + start + '" TO "' + end + '"]'; + mergedOptions.filter = mergedOptions.filter + ? firstOccurrenceFilter + " (" + mergedOptions.filter + ")" + : firstOccurrenceFilter; + return mergedOptions; + }); + } + this.$onInit = function $onInit() { vm._organizations = []; vm._source = "app.New"; @@ -212,10 +229,10 @@ vm.newest = { header: "New Stacks", - get: eventService.getAll, + get: getNewStacks, options: { limit: 15, - mode: "stack_new", + sort: "-first_occurrence", }, source: vm._source + ".Events", }; diff --git a/src/Exceptionless.Web/ClientApp.angular/app/users-controller.js b/src/Exceptionless.Web/ClientApp.angular/app/users-controller.js index f6ed12eaf3..72a00a992f 100644 --- a/src/Exceptionless.Web/ClientApp.angular/app/users-controller.js +++ b/src/Exceptionless.Web/ClientApp.angular/app/users-controller.js @@ -212,10 +212,10 @@ vm.mostUsers = { header: "Most Users", - get: eventService.getAll, + get: stackService.getRollups, options: { limit: 15, - mode: "stack_users", + sort: "-users", }, source: vm._source + ".Events", }; diff --git a/src/Exceptionless.Web/ClientApp.angular/components/stack/stack-service.js b/src/Exceptionless.Web/ClientApp.angular/components/stack/stack-service.js index ce22340668..91bb8403c6 100644 --- a/src/Exceptionless.Web/ClientApp.angular/components/stack/stack-service.js +++ b/src/Exceptionless.Web/ClientApp.angular/components/stack/stack-service.js @@ -33,58 +33,25 @@ return Restangular.one("stacks", id).get(); } - function getFrequent(options) { - var mergedOptions = filterService.apply(options); - var organization = filterService.getOrganizationId(); - if (organization) { - return Restangular.one("organizations", organization) - .one("stacks") - .all("frequent") - .getList(mergedOptions); - } - - var project = filterService.getProjectId(); - if (project) { - return Restangular.one("projects", project).one("stacks").all("frequent").getList(mergedOptions); - } - - return Restangular.one("stacks").all("frequent").getList(mergedOptions); - } - - function getUsers(options) { - var mergedOptions = filterService.apply(options); - var organization = filterService.getOrganizationId(); - if (organization) { - return Restangular.one("organizations", organization) - .one("stacks") - .all("users") - .getList(mergedOptions); - } - - var project = filterService.getProjectId(); - if (project) { - return Restangular.one("projects", project).one("stacks").all("users").getList(mergedOptions); - } - - return Restangular.one("stacks").all("users").getList(mergedOptions); - } - - function getNew(options) { - var mergedOptions = filterService.apply(options); + function getRollups(options, optionsCallback) { + optionsCallback = angular.isFunction(optionsCallback) + ? optionsCallback + : function (o) { + return o; + }; + var mergedOptions = optionsCallback(filterService.apply(options)); + mergedOptions.mode = "stack"; var organization = filterService.getOrganizationId(); if (organization) { - return Restangular.one("organizations", organization) - .one("stacks") - .all("new") - .getList(mergedOptions); + return Restangular.one("organizations", organization).all("events").getList(mergedOptions); } var project = filterService.getProjectId(); if (project) { - return Restangular.one("projects", project).one("stacks").all("new").getList(mergedOptions); + return Restangular.one("projects", project).all("events").getList(mergedOptions); } - return Restangular.one("stacks").all("new").getList(mergedOptions); + return Restangular.all("events").getList(mergedOptions); } function markCritical(id) { @@ -122,9 +89,7 @@ changeStatus: changeStatus, getAll: getAll, getById: getById, - getFrequent: getFrequent, - getUsers: getUsers, - getNew: getNew, + getRollups: getRollups, markCritical: markCritical, markNotCritical: markNotCritical, markFixed: markFixed, diff --git a/src/Exceptionless.Web/ClientApp/e2e/tests/list-query-cache.e2e.ts b/src/Exceptionless.Web/ClientApp/e2e/tests/list-query-cache.e2e.ts index 797ae09030..5a1ff6937f 100644 --- a/src/Exceptionless.Web/ClientApp/e2e/tests/list-query-cache.e2e.ts +++ b/src/Exceptionless.Web/ClientApp/e2e/tests/list-query-cache.e2e.ts @@ -133,7 +133,7 @@ function recordListRequest(counts: RequestCounts, request: Request): void { const isStats = url.pathname.endsWith('/count'); const mode = url.searchParams.get('mode'); - if (mode === 'stack_frequent') { + if (mode === 'stack') { counts[isStats ? 'stackStats' : 'stackList']++; } else if (isStats || mode === 'summary') { counts[isStats ? 'eventStats' : 'eventList']++; diff --git a/src/Exceptionless.Web/ClientApp/e2e/tests/stack-effects-chaos.e2e.ts b/src/Exceptionless.Web/ClientApp/e2e/tests/stack-effects-chaos.e2e.ts index 3f8a565737..b117b0f28c 100644 --- a/src/Exceptionless.Web/ClientApp/e2e/tests/stack-effects-chaos.e2e.ts +++ b/src/Exceptionless.Web/ClientApp/e2e/tests/stack-effects-chaos.e2e.ts @@ -131,9 +131,11 @@ test('stack effects stay bounded through background, paging, and navigation chao await measureAction(diagnostics, 'paging', async () => { for (let index = 0; index < 4; index++) { await page.getByRole('button', { name: 'Go to next page' }).click(); - await expect(page).toHaveURL(/(?:\?|&)page=2(?:&|$)/); + await expect(page).toHaveURL(/(?:\?|&)after=[^&]+(?:&|$)/); + await expect(page).not.toHaveURL(/(?:\?|&)page=/); await page.getByRole('button', { name: 'Go to previous page' }).click(); - await expect(page).not.toHaveURL(/(?:\?|&)page=2(?:&|$)/); + await expect(page).not.toHaveURL(/(?:\?|&)(?:before|after)=/); + await expect(page).not.toHaveURL(/(?:\?|&)page=/); } }); expect(actionSample(diagnostics, 'paging').listRequests).toBe(1); @@ -268,7 +270,7 @@ function createChaosEvent(appUrl: string, run: string, index: number): { event: function isStackListRequest(request: Request, organizationId: string): boolean { const url = new URL(request.url()); - return url.pathname === `/api/v2/organizations/${organizationId}/events` && url.searchParams.get('mode') === 'stack_frequent'; + return url.pathname === `/api/v2/organizations/${organizationId}/events` && url.searchParams.get('mode') === 'stack'; } function isStackListResponse(response: Response, organizationId: string): boolean { @@ -322,7 +324,7 @@ function recordRequest(diagnostics: RuntimeDiagnostics, request: Request, organi } const url = new URL(request.url()); - if (url.pathname === `/api/v2/organizations/${organizationId}/events/count` && url.searchParams.get('mode') === 'stack_frequent') { + if (url.pathname === `/api/v2/organizations/${organizationId}/events/count` && url.searchParams.get('mode') === 'stack') { diagnostics.countRequests++; } } diff --git a/src/Exceptionless.Web/ClientApp/e2e/tests/stack-triage.e2e.ts b/src/Exceptionless.Web/ClientApp/e2e/tests/stack-triage.e2e.ts index efbebab3d9..477694bd76 100644 --- a/src/Exceptionless.Web/ClientApp/e2e/tests/stack-triage.e2e.ts +++ b/src/Exceptionless.Web/ClientApp/e2e/tests/stack-triage.e2e.ts @@ -1,5 +1,40 @@ import { expect, test } from '../fixtures/e2e-test'; import { ExceptionlessE2EJourney } from '../support/exceptionless-journey'; +import { createRepresentativeEvent } from '../support/synthetic-event'; + +test('project stack management uses cursor pagination without page numbers @signup', async ({ e2eApi, e2eScenario, page }) => { + const events = Array.from({ length: 6 }, (_, index) => { + const referenceId = `pw-stack-management-${e2eScenario.run}-${index}`; + const event = createRepresentativeEvent({ + appUrl: e2eApi.environment.appUrl, + message: `Project stack management ${e2eScenario.run} ${index}`, + referenceId, + runId: e2eScenario.run + }); + const simpleError = (event.data as Record)['@simple_error'] as Record; + simpleError.type = `ProjectStackManagementException${index}`; + simpleError.stack_trace = `Error: ${referenceId}\n at stack-management-${index}.ts:${index + 1}:1`; + return { event, referenceId }; + }); + + await Promise.all(events.map(({ event }) => e2eApi.submitEvent(e2eScenario.projectId, e2eScenario.projectToken, event))); + await Promise.all(events.map(({ referenceId }) => e2eApi.pollForEventByReference(e2eScenario.userToken, e2eScenario.projectId, referenceId))); + + const listResponse = page.waitForResponse((response) => new URL(response.url()).pathname === `/api/v2/projects/${e2eScenario.projectId}/stacks`); + await page.goto(`/next/project/${e2eScenario.projectId}/stacks?filter=status%3Aopen&limit=5`); + expect((await listResponse).ok()).toBe(true); + await expect(page.getByText('Manage project stacks, including restoring ignored or discarded stacks')).toBeVisible(); + await expect(page.locator('tbody tr:visible')).toHaveCount(5); + + await page.getByRole('button', { name: 'Go to next page' }).click(); + await expect(page).toHaveURL(/(?:\?|&)after=[^&]+(?:&|$)/); + await expect(page).not.toHaveURL(/(?:\?|&)page=/); + await expect(page.locator('tbody tr:visible').first()).toBeVisible(); + + await page.getByRole('button', { name: 'Go to previous page' }).click(); + await expect(page).not.toHaveURL(/(?:\?|&)(?:before|after|page)=/); + await expect(page.locator('tbody tr:visible')).toHaveCount(5); +}); test('new user can mark an open stack fixed from event details @signup', async ({ e2eApi, e2eScenario, page }) => { const journey = ExceptionlessE2EJourney.fromScenario(page, e2eApi, e2eScenario); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/api.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/events/api.svelte.ts index 16bcce87b3..2554418dfa 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/api.svelte.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/api.svelte.ts @@ -13,18 +13,21 @@ import type { PersistentEvent } from './models'; export interface OrganizationEventNotificationRefresher { cancel: () => void; - schedule: (organizationId?: string, refreshImmediately?: boolean) => void; + schedule: (organizationId?: string, refreshImmediately?: boolean, includeStackLists?: boolean) => void; } export function createOrganizationEventNotificationRefresher(queryClient: QueryClient): OrganizationEventNotificationRefresher { const pendingOrganizationIds = new SvelteSet(); + let pendingStackListRefresh = false; let trailingRefresh: ReturnType | undefined; const refresh = () => { const organizationIds = [...pendingOrganizationIds]; + const includeStackLists = pendingStackListRefresh; void queryClient.invalidateQueries({ predicate: (query) => isOrganizationEventDashboardQueryKey(query.queryKey) && + (includeStackLists || !isOrganizationStackListQueryKey(query.queryKey)) && (organizationIds.includes(undefined) || organizationIds.includes(query.queryKey[2] as string)), queryKey: queryKeys.type, refetchType: 'active' @@ -34,13 +37,15 @@ export function createOrganizationEventNotificationRefresher(queryClient: QueryC return { cancel: () => { pendingOrganizationIds.clear(); + pendingStackListRefresh = false; if (trailingRefresh !== undefined) { clearTimeout(trailingRefresh); trailingRefresh = undefined; } }, - schedule: (organizationId?: string, refreshImmediately = true) => { + schedule: (organizationId?: string, refreshImmediately = true, includeStackLists = false) => { pendingOrganizationIds.add(organizationId); + pendingStackListRefresh ||= includeStackLists; if (trailingRefresh !== undefined) { return; } @@ -53,6 +58,7 @@ export function createOrganizationEventNotificationRefresher(queryClient: QueryC trailingRefresh = undefined; refresh(); pendingOrganizationIds.clear(); + pendingStackListRefresh = false; }, ORGANIZATION_EVENT_NOTIFICATION_THROTTLE_MS); } }; @@ -163,7 +169,7 @@ export interface GetEventsByReferenceRequest { }; } -export type GetEventsMode = 'stack_frequent' | 'stack_new' | 'stack_recent' | 'stack_users' | 'summary' | null; +export type GetEventsMode = 'stack' | 'summary' | null; export interface GetEventsParams { after?: string; @@ -183,7 +189,7 @@ export interface GetOrganizationCountRequest { params?: { aggregations?: string; filter?: string; - mode?: GetEventsMode; + mode?: 'stack'; offset?: string; time?: string; }; @@ -216,7 +222,7 @@ export interface GetProjectCountRequest { params?: { aggregations?: string; filter?: string; - mode?: 'stack_new'; + mode?: 'stack'; offset?: string; time?: string; }; @@ -246,7 +252,6 @@ export interface GetStackCountRequest { params?: { aggregations?: string; filter?: string; - mode?: 'stack_new'; offset?: string; time?: string; }; @@ -434,8 +439,8 @@ export function getOrganizationCountQuery(request: GetOrganizationCountRequest) }); } -export function getOrganizationEventsQuery(request: GetOrganizationEventsRequest) { - return createQuery[]>, ProblemDetails>(() => { +export function getOrganizationEventsQuery>(request: GetOrganizationEventsRequest) { + return createQuery, ProblemDetails>(() => { const organizationId = request.route.organizationId; const params = request.params ? { @@ -448,7 +453,7 @@ export function getOrganizationEventsQuery(request: GetOrganizationEventsRequest placeholderData: keepPreviousData, queryFn: async () => { const client = useFetchClient(); - return await client.getJSON[]>(`organizations/${organizationId}/events`, { + return await client.getJSON(`organizations/${organizationId}/events`, { params: params as Record }); }, @@ -642,3 +647,7 @@ function isOrganizationEventDashboardQueryKey(queryKey: readonly unknown[]): boo function isOrganizationEventsQueryKey(queryKey: readonly unknown[]): boolean { return queryKey[0] === queryKeys.type[0] && queryKey[1] === 'organizations' && queryKey[3] === 'events'; } + +function isOrganizationStackListQueryKey(queryKey: readonly unknown[]): boolean { + return isOrganizationEventsQueryKey(queryKey) && (queryKey[4] as GetEventsParams | undefined)?.mode === 'stack'; +} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/api.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/events/api.test.ts index d6e7407430..6501bfc387 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/api.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/api.test.ts @@ -87,6 +87,8 @@ describe('createOrganizationEventNotificationRefresher', () => { expect(firstInvalidation?.refetchType).toBe('active'); expect(firstInvalidation?.predicate?.({ queryKey: queryKeys.organizationsEvents('organization-id') } as never)).toBe(true); expect(firstInvalidation?.predicate?.({ queryKey: queryKeys.organizationsCount('organization-id') } as never)).toBe(true); + expect(firstInvalidation?.predicate?.({ queryKey: queryKeys.organizationsEvents('organization-id', { mode: 'stack' }) } as never)).toBe(false); + expect(firstInvalidation?.predicate?.({ queryKey: queryKeys.organizationsCount('organization-id', { mode: 'stack' }) } as never)).toBe(true); expect(firstInvalidation?.predicate?.({ queryKey: queryKeys.organizationsEvents('other-organization-id') } as never)).toBe(false); expect(firstInvalidation?.predicate?.({ queryKey: queryKeys.id('event-id') } as never)).toBe(false); @@ -105,6 +107,23 @@ describe('createOrganizationEventNotificationRefresher', () => { await vi.advanceTimersByTimeAsync(ORGANIZATION_EVENT_NOTIFICATION_THROTTLE_MS); expect(invalidateSpy).toHaveBeenCalledTimes(3); }); + + it('includes stack-mode lists only when scheduled by a stack notification', async () => { + // Arrange + vi.useFakeTimers(); + const queryClient = new QueryClient(); + const invalidateSpy = vi.spyOn(queryClient, 'invalidateQueries').mockImplementation(async () => {}); + const refresher = createOrganizationEventNotificationRefresher(queryClient); + + // Act + refresher.schedule('organization-id', true, true); + + // Assert + const invalidation = invalidateSpy.mock.calls[0]?.[0]; + expect(invalidation?.predicate?.({ queryKey: queryKeys.organizationsEvents('organization-id', { mode: 'stack' }) } as never)).toBe(true); + + refresher.cancel(); + }); }); describe('createEventWithNavigationQueryOptions', () => { diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/events-data-table.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/events-data-table.svelte index 03564d2ad5..ccdcf65329 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/events-data-table.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/events-data-table.svelte @@ -1,11 +1,11 @@ -