diff --git a/src/Exceptionless.Core/Jobs/CleanupDataJob.cs b/src/Exceptionless.Core/Jobs/CleanupDataJob.cs index f212c67cf4..65ebc19597 100644 --- a/src/Exceptionless.Core/Jobs/CleanupDataJob.cs +++ b/src/Exceptionless.Core/Jobs/CleanupDataJob.cs @@ -86,7 +86,7 @@ ILoggerFactory loggerFactory protected override Task GetLockAsync(CancellationToken cancellationToken = default) { - return _lockProvider.TryAcquireAsync(nameof(CleanupDataJob), TimeSpan.FromMinutes(15), cancellationToken); + return _lockProvider.TryAcquireAsync(nameof(CleanupDataJob), TimeSpan.FromHours(2), cancellationToken); } protected override async Task RunInternalAsync(JobContext context) @@ -201,8 +201,13 @@ private async Task CleanupSoftDeletedOrganizationsAsync(JobContext context) while (organizationResults.Documents.Count > 0 && !context.CancellationToken.IsCancellationRequested) { + await RenewLockAsync(context); + foreach (var organization in organizationResults.Documents) { + if (context.CancellationToken.IsCancellationRequested) + break; + using var _ = _logger.BeginScope(new ExceptionlessState().Organization(organization.Id)); try { @@ -229,8 +234,13 @@ private async Task CleanupSoftDeletedProjectsAsync(JobContext context) while (projectResults.Documents.Count > 0 && !context.CancellationToken.IsCancellationRequested) { + await RenewLockAsync(context); + foreach (var project in projectResults.Documents) { + if (context.CancellationToken.IsCancellationRequested) + break; + using var _ = _logger.BeginScope(new ExceptionlessState().Organization(project.OrganizationId).Project(project.Id)); try { @@ -383,8 +393,13 @@ private async Task EnforceRetentionAsync(JobContext context) var results = await _organizationRepository.FindAsync(q => q.Include(o => o.Id, o => o.Name, o => o.RetentionDays), o => o.SearchAfterPaging().PageLimit(100)); while (results.Documents.Count > 0 && !context.CancellationToken.IsCancellationRequested) { + await RenewLockAsync(context); + foreach (var organization in results.Documents) { + if (context.CancellationToken.IsCancellationRequested) + break; + using var _ = _logger.BeginScope(new ExceptionlessState().Organization(organization.Id)); int retentionDays = _billingManager.GetBillingPlanByUpsellingRetentionPeriod(organization.RetentionDays)?.RetentionDays ?? _appOptions.MaximumRetentionDays; @@ -454,6 +469,8 @@ private async Task EnforceEventRetentionDaysAsync(Organization organization, int private Task RenewLockAsync(JobContext context) { + // Called at each page boundary to prevent the distributed lock from expiring + // during long-running bulk cleanup operations that span multiple pages. _lastRun = _timeProvider.GetUtcNow().UtcDateTime; return context.RenewLockAsync(); } diff --git a/src/Exceptionless.Core/Jobs/CleanupOrphanedDataJob.cs b/src/Exceptionless.Core/Jobs/CleanupOrphanedDataJob.cs index 37a67ae24a..a8785970ef 100644 --- a/src/Exceptionless.Core/Jobs/CleanupOrphanedDataJob.cs +++ b/src/Exceptionless.Core/Jobs/CleanupOrphanedDataJob.cs @@ -1,51 +1,43 @@ -using Elastic.Clients.Elasticsearch; -using Elastic.Clients.Elasticsearch.Aggregations; -using Elastic.Clients.Elasticsearch.Core.ReindexRethrottle; -using Elastic.Clients.Elasticsearch.QueryDsl; using Exceptionless.Core.Models; using Exceptionless.Core.Repositories; -using Exceptionless.Core.Repositories.Configuration; using Foundatio.Caching; using Foundatio.Jobs; using Foundatio.Lock; using Foundatio.Repositories; -using Foundatio.Repositories.Elasticsearch.Extensions; -using Foundatio.Repositories.Elasticsearch.Utility; -using Foundatio.Repositories.Extensions; +using Foundatio.Repositories.Exceptions; using Foundatio.Repositories.Models; using Foundatio.Resilience; using Microsoft.Extensions.Diagnostics.HealthChecks; using Microsoft.Extensions.Logging; -using LogLevel = Microsoft.Extensions.Logging.LogLevel; namespace Exceptionless.Core.Jobs; [Job(Description = "Deletes orphaned data.", IsContinuous = false)] public class CleanupOrphanedDataJob : JobWithLockBase, IHealthCheck { - private readonly ExceptionlessElasticConfiguration _config; - private readonly ElasticsearchClient _elasticClient; - private readonly IStackRepository _stackRepository; - private readonly IProjectRepository _projectRepository; private readonly IOrganizationRepository _organizationRepository; + private readonly IProjectRepository _projectRepository; + private readonly IStackRepository _stackRepository; private readonly IEventRepository _eventRepository; private readonly ICacheClient _cacheClient; private readonly ILockProvider _lockProvider; private DateTime? _lastRun; - public CleanupOrphanedDataJob(ExceptionlessElasticConfiguration config, IStackRepository stackRepository, - IProjectRepository projectRepository, IOrganizationRepository organizationRepository, - IEventRepository eventRepository, ICacheClient cacheClient, ILockProvider lockProvider, + public CleanupOrphanedDataJob( + IOrganizationRepository organizationRepository, + IProjectRepository projectRepository, + IStackRepository stackRepository, + IEventRepository eventRepository, + ICacheClient cacheClient, + ILockProvider lockProvider, TimeProvider timeProvider, IResiliencePolicyProvider resiliencePolicyProvider, ILoggerFactory loggerFactory ) : base(timeProvider, resiliencePolicyProvider, loggerFactory) { - _config = config; - _elasticClient = config.Client; - _stackRepository = stackRepository; - _projectRepository = projectRepository; _organizationRepository = organizationRepository; + _projectRepository = projectRepository; + _stackRepository = stackRepository; _eventRepository = eventRepository; _cacheClient = cacheClient; _lockProvider = lockProvider; @@ -58,6 +50,8 @@ ILoggerFactory loggerFactory protected override async Task RunInternalAsync(JobContext context) { + _lastRun = _timeProvider.GetUtcNow().UtcDateTime; + await DeleteOrphanedEventsByStackAsync(context); await DeleteOrphanedEventsByProjectAsync(context); await DeleteOrphanedEventsByOrganizationAsync(context); @@ -69,200 +63,177 @@ protected override async Task RunInternalAsync(JobContext context) public async Task DeleteOrphanedEventsByStackAsync(JobContext context) { - // get approximate number of unique stack ids - var stackCardinality = await _elasticClient.SearchAsync(s => s - .Indices(GetEventIndexPattern()) - .Size(0) - .AddAggregation("cardinality_stack_id", a => a.Cardinality(c => c.Field(f => f.StackId).PrecisionThreshold(40000)))); - - double? uniqueStackIdCount = stackCardinality.Aggregations?.GetCardinality("cardinality_stack_id")?.Value; - if (!uniqueStackIdCount.HasValue || uniqueStackIdCount.Value <= 0) - return; - - // break into batches of 500 - const int batchSize = 500; - int buckets = (int)uniqueStackIdCount.Value / batchSize; - buckets = Math.Max(1, buckets); - int totalOrphanedEventCount = 0; - int totalStackIds = 0; - - for (int batchNumber = 0; batchNumber < buckets; batchNumber++) + _logger.LogInformation("Starting orphaned events cleanup by stack"); + long totalOrphanedEvents = 0; + long totalStackIds = 0; + string? nextValue = null; + bool hasMore = true; + + while (hasMore && !context.CancellationToken.IsCancellationRequested) { await RenewLockAsync(context); - var stackIdTerms = await _elasticClient.SearchAsync(s => s - .Indices(GetEventIndexPattern()) - .Size(0) - .AddAggregation("terms_stack_id", a => a.Terms(c => c.Field(f => f.StackId).Include(new TermsInclude(batchNumber, buckets)).Size(batchSize * 2)))); + var page = await _eventRepository.GetDistinctStackIdsAsync(500, nextValue, context.CancellationToken); + var stackIds = page.Values; + if (stackIds.Count == 0) + break; - string[] stackIds = stackIdTerms.Aggregations?.GetStringTerms("terms_stack_id")?.Buckets.Select(b => b.Key.ToString()!).ToArray() ?? []; - if (stackIds.Length == 0) - continue; + nextValue = page.NextValue; + hasMore = !String.IsNullOrEmpty(nextValue); + totalStackIds += stackIds.Count; - totalStackIds += stackIds.Length; - - var stacks = await _stackRepository.GetByIdsAsync(stackIds, o => o.ImmediateConsistency()); - var foundStackIds = stacks.Select(stack => stack.Id).ToHashSet(StringComparer.OrdinalIgnoreCase); - string[] missingStackIds = stackIds.Where(stackId => !foundStackIds.Contains(stackId)).ToArray(); + var existingStacks = await _stackRepository.GetByIdsAsync(stackIds.ToArray(), o => o.Include(s => s.Id, s => s.IsDeleted)); + var existingStackIds = existingStacks.Select(s => s.Id).ToHashSet(StringComparer.OrdinalIgnoreCase); + string[] missingStackIds = stackIds.Where(id => !existingStackIds.Contains(id)).ToArray(); if (missingStackIds.Length == 0) - { - _logger.LogInformation("{BatchNumber}/{BatchCount}: Did not find any missing stacks out of {StackIdCount}", batchNumber, buckets, stackIds.Length); continue; - } - totalOrphanedEventCount += missingStackIds.Length; - _logger.LogInformation("{BatchNumber}/{BatchCount}: Found {OrphanedEventCount} orphaned events from missing stacks {MissingStackIds} out of {StackIdCount}", batchNumber, buckets, missingStackIds.Length, missingStackIds, stackIds.Length); - await _elasticClient.DeleteByQueryAsync(r => r - .Indices(GetEventIndexPattern()) - .Query(q => q.Terms(t => t.Field(f => f.StackId).Terms(new TermsQueryField(missingStackIds.Select(FieldValueHelper.ToFieldValue).ToList()))))); + // Redirect tombstones are intentionally retained so events from an in-flight ingestion + // context can never be mistaken for orphaned data. Move those late events to the + // canonical stack and refresh its metadata before deleting only truly missing stacks. + var redirectedStacks = await _stackRepository.GetByIdsAsync( + missingStackIds, + o => o.SoftDeleteMode(SoftDeleteQueryMode.All)); + var redirectedStackIds = redirectedStacks + .Where(s => !String.IsNullOrEmpty(s.RedirectToStackId)) + .Select(s => s.Id) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + var redirectedStackDocuments = redirectedStacks + .Where(s => redirectedStackIds.Contains(s.Id)) + .ToList(); + + if (redirectedStackDocuments.Count > 0) + await ReconcileRedirectedStacksAsync(redirectedStackDocuments, context); + + string[] orphanedStackIds = missingStackIds.Where(id => !redirectedStackIds.Contains(id)).ToArray(); + if (orphanedStackIds.Length == 0) + continue; + + long deletedCount = await _eventRepository.RemoveAllByStackIdsAsync(orphanedStackIds, o => o.Notifications(false)); + totalOrphanedEvents += deletedCount; + + _logger.LogInformation("Deleted {DeletedCount} orphaned events from {MissingStackCount} missing stacks out of {StackIdCount} checked", deletedCount, orphanedStackIds.Length, stackIds.Count); } - _logger.LogInformation("Found {OrphanedEventCount} orphaned events from missing stacks out of {StackIdCount}", totalOrphanedEventCount, totalStackIds); + await ReconcileDirtyRedirectedStacksAsync(context); + + _logger.LogInformation("Completed orphaned events cleanup by stack: deleted {TotalOrphanedEvents} events, checked {TotalStackIds} stacks", totalOrphanedEvents, totalStackIds); } public async Task DeleteOrphanedEventsByProjectAsync(JobContext context) { - // get approximate number of unique project ids - var projectCardinality = await _elasticClient.SearchAsync(s => s - .Indices(GetEventIndexPattern()) - .Size(0) - .AddAggregation("cardinality_project_id", a => a.Cardinality(c => c.Field(f => f.ProjectId).PrecisionThreshold(40000)))); - - double? uniqueProjectIdCount = projectCardinality.Aggregations?.GetCardinality("cardinality_project_id")?.Value; - if (!uniqueProjectIdCount.HasValue || uniqueProjectIdCount.Value <= 0) - return; - - // break into batches of 500 - const int batchSize = 500; - int buckets = (int)uniqueProjectIdCount.Value / batchSize; - buckets = Math.Max(1, buckets); - int totalOrphanedEventCount = 0; - int totalProjectIds = 0; - - for (int batchNumber = 0; batchNumber < buckets; batchNumber++) + _logger.LogInformation("Starting orphaned events cleanup by project"); + long totalOrphanedEvents = 0; + long totalProjectIds = 0; + string? nextValue = null; + bool hasMore = true; + + while (hasMore && !context.CancellationToken.IsCancellationRequested) { await RenewLockAsync(context); - var projectIdTerms = await _elasticClient.SearchAsync(s => s - .Indices(GetEventIndexPattern()) - .Size(0) - .AddAggregation("terms_project_id", a => a.Terms(c => c.Field(f => f.ProjectId).Include(new TermsInclude(batchNumber, buckets)).Size(batchSize * 2)))); + var page = await _eventRepository.GetDistinctProjectIdsAsync(500, nextValue, context.CancellationToken); + var projectIds = page.Values; + if (projectIds.Count == 0) + break; - string[] projectIds = projectIdTerms.Aggregations?.GetStringTerms("terms_project_id")?.Buckets.Select(b => b.Key.ToString()!).ToArray() ?? []; - if (projectIds.Length == 0) - continue; + nextValue = page.NextValue; + hasMore = !String.IsNullOrEmpty(nextValue); + totalProjectIds += projectIds.Count; - totalProjectIds += projectIds.Length; - - var projects = await _projectRepository.GetByIdsAsync(projectIds, o => o.ImmediateConsistency()); - var foundProjectIds = projects.Select(project => project.Id).ToHashSet(StringComparer.OrdinalIgnoreCase); - string[] missingProjectIds = projectIds.Where(projectId => !foundProjectIds.Contains(projectId)).ToArray(); + var existingProjects = await _projectRepository.GetByIdsAsync(projectIds.ToArray(), o => o.Include(p => p.Id, p => p.IsDeleted)); + var existingProjectIds = existingProjects.Select(p => p.Id).ToHashSet(StringComparer.OrdinalIgnoreCase); + string[] missingProjectIds = projectIds.Where(id => !existingProjectIds.Contains(id)).ToArray(); if (missingProjectIds.Length == 0) - { - _logger.LogInformation("{BatchNumber}/{BatchCount}: Did not find any missing projects out of {ProjectIdCount}", batchNumber, buckets, projectIds.Length); continue; - } - totalOrphanedEventCount += missingProjectIds.Length; - _logger.LogInformation("{BatchNumber}/{BatchCount}: Found {OrphanedEventCount} orphaned events from missing projects {MissingProjectIds} out of {ProjectIdCount}", batchNumber, buckets, missingProjectIds.Length, missingProjectIds, projectIds.Length); - await _elasticClient.DeleteByQueryAsync(r => r - .Indices(GetEventIndexPattern()) - .Query(q => q.Terms(t => t.Field(f => f.ProjectId).Terms(new TermsQueryField(missingProjectIds.Select(FieldValueHelper.ToFieldValue).ToList()))))); + long deletedCount = await _eventRepository.RemoveAllByProjectIdsAsync(missingProjectIds, o => o.Notifications(false)); + totalOrphanedEvents += deletedCount; + + _logger.LogInformation("Deleted {DeletedCount} orphaned events from {MissingProjectCount} missing projects out of {ProjectIdCount} checked", deletedCount, missingProjectIds.Length, projectIds.Count); } - _logger.LogInformation("Found {OrphanedEventCount} orphaned events from missing projects out of {ProjectIdCount}", totalOrphanedEventCount, totalProjectIds); + _logger.LogInformation("Completed orphaned events cleanup by project: deleted {TotalOrphanedEvents} events, checked {TotalProjectIds} projects", totalOrphanedEvents, totalProjectIds); } public async Task DeleteOrphanedEventsByOrganizationAsync(JobContext context) { - // get approximate number of unique organization ids - var organizationCardinality = await _elasticClient.SearchAsync(s => s - .Indices(GetEventIndexPattern()) - .Size(0) - .AddAggregation("cardinality_organization_id", a => a.Cardinality(c => c.Field(f => f.OrganizationId).PrecisionThreshold(40000)))); - - double? uniqueOrganizationIdCount = organizationCardinality.Aggregations?.GetCardinality("cardinality_organization_id")?.Value; - if (!uniqueOrganizationIdCount.HasValue || uniqueOrganizationIdCount.Value <= 0) - return; - - // break into batches of 500 - const int batchSize = 500; - int buckets = (int)uniqueOrganizationIdCount.Value / batchSize; - buckets = Math.Max(1, buckets); - int totalOrphanedEventCount = 0; - int totalOrganizationIds = 0; - - for (int batchNumber = 0; batchNumber < buckets; batchNumber++) + _logger.LogInformation("Starting orphaned events cleanup by organization"); + long totalOrphanedEvents = 0; + long totalOrganizationIds = 0; + string? nextValue = null; + bool hasMore = true; + + while (hasMore && !context.CancellationToken.IsCancellationRequested) { await RenewLockAsync(context); - var organizationIdTerms = await _elasticClient.SearchAsync(s => s - .Indices(GetEventIndexPattern()) - .Size(0) - .AddAggregation("terms_organization_id", a => a.Terms(c => c.Field(f => f.OrganizationId).Include(new TermsInclude(batchNumber, buckets)).Size(batchSize * 2)))); + var page = await _eventRepository.GetDistinctOrganizationIdsAsync(500, nextValue, context.CancellationToken); + var organizationIds = page.Values; + if (organizationIds.Count == 0) + break; - string[] organizationIds = organizationIdTerms.Aggregations?.GetStringTerms("terms_organization_id")?.Buckets.Select(b => b.Key.ToString()!).ToArray() ?? []; - if (organizationIds.Length == 0) - continue; - - totalOrganizationIds += organizationIds.Length; + nextValue = page.NextValue; + hasMore = !String.IsNullOrEmpty(nextValue); + totalOrganizationIds += organizationIds.Count; - var organizations = await _organizationRepository.GetByIdsAsync(organizationIds, o => o.ImmediateConsistency()); - var foundOrganizationIds = organizations.Select(organization => organization.Id).ToHashSet(StringComparer.OrdinalIgnoreCase); - string[] missingOrganizationIds = organizationIds.Where(organizationId => !foundOrganizationIds.Contains(organizationId)).ToArray(); + var existingOrganizations = await _organizationRepository.GetByIdsAsync(organizationIds.ToArray(), o => o.Include(organization => organization.Id, organization => organization.IsDeleted)); + var existingOrganizationIds = existingOrganizations.Select(organization => organization.Id).ToHashSet(StringComparer.OrdinalIgnoreCase); + string[] missingOrganizationIds = organizationIds.Where(id => !existingOrganizationIds.Contains(id)).ToArray(); if (missingOrganizationIds.Length == 0) - { - _logger.LogInformation("{BatchNumber}/{BatchCount}: Did not find any missing organizations out of {OrganizationIdCount}", batchNumber, buckets, organizationIds.Length); continue; - } - totalOrphanedEventCount += missingOrganizationIds.Length; - _logger.LogInformation("{BatchNumber}/{BatchCount}: Found {OrphanedEventCount} orphaned events from missing organizations {MissingOrganizationIds} out of {OrganizationIdCount}", batchNumber, buckets, missingOrganizationIds.Length, missingOrganizationIds, organizationIds.Length); - await _elasticClient.DeleteByQueryAsync(r => r - .Indices(GetEventIndexPattern()) - .Query(q => q.Terms(t => t.Field(f => f.OrganizationId).Terms(new TermsQueryField(missingOrganizationIds.Select(FieldValueHelper.ToFieldValue).ToList()))))); + long deletedCount = await _eventRepository.RemoveAllByOrganizationIdsAsync(missingOrganizationIds, o => o.Notifications(false)); + totalOrphanedEvents += deletedCount; + + _logger.LogInformation("Deleted {DeletedCount} orphaned events from {MissingOrganizationCount} missing organizations out of {OrganizationIdCount} checked", deletedCount, missingOrganizationIds.Length, organizationIds.Count); } - _logger.LogInformation("Found {OrphanedEventCount} orphaned events from missing organizations out of {OrganizationIdCount}", totalOrphanedEventCount, totalOrganizationIds); + _logger.LogInformation("Completed orphaned events cleanup by organization: deleted {TotalOrphanedEvents} events, checked {TotalOrganizationIds} organizations", totalOrphanedEvents, totalOrganizationIds); } public async Task FixDuplicateStacks(JobContext context) { _logger.LogInformation("Getting duplicate stacks"); - var duplicateStackAgg = await _elasticClient.SearchAsync(q => q - .Indices(_config.Stacks.VersionedName) - .Query(q => q.QueryString(qs => qs.Query("is_deleted:false"))) - .Size(0) - .AddAggregation("stacks", a => a.Terms(t => t.Field(f => f.DuplicateSignature).MinDocCount(2).Size(10000)))); - _logger.LogRequest(duplicateStackAgg, LogLevel.Trace); - - var buckets = duplicateStackAgg.Aggregations?.GetStringTerms("stacks")?.Buckets.ToList() ?? []; - int total = buckets.Count; + int total = 0; int processed = 0; int error = 0; long totalUpdatedEventCount = 0; var lastStatus = _timeProvider.GetUtcNow().UtcDateTime; - int batch = 1; + int batch = 0; - while (buckets.Count > 0) + // Loop until no more duplicate signatures exist. Each iteration forces an index refresh + // (via ImmediateConsistency on GetDuplicateSignaturesAsync) so soft-deleted stacks are + // excluded from subsequent aggregation calls, preventing re-processing. + while (!context.CancellationToken.IsCancellationRequested) { - _logger.LogInformation($"Found {buckets.Count} duplicate stacks in batch #{batch}."); + var duplicateSignatures = await _stackRepository.GetDuplicateSignaturesAsync(); + if (duplicateSignatures.Count == 0) + break; + + batch++; + total += duplicateSignatures.Count; + _logger.LogInformation("Found {Total} duplicate stacks in batch #{Batch}", duplicateSignatures.Count, batch); await RenewLockAsync(context); - foreach (var duplicateSignature in buckets) + int batchProcessed = 0; + foreach (var duplicateSignature in duplicateSignatures) { + if (context.CancellationToken.IsCancellationRequested) + break; + string? projectId = null; string? signature = null; try { - string[] parts = duplicateSignature.Key.ToString().Split(':'); + string[] parts = duplicateSignature.Split(':'); if (parts.Length != 2) { - _logger.LogError("Error parsing duplicate signature {DuplicateSignature}", duplicateSignature.Key.ToString()); + _logger.LogError("Error parsing duplicate signature {DuplicateSignature}", duplicateSignature); continue; } projectId = parts[0]; @@ -275,123 +246,178 @@ public async Task FixDuplicateStacks(JobContext context) continue; } + var targetCandidates = stacks.Documents.Where(s => String.IsNullOrEmpty(s.RedirectToStackId)).ToList(); + if (targetCandidates.Count == 0) + { + _logger.LogError("Did not find a canonical stack for signature {SignatureHash} and project {ProjectId}", signature, projectId); + continue; + } + var eventCounts = await _eventRepository.CountAsync(q => q.Stack(stacks.Documents.Select(s => s.Id)).AggregationsExpression("terms:stack_id")); - var eventCountBuckets = eventCounts.Aggregations.Terms("terms_stack_id")?.Buckets ?? new List>(); + var eventCountBuckets = eventCounts.Aggregations.Terms("terms_stack_id")?.Buckets ?? new List>(); - // we only need to update events if more than one stack has events associated to it + // We only need to update events if more than one stack has events associated to it. bool shouldUpdateEvents = eventCountBuckets.Count > 1; - // default to using the oldest stack - var targetStack = stacks.Documents.OrderBy(s => s.CreatedUtc).First(); - var duplicateStacks = stacks.Documents.OrderBy(s => s.CreatedUtc).Skip(1).ToList(); + // Default to using the oldest stack. + var targetStack = targetCandidates.OrderBy(s => s.CreatedUtc).First(); + var duplicateStacks = stacks.Documents.Where(s => s.Id != targetStack.Id).OrderBy(s => s.CreatedUtc).ToList(); - // use the stack that has the most events on it so we can reduce the number of updates - if (eventCountBuckets.Count > 0) + // Use the stack that has the most events on it so we can reduce the number of updates. + var targetCandidateIds = targetCandidates.Select(s => s.Id).ToHashSet(StringComparer.Ordinal); + var targetBuckets = eventCountBuckets.Where(b => targetCandidateIds.Contains(b.Key)).ToList(); + if (targetBuckets.Count > 0) { - string targetStackId = eventCountBuckets.OrderByDescending(b => b.Total).First().Key; - targetStack = stacks.Documents.Single(d => d.Id == targetStackId); - duplicateStacks = stacks.Documents.Where(d => d.Id != targetStackId).ToList(); + string targetStackId = targetBuckets.OrderByDescending(b => b.Total).First().Key; + targetStack = targetCandidates.Single(d => d.Id == targetStackId); + duplicateStacks = stacks.Documents.Where(d => d.Id != targetStackId).OrderBy(d => d.CreatedUtc).ToList(); } - targetStack.CreatedUtc = stacks.Documents.Min(d => d.CreatedUtc); - targetStack.Status = stacks.Documents.FirstOrDefault(d => d.Status != StackStatus.Open)?.Status ?? StackStatus.Open; - targetStack.LastOccurrence = stacks.Documents.Max(d => d.LastOccurrence); - targetStack.SnoozeUntilUtc = stacks.Documents.Max(d => d.SnoozeUntilUtc); - targetStack.DateFixed = stacks.Documents.Max(d => d.DateFixed); ; - targetStack.TotalOccurrences += duplicateStacks.Sum(d => d.TotalOccurrences); - targetStack.Tags.AddRange(duplicateStacks.SelectMany(d => d.Tags)); - targetStack.References = stacks.Documents.SelectMany(d => d.References).Distinct().ToList(); - targetStack.OccurrencesAreCritical = stacks.Documents.Any(d => d.OccurrencesAreCritical); - - duplicateStacks.ForEach(s => s.IsDeleted = true); - await _stackRepository.SaveAsync(duplicateStacks); - await _stackRepository.SaveAsync(targetStack); - processed++; + // Publish durable redirects before moving events. New ingestion resolves these + // immediately, and any already in-flight writes are preserved by the orphan pass. + foreach (var duplicateStack in duplicateStacks) + await _stackRepository.SetDuplicateStackRedirectAsync(duplicateStack, targetStack.Id); - long eventsToMove = eventCountBuckets.Where(b => b.Key != targetStack.Id).Sum(b => b.Total) ?? 0; - _logger.LogInformation("De-duped stack: Target={TargetId} Events={EventCount} Dupes={DuplicateIds} HasEvents={HasEvents}", targetStack.Id, eventsToMove, duplicateStacks.Select(s => s.Id), shouldUpdateEvents); + // Always run strict verification, even when open-index counts show no source + // events. Closed or unavailable daily indices must block stack deletion. + long affectedRecords = await AwaitWithLockRenewalAsync( + _eventRepository.ReassignStackAsync( + duplicateStacks.Select(s => s.Id), targetStack.Id, context.CancellationToken), + context); + + _logger.LogInformation("Migrated stack events: Target={TargetId} Events={UpdatedEvents} Dupes={DuplicateIds}", targetStack.Id, affectedRecords, duplicateStacks.Select(s => s.Id)); + totalUpdatedEventCount += affectedRecords; + + context.CancellationToken.ThrowIfCancellationRequested(); - if (shouldUpdateEvents) + // Apply each source's metadata to the target using a durable occurrence ledger, + // then hide the redirected source. This ordering is retry-safe when any write fails. + foreach (var duplicateStack in duplicateStacks) { - var response = await _elasticClient.UpdateByQueryAsync(u => u - .Indices(GetEventIndexPattern()) - .Query(q => q.Bool(b => b.Must(m => m - .Terms(t => t.Field(f => f.StackId).Terms(new TermsQueryField(duplicateStacks.Select(s => FieldValueHelper.ToFieldValue(s.Id)).ToList()))) - ))) - .Script(s => s.Source($"ctx._source.stack_id = '{targetStack.Id}'").Lang(ScriptLanguage.Painless)) - .Conflicts(Conflicts.Proceed) - .WaitForCompletion(false)); - _logger.LogRequest(response, LogLevel.Trace); - - var taskStartedTime = _timeProvider.GetUtcNow().UtcDateTime; - var taskId = response.Task; - int attempts = 0; - long affectedRecords = 0; - do - { - attempts++; - var taskStatus = await _elasticClient.Tasks.GetAsync(taskId!.FullyQualifiedId); - var status = taskStatus.Task.Status as ReindexStatus; - if (taskStatus.Completed) - { - // TODO: need to check to see if the task failed or completed successfully. Throw if it failed. - if (_timeProvider.GetUtcNow().UtcDateTime.Subtract(taskStartedTime) > TimeSpan.FromSeconds(30)) - _logger.LogInformation("Script operation task ({TaskId}) completed: Created: {Created} Updated: {Updated} Deleted: {Deleted} Conflicts: {Conflicts} Total: {Total}", taskId, status?.Created, status?.Updated, status?.Deleted, status?.VersionConflicts, status?.Total); - - affectedRecords += (status?.Created ?? 0) + (status?.Updated ?? 0) + (status?.Deleted ?? 0); - break; - } - - if (_timeProvider.GetUtcNow().UtcDateTime.Subtract(taskStartedTime) > TimeSpan.FromSeconds(30)) - { - await RenewLockAsync(context); - _logger.LogInformation("Checking script operation task ({TaskId}) status: Created: {Created} Updated: {Updated} Deleted: {Deleted} Conflicts: {Conflicts} Total: {Total}", taskId, status?.Created, status?.Updated, status?.Deleted, status?.VersionConflicts, status?.Total); - } - - var delay = TimeSpan.FromMilliseconds(50); - if (attempts > 20) - delay = TimeSpan.FromSeconds(5); - else if (attempts > 10) - delay = TimeSpan.FromSeconds(1); - else if (attempts > 5) - delay = TimeSpan.FromMilliseconds(250); - - await Task.Delay(delay, _timeProvider, context.CancellationToken); - } while (true); - - _logger.LogInformation("Migrated stack events: Target={TargetId} Events={UpdatedEvents} Dupes={DuplicateIds}", targetStack.Id, affectedRecords, duplicateStacks.Select(s => s.Id)); - - totalUpdatedEventCount += affectedRecords; + await _stackRepository.MergeDuplicateStackAsync(targetStack.Id, duplicateStack); + await _stackRepository.SetDuplicateStackRedirectAsync(duplicateStack, targetStack.Id, isDeleted: true); } + processed++; + batchProcessed++; + + long eventsToMove = eventCountBuckets.Where(b => b.Key != targetStack.Id).Sum(b => b.Total) ?? 0; + _logger.LogInformation("De-duped stack: Target={TargetId} Events={EventCount} Dupes={DuplicateIds} HasEvents={HasEvents}", targetStack.Id, eventsToMove, duplicateStacks.Select(s => s.Id), shouldUpdateEvents); + if (_timeProvider.GetUtcNow().UtcDateTime.Subtract(lastStatus) > TimeSpan.FromSeconds(5)) { lastStatus = _timeProvider.GetUtcNow().UtcDateTime; + await RenewLockAsync(context); _logger.LogInformation("Total={Processed}/{Total} Errors={ErrorCount}", processed, total, error); - await _cacheClient.RemoveByPrefixAsync(nameof(Stack)); } } + catch (OperationCanceledException) when (context.CancellationToken.IsCancellationRequested) + { + throw; + } catch (Exception ex) { + // Intentionally broad: log and continue processing other groups rather than + // aborting the entire job for a single corrupt or transiently failing signature. error++; - _logger.LogError(ex, "Error fixing duplicate stack {ProjectId} {SignatureHash}", projectId, signature); + _logger.LogError(ex, "Error fixing duplicate stack {ProjectId} {SignatureHash}: {Message}", projectId, signature, ex.Message); } } - await _elasticClient.Indices.RefreshAsync(_config.Stacks.VersionedName); - duplicateStackAgg = await _elasticClient.SearchAsync(q => q - .Indices(_config.Stacks.VersionedName) - .Query(q => q.QueryString(qs => qs.Query("is_deleted:false"))) - .Size(0) - .AddAggregation("stacks", a => a.Terms(t => t.Field(f => f.DuplicateSignature).MinDocCount(2).Size(10000)))); - _logger.LogRequest(duplicateStackAgg, LogLevel.Trace); + _logger.LogInformation("Batch #{Batch} complete: Processed={BatchProcessed} Total={Processed}/{Total} Errors={ErrorCount} UpdatedEvents={UpdatedEventCount}", batch, batchProcessed, processed, total, error, totalUpdatedEventCount); + await _cacheClient.RemoveByPrefixAsync(nameof(Stack)); + + // If nothing was processed this batch (all errors), stop to avoid an infinite loop + // where the same failing signatures are retried indefinitely. + if (batchProcessed == 0) + break; + } - buckets = duplicateStackAgg.Aggregations?.GetStringTerms("stacks")?.Buckets.ToList() ?? []; - total += buckets.Count; - batch++; + _logger.LogInformation("Done de-duping stacks: Total={Processed}/{Total} Errors={ErrorCount} UpdatedEvents={UpdatedEventCount}", processed, total, error, totalUpdatedEventCount); + } - _logger.LogInformation("Done de-duping stacks: Total={Processed}/{Total} Errors={ErrorCount}", processed, total, error); - await _cacheClient.RemoveByPrefixAsync(nameof(Stack)); + private async Task ReconcileDirtyRedirectedStacksAsync(JobContext context) + { + var redirectedStacks = await _stackRepository.GetRedirectedStacksNeedingReconciliationAsync(); + while (redirectedStacks.Documents.Count > 0 && !context.CancellationToken.IsCancellationRequested) + { + await ReconcileRedirectedStacksAsync(redirectedStacks.Documents, context); + + if (!await redirectedStacks.NextPageAsync()) + break; + } + } + + private async Task ReconcileRedirectedStacksAsync(IReadOnlyCollection redirectedStacks, JobContext context) + { + var stacksByTarget = new Dictionary Sources)>(StringComparer.Ordinal); + foreach (var sourceStack in redirectedStacks) + { + context.CancellationToken.ThrowIfCancellationRequested(); + + try + { + var targetStack = await _stackRepository.GetCanonicalStackAsync(sourceStack.RedirectToStackId!); + if (targetStack is null) + { + _logger.LogWarning( + "Preserving redirected stack {SourceStackId} because target stack {TargetStackId} is unavailable", + sourceStack.Id, + sourceStack.RedirectToStackId); + continue; + } + + if (!stacksByTarget.TryGetValue(targetStack.Id, out var group)) + { + group = (targetStack, []); + stacksByTarget[targetStack.Id] = group; + } + + group.Sources.Add(sourceStack); + } + catch (DocumentException ex) + { + _logger.LogError( + ex, + "Unable to resolve redirected stack {SourceStackId} to target {TargetStackId}", + sourceStack.Id, + sourceStack.RedirectToStackId); + } + } + + foreach (var (_, group) in stacksByTarget) + { + // Merge metadata first. If event reassignment fails, the contribution ledger makes + // this safe to retry. Counter patches mark a tombstone dirty, while event-bearing + // tombstones are discovered directly by the orphan scan. + foreach (var sourceStack in group.Sources) + await _stackRepository.MergeDuplicateStackAsync(group.Target.Id, sourceStack); + + long reassigned = await AwaitWithLockRenewalAsync( + _eventRepository.ReassignStackAsync( + group.Sources.Select(s => s.Id), group.Target.Id, context.CancellationToken), + context); + + foreach (var sourceStack in group.Sources) + { + if (!sourceStack.IsDeleted) + { + await _stackRepository.SetDuplicateStackRedirectAsync(sourceStack, group.Target.Id, isDeleted: true); + sourceStack.IsDeleted = true; + sourceStack.RedirectToStackId = group.Target.Id; + } + + await _stackRepository.MarkDuplicateStackReconciledAsync(sourceStack); + } + + if (reassigned > 0) + { + _logger.LogInformation( + "Reassigned {EventCount} late event(s) from {SourceStackCount} duplicate stack(s) to {TargetStackId}", + reassigned, + group.Sources.Count, + group.Target.Id); + } } } @@ -401,9 +427,18 @@ private Task RenewLockAsync(JobContext context) return context.RenewLockAsync(); } - private string GetEventIndexPattern() + private async Task AwaitWithLockRenewalAsync(Task operation, JobContext context) { - return $"{_config.Events.VersionedName}-*"; + while (!operation.IsCompleted) + { + var completed = await Task.WhenAny(operation, Task.Delay(TimeSpan.FromSeconds(30), _timeProvider)); + if (completed == operation) + break; + + await RenewLockAsync(context); + } + + return await operation; } public Task CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default) diff --git a/src/Exceptionless.Core/Jobs/StackStatusJob.cs b/src/Exceptionless.Core/Jobs/StackStatusJob.cs index 0f0c8d3d2f..294916e6ce 100644 --- a/src/Exceptionless.Core/Jobs/StackStatusJob.cs +++ b/src/Exceptionless.Core/Jobs/StackStatusJob.cs @@ -1,5 +1,4 @@ -using Exceptionless.Core.Extensions; -using Exceptionless.Core.Repositories; +using Exceptionless.Core.Repositories; using Foundatio.Caching; using Foundatio.Jobs; using Foundatio.Lock; @@ -42,10 +41,7 @@ protected override async Task RunInternalAsync(JobContext context) var results = await _stackRepository.GetExpiredSnoozedStatuses(_timeProvider.GetUtcNow().UtcDateTime, o => o.PageLimit(LIMIT)); while (results.Documents.Count > 0 && !context.CancellationToken.IsCancellationRequested) { - foreach (var stack in results.Documents) - stack.MarkOpen(); - - await _stackRepository.SaveAsync(results.Documents); + await _stackRepository.MarkOpenAsync(results.Documents.Select(stack => stack.Id)); // Sleep so we are not hammering the backend. await Task.Delay(TimeSpan.FromSeconds(2.5), _timeProvider); diff --git a/src/Exceptionless.Core/Models/Stack.cs b/src/Exceptionless.Core/Models/Stack.cs index 911835df31..f5233b39ea 100644 --- a/src/Exceptionless.Core/Models/Stack.cs +++ b/src/Exceptionless.Core/Models/Stack.cs @@ -4,12 +4,13 @@ using System.Runtime.Serialization; using System.Text.Json.Serialization; using Exceptionless.Core.Attributes; +using Exceptionless.Core.Serialization; using Foundatio.Repositories.Models; namespace Exceptionless.Core.Models; [DebuggerDisplay("Id={Id} Type={Type} Status={Status} IsDeleted={IsDeleted} Title={Title} TotalOccurrences={TotalOccurrences}")] -public class Stack : IOwnedByOrganizationAndProjectWithIdentity, IHaveDates, ISupportSoftDeletes, IValidatableObject +public class Stack : IOwnedByOrganizationAndProjectWithIdentity, IHaveDates, ISupportSoftDeletes, IVersioned, IValidatableObject { /// /// Unique id that identifies a stack. @@ -120,6 +121,36 @@ public class Stack : IOwnedByOrganizationAndProjectWithIdentity, IHaveDates, ISu public DateTime UpdatedUtc { get; set; } public bool IsDeleted { get; set; } + /// + /// The canonical stack for events that still reference this duplicate stack. + /// This is internal cleanup state and is not part of the public API contract. + /// + [JsonInclude] + [JsonIgnoreForExternalSerialization] + internal string? RedirectToStackId { get; set; } + + /// + /// Tracks how many occurrences from each duplicate stack have already been merged. + /// This is internal cleanup state and is not part of the public API contract. + /// + [JsonInclude] + [JsonIgnoreForExternalSerialization] + internal IDictionary MergedDuplicateStackTotals { get; set; } = new Dictionary(); + + [JsonInclude] + [JsonIgnoreForExternalSerialization] + internal bool NeedsRedirectReconciliation { get; set; } + + [JsonInclude] + [JsonIgnoreForExternalSerialization] + internal string ElasticVersion { get; set; } = null!; + + string IVersioned.Version + { + get => ElasticVersion; + set => ElasticVersion = value; + } + public bool AllowNotifications => Status != StackStatus.Fixed && Status != StackStatus.Ignored && Status != StackStatus.Discarded && Status != StackStatus.Snoozed; public static class KnownTypes diff --git a/src/Exceptionless.Core/Pipeline/010_AssignToStackAction.cs b/src/Exceptionless.Core/Pipeline/010_AssignToStackAction.cs index c0d067e00f..6a55257bbc 100644 --- a/src/Exceptionless.Core/Pipeline/010_AssignToStackAction.cs +++ b/src/Exceptionless.Core/Pipeline/010_AssignToStackAction.cs @@ -111,7 +111,7 @@ public override async Task ProcessBatchAsync(ICollection contexts) } else { - ctx.Stack = await _stackRepository.GetByIdAsync(ctx.Event.StackId, o => o.Cache()); + ctx.Stack = await _stackRepository.GetCanonicalStackAsync(ctx.Event.StackId); if (ctx.Stack is null || ctx.Stack.ProjectId != ctx.Event.ProjectId) { ctx.SetError("Invalid StackId."); @@ -168,8 +168,8 @@ await _publisher.PublishAsync(new EntityChanged } var stacksToSave = stacks.Where(s => s.Value.ShouldSave).Select(kvp => kvp.Value.Stack).ToList(); - if (stacksToSave.Count > 0) - await _stackRepository.SaveAsync(stacksToSave, o => o.Cache().Notifications(false)); // notification will get sent later in the update stats step + foreach (var stack in stacksToSave) + await _stackRepository.AddEventTagsAsync(stack.Id, stack.Tags); // notification will get sent later in the update stats step // Set stack ids after they have been saved and created contexts.ForEach(ctx => diff --git a/src/Exceptionless.Core/Repositories/Configuration/Indexes/StackIndex.cs b/src/Exceptionless.Core/Repositories/Configuration/Indexes/StackIndex.cs index 02b15bdd89..5d104bf141 100644 --- a/src/Exceptionless.Core/Repositories/Configuration/Indexes/StackIndex.cs +++ b/src/Exceptionless.Core/Repositories/Configuration/Indexes/StackIndex.cs @@ -47,6 +47,8 @@ public override void ConfigureIndexMapping(TypeMappingDescriptor map) .Keyword(e => e.SignatureHash, k => k.IgnoreAbove(1024)) .FieldAlias(Alias.SignatureHash, a => a.Path(f => f.SignatureHash)) .Keyword(e => e.DuplicateSignature) + .Keyword(e => e.RedirectToStackId) + .Boolean(e => e.NeedsRedirectReconciliation) .Keyword(e => e.Type, k => k.IgnoreAbove(1024)) .Date(e => e.FirstOccurrence) .FieldAlias(Alias.FirstOccurrence, a => a.Path(f => f.FirstOccurrence)) diff --git a/src/Exceptionless.Core/Repositories/EventRepository.cs b/src/Exceptionless.Core/Repositories/EventRepository.cs index 3fba37a69d..17a9ed40d4 100644 --- a/src/Exceptionless.Core/Repositories/EventRepository.cs +++ b/src/Exceptionless.Core/Repositories/EventRepository.cs @@ -1,21 +1,30 @@ -using Elastic.Clients.Elasticsearch.QueryDsl; +using System.Linq.Expressions; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.Aggregations; +using Elastic.Clients.Elasticsearch.QueryDsl; +using Elastic.Transport; +using Elastic.Transport.Products.Elasticsearch; using Exceptionless.Core.Models; using Exceptionless.Core.Repositories.Configuration; using Exceptionless.Core.Repositories.Queries; using Exceptionless.Core.Validation; using Exceptionless.DateTimeExtensions; using Foundatio.Repositories; +using Foundatio.Repositories.Elasticsearch.Extensions; +using Foundatio.Repositories.Exceptions; using Foundatio.Repositories.Models; namespace Exceptionless.Core.Repositories; public class EventRepository : RepositoryOwnedByOrganizationAndProject, IEventRepository { + private readonly ExceptionlessElasticConfiguration _configuration; private readonly TimeProvider _timeProvider; public EventRepository(ExceptionlessElasticConfiguration configuration, AppOptions options, MiniValidationValidator validator) : base(configuration.Events, validator, options) { + _configuration = configuration; _timeProvider = configuration.TimeProvider; DisableCache(); // NOTE: If cache is ever enabled, then fast paths for patching/deleting with scripts will be super slow! @@ -74,7 +83,7 @@ public Task RemoveAllAsync(string organizationId, string? clientIpAddress, if (!String.IsNullOrEmpty(clientIpAddress)) query = query.FieldEquals(EventIndex.Alias.IpAddress, clientIpAddress); - return RemoveAllAsync(q => query, options); + return RemoveAllIgnoringMissingEventIndexesAsync(q => query, options); } public Task> GetByReferenceIdAsync(string projectId, string referenceId) @@ -188,12 +197,244 @@ public override Task> GetByProjectIdAsync(string pr return FindAsync(q => q.Project(projectId).SortDescending(e => e.Date).SortDescending(e => e.Id), options); } + public override Task RemoveAllByOrganizationIdAsync(string organizationId) + { + ArgumentException.ThrowIfNullOrEmpty(organizationId); + + return RemoveAllIgnoringMissingEventIndexesAsync(q => q.Organization(organizationId)); + } + + public override Task RemoveAllByProjectIdAsync(string organizationId, string projectId) + { + ArgumentException.ThrowIfNullOrEmpty(organizationId); + ArgumentException.ThrowIfNullOrEmpty(projectId); + + return RemoveAllIgnoringMissingEventIndexesAsync(q => q.Organization(organizationId).Project(projectId)); + } + public Task RemoveAllByStackIdsAsync(string[] stackIds) + => RemoveAllByStackIdsAsync(stackIds, null); + + public Task RemoveAllByStackIdsAsync(string[] stackIds, CommandOptionsDescriptor? options) { ArgumentNullException.ThrowIfNull(stackIds); - if (stackIds.Length == 0) + if (stackIds is []) throw new ArgumentOutOfRangeException(nameof(stackIds)); - return RemoveAllAsync(q => q.Stack(stackIds)); + return RemoveAllIgnoringMissingEventIndexesAsync(q => q.Stack(stackIds), options); + } + + public Task RemoveAllByProjectIdsAsync(string[] projectIds, CommandOptionsDescriptor? options = null) + { + ArgumentNullException.ThrowIfNull(projectIds); + if (projectIds is []) + throw new ArgumentOutOfRangeException(nameof(projectIds)); + + return RemoveAllIgnoringMissingEventIndexesAsync(q => q.Project(projectIds), options); + } + + public Task RemoveAllByOrganizationIdsAsync(string[] organizationIds, CommandOptionsDescriptor? options = null) + { + ArgumentNullException.ThrowIfNull(organizationIds); + if (organizationIds is []) + throw new ArgumentOutOfRangeException(nameof(organizationIds)); + + return RemoveAllIgnoringMissingEventIndexesAsync(q => q.Organization(organizationIds), options); + } + + private async Task RemoveAllIgnoringMissingEventIndexesAsync( + RepositoryQueryDescriptor query, CommandOptionsDescriptor? options = null) + { + try + { + return await RemoveAllAsync(query, options); + } + catch (RepositoryException ex) when (IsIndexNotFound(ex.InnerException as TransportException)) + { + return 0; + } + catch (TransportException ex) when (IsIndexNotFound(ex)) + { + return 0; + } + } + + private static bool IsIndexNotFound(TransportException? ex) + { + if (ex?.ApiCallDetails?.HttpStatusCode != 404) + return false; + + return ex.ApiCallDetails.ProductError is ElasticsearchServerError serverError + ? IsIndexNotFound(serverError) + : ex.DebugInformation.Contains("index_not_found_exception", StringComparison.Ordinal); + } + + private static bool IsIndexNotFound(ElasticsearchServerError serverError) + { + if (serverError.Status != 404 || serverError.Error is null) + return false; + + return String.Equals(serverError.Error.Type, "index_not_found_exception", StringComparison.Ordinal) + || serverError.Error.RootCause?.Any(IsIndexNotFound) == true; + } + + private static bool IsIndexNotFound(Elastic.Transport.Products.Elasticsearch.ErrorCause? cause) + { + return cause is not null + && (String.Equals(cause.Type, "index_not_found_exception", StringComparison.Ordinal) + || cause.CausedBy is not null && IsIndexNotFound(cause.CausedBy)); + } + + /// + /// Reassigns all events from the source stacks to the target stack using a parameterized + /// Painless script (no string interpolation) to prevent script injection. + /// + /// + /// Foundatio's update-by-query task cannot be interrupted safely once submitted. The token is + /// observed before submission and by both strict verification reads; the caller must retain its + /// lease until this method returns so cancellation cannot leave an unobserved background write. + /// + public async Task ReassignStackAsync(IEnumerable sourceStackIds, string targetStackId, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(sourceStackIds); + ArgumentException.ThrowIfNullOrEmpty(targetStackId); + + // Materialize to avoid multiple enumeration and guard against empty; an empty + // .Stack() filter would match ALL events and reassign them to the target stack. + var sourceIds = sourceStackIds.Distinct(StringComparer.Ordinal).ToList(); + if (sourceIds.Count == 0) + return 0; + if (sourceIds.Contains(targetStackId, StringComparer.Ordinal)) + throw new ArgumentException("Source and target stack ids must be different.", nameof(sourceStackIds)); + + const int maxAttempts = 5; + long remaining = await CountEventsByStackIdsStrictAsync(sourceIds, cancellationToken); + long affected = 0; + + for (int attempt = 1; remaining > 0 && attempt <= maxAttempts; attempt++) + { + cancellationToken.ThrowIfCancellationRequested(); + + affected += await PatchAllAsync( + q => q.Stack(sourceIds), + new ScriptPatch("ctx._source.stack_id = params.targetStackId") + { + Params = new Dictionary { ["targetStackId"] = targetStackId } + }, + o => o.ImmediateConsistency().Notifications(false)); + + // ScriptPatch uses update-by-query with Conflicts.Proceed. Verify strictly against + // the same alias and retry any documents skipped by version conflicts. + remaining = await CountEventsByStackIdsStrictAsync(sourceIds, cancellationToken); + } + + if (remaining > 0) + throw new DocumentException($"Unable to reassign {remaining} event(s) after {maxAttempts} attempts."); + + return affected; + } + + private async Task CountEventsByStackIdsStrictAsync(IReadOnlyCollection stackIds, CancellationToken cancellationToken) + { + var count = await _configuration.Client.CountAsync(s => s + .Indices(_configuration.Events.Name) + .AllowNoIndices(true) + .IgnoreUnavailable(false) + .ExpandWildcards(ExpandWildcard.All) + .Query(q => q.Terms(t => t + .Field(e => e.StackId) + .Terms(new TermsQueryField(stackIds.Select(id => (FieldValue)id).ToList())))), cancellationToken); + + // A brand-new installation can have no event alias yet. Fall back to every concrete + // version only for that specific state so an alias transition can never look like zero. + if (!count.IsValidResponse && count.ElasticsearchServerError is not null && IsIndexNotFound(count.ElasticsearchServerError)) + { + count = await _configuration.Client.CountAsync(s => s + .Indices($"{_configuration.Events.Name}-v*-*") + .AllowNoIndices(true) + .IgnoreUnavailable(false) + .ExpandWildcards(ExpandWildcard.All) + .Query(q => q.Terms(t => t + .Field(e => e.StackId) + .Terms(new TermsQueryField(stackIds.Select(id => (FieldValue)id).ToList())))), cancellationToken); + } + + if (!count.IsValidResponse || count.Shards.Failed > 0) + throw new DocumentException($"Unable to verify event reassignment through the event alias: {count.DebugInformation}", count.ApiCallDetails.OriginalException); + + return count.Count; + } + + public Task GetDistinctStackIdsAsync(int batchSize, string? afterValue = null, CancellationToken cancellationToken = default) + { + return GetDistinctFieldValuesAsync("stack_id", e => e.StackId, batchSize, afterValue, cancellationToken); + } + + public Task GetDistinctProjectIdsAsync(int batchSize, string? afterValue = null, CancellationToken cancellationToken = default) + { + return GetDistinctFieldValuesAsync("project_id", e => e.ProjectId, batchSize, afterValue, cancellationToken); + } + + public Task GetDistinctOrganizationIdsAsync(int batchSize, string? afterValue = null, CancellationToken cancellationToken = default) + { + return GetDistinctFieldValuesAsync("organization_id", e => e.OrganizationId, batchSize, afterValue, cancellationToken); + } + + /// + /// Uses a composite aggregation to paginate through all distinct values of a field. + /// Composite aggregations are preferred over terms aggregations for high-cardinality fields + /// because terms aggregations can silently miss values when the unique count exceeds the + /// configured size parameter. Composite aggregations guarantee correct iteration via an + /// after_key cursor, at the cost of requiring sequential page fetches. + /// + private async Task GetDistinctFieldValuesAsync( + string fieldName, + Expression> fieldExpression, + int batchSize, + string? afterValue, + CancellationToken cancellationToken) + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(batchSize); + + string aggregationName = $"composite_{fieldName}"; + var sources = new List> + { + new(fieldName, new CompositeAggregationSource + { + Terms = new CompositeTermsAggregation { Field = fieldExpression } + }) + }; + + var search = await _configuration.Client.SearchAsync(s => + { + s.Indices($"{_configuration.Events.VersionedName}-*") + .Size(0) + .AddAggregation(aggregationName, a => a.Composite(c => + { + c.Size(batchSize) + .Sources(sources); + + if (!String.IsNullOrEmpty(afterValue)) + c.After(new Dictionary { [fieldName] = afterValue }); + })); + }, cancellationToken); + + if (!search.IsValidResponse) + { + if (search.ElasticsearchServerError is not null && IsIndexNotFound(search.ElasticsearchServerError)) + return new DistinctValuePage([], null); + + throw new InvalidOperationException($"Error retrieving distinct event values for '{fieldName}': {search.DebugInformation}", search.ApiCallDetails.OriginalException); + } + + var composite = search.Aggregations?.GetComposite(aggregationName); + var values = composite?.Buckets is { Count: > 0 } + ? composite.Buckets.Select(bucket => bucket.Key[fieldName].ToString()!).ToArray() + : []; + string? nextValue = composite?.AfterKey is not null && composite.AfterKey.TryGetValue(fieldName, out var next) + ? next.ToString() + : null; + + return new DistinctValuePage(values, nextValue); } } diff --git a/src/Exceptionless.Core/Repositories/Interfaces/IEventRepository.cs b/src/Exceptionless.Core/Repositories/Interfaces/IEventRepository.cs index c7c2272cbc..531230f899 100644 --- a/src/Exceptionless.Core/Repositories/Interfaces/IEventRepository.cs +++ b/src/Exceptionless.Core/Repositories/Interfaces/IEventRepository.cs @@ -13,8 +13,17 @@ public interface IEventRepository : IRepositoryOwnedByOrganizationAndProject UpdateSessionStartLastActivityAsync(string id, DateTime lastActivityUtc, bool isSessionEnd = false, bool hasError = false, bool sendNotifications = true); Task RemoveAllAsync(string organizationId, string? clientIpAddress, DateTime? utcStart, DateTime? utcEnd, CommandOptionsDescriptor? options = null); Task RemoveAllByStackIdsAsync(string[] stackIds); + Task RemoveAllByStackIdsAsync(string[] stackIds, CommandOptionsDescriptor? options); + Task RemoveAllByProjectIdsAsync(string[] projectIds, CommandOptionsDescriptor? options = null); + Task RemoveAllByOrganizationIdsAsync(string[] organizationIds, CommandOptionsDescriptor? options = null); + Task ReassignStackAsync(IEnumerable sourceStackIds, string targetStackId, CancellationToken cancellationToken = default); + Task GetDistinctStackIdsAsync(int batchSize, string? afterValue = null, CancellationToken cancellationToken = default); + Task GetDistinctProjectIdsAsync(int batchSize, string? afterValue = null, CancellationToken cancellationToken = default); + Task GetDistinctOrganizationIdsAsync(int batchSize, string? afterValue = null, CancellationToken cancellationToken = default); } +public sealed record DistinctValuePage(IReadOnlyCollection Values, string? NextValue); + public static class EventRepositoryExtensions { public static async Task GetPreviousAndNextEventIdsAsync(this IEventRepository repository, string id, AppFilter? systemFilter = null, DateTime? utcStart = null, DateTime? utcEnd = null) diff --git a/src/Exceptionless.Core/Repositories/Interfaces/IStackRepository.cs b/src/Exceptionless.Core/Repositories/Interfaces/IStackRepository.cs index 13199d28c6..7a508fb020 100644 --- a/src/Exceptionless.Core/Repositories/Interfaces/IStackRepository.cs +++ b/src/Exceptionless.Core/Repositories/Interfaces/IStackRepository.cs @@ -7,6 +7,7 @@ namespace Exceptionless.Core.Repositories; public interface IStackRepository : IRepositoryOwnedByOrganizationAndProject { Task GetStackBySignatureHashAsync(string projectId, string signatureHash); + Task GetCanonicalStackAsync(string stackId); Task> GetIdsByQueryAsync(RepositoryQueryDescriptor query, CommandOptionsDescriptor? options = null); Task> GetExpiredSnoozedStatuses(DateTime utcNow, CommandOptionsDescriptor? options = null); Task MarkAsRegressedAsync(string stackId); @@ -14,5 +15,12 @@ public interface IStackRepository : IRepositoryOwnedByOrganizationAndProject SetEventCounterAsync(string stackId, DateTime firstOccurrenceUtc, DateTime lastOccurrenceUtc, long totalOccurrences, bool sendNotifications = true); Task> GetStacksForCleanupAsync(string organizationId, DateTime cutoff); Task> GetSoftDeleted(); + Task> GetRedirectedStacksNeedingReconciliationAsync(); Task SoftDeleteByProjectIdAsync(string organizationId, string projectId); + Task> GetDuplicateSignaturesAsync(int maxResults = 10000); + Task AddEventTagsAsync(string stackId, IEnumerable tags); + Task MarkOpenAsync(IEnumerable stackIds); + Task SetDuplicateStackRedirectAsync(Stack sourceStack, string targetStackId, bool isDeleted = false); + Task MarkDuplicateStackReconciledAsync(Stack sourceStack); + Task MergeDuplicateStackAsync(string targetStackId, Stack sourceStack); } diff --git a/src/Exceptionless.Core/Repositories/StackRepository.cs b/src/Exceptionless.Core/Repositories/StackRepository.cs index 29b0c3407e..54ed08f8be 100644 --- a/src/Exceptionless.Core/Repositories/StackRepository.cs +++ b/src/Exceptionless.Core/Repositories/StackRepository.cs @@ -41,11 +41,39 @@ public Task> GetStacksForCleanupAsync(string organizationId, public Task> GetSoftDeleted() { return FindAsync( - q => q.Include(f => f.Id, f => f.OrganizationId, f => f.ProjectId, f => f.SignatureHash), + q => q + .FieldEmpty(f => f.RedirectToStackId) + .Include(f => f.Id, f => f.OrganizationId, f => f.ProjectId, f => f.SignatureHash), o => o.SoftDeleteMode(SoftDeleteQueryMode.DeletedOnly).SearchAfterPaging().PageLimit(500) ); } + public Task> GetRedirectedStacksNeedingReconciliationAsync() + { + return FindAsync( + q => q + .FieldHasValue(f => f.RedirectToStackId) + .FieldEquals(f => f.NeedsRedirectReconciliation, true), + o => o.SoftDeleteMode(SoftDeleteQueryMode.All).SearchAfterPaging().PageLimit(500)); + } + + public override Task RemoveAllByOrganizationIdAsync(string organizationId) + { + ArgumentException.ThrowIfNullOrEmpty(organizationId); + return RemoveAllAsync( + q => q.Organization(organizationId), + o => o.SoftDeleteMode(SoftDeleteQueryMode.All)); + } + + public override Task RemoveAllByProjectIdAsync(string organizationId, string projectId) + { + ArgumentException.ThrowIfNullOrEmpty(organizationId); + ArgumentException.ThrowIfNullOrEmpty(projectId); + return RemoveAllAsync( + q => q.Organization(organizationId).Project(projectId), + o => o.SoftDeleteMode(SoftDeleteQueryMode.All)); + } + public async Task IncrementEventCounterAsync(string organizationId, string projectId, string stackId, DateTime minOccurrenceDateUtc, DateTime maxOccurrenceDateUtc, int count, bool sendNotifications = true) { // If total occurrences are zero (stack data was reset), then set first occurrence date @@ -72,7 +100,10 @@ Instant parseDate(def dt) { ctx._source.updated_utc = params.updatedUtc; } -ctx._source.total_occurrences += params.count;"; +ctx._source.total_occurrences += params.count; +if (ctx._source.redirect_to_stack_id != null) { + ctx._source.needs_redirect_reconciliation = true; +}"; var operation = new ScriptPatch(script.TrimScript()) { @@ -128,6 +159,10 @@ Instant parseDate(def dt) { if (parseDate(ctx._source.updated_utc).isBefore(parseDate(params.updatedUtc))) { ctx._source.updated_utc = params.updatedUtc; +} + +if (ctx._source.redirect_to_stack_id != null) { + ctx._source.needs_redirect_reconciliation = true; }"; var operation = new ScriptPatch(script.TrimScript()) @@ -155,7 +190,15 @@ Instant parseDate(def dt) { { string key = GetStackSignatureCacheKey(projectId, signatureHash); var hit = await FindOneAsync(q => q.Project(projectId).FieldEquals(s => s.SignatureHash, signatureHash), o => o.Cache(key)); - return hit?.Document; + return hit?.Document is null ? null : await ResolveCanonicalStackAsync(hit.Document); + } + + public async Task GetCanonicalStackAsync(string stackId) + { + ArgumentException.ThrowIfNullOrEmpty(stackId); + + var stack = await GetByIdAsync(stackId, o => o.Cache().SoftDeleteMode(SoftDeleteQueryMode.All)); + return stack is null ? null : await ResolveCanonicalStackAsync(stack); } public Task> GetIdsByQueryAsync(RepositoryQueryDescriptor query, CommandOptionsDescriptor? options = null) @@ -165,15 +208,24 @@ public Task> GetIdsByQueryAsync(RepositoryQueryDescriptor(stack => + { + if (stack.Status == StackStatus.Regressed) + return false; + + stack.Status = StackStatus.Regressed; + return true; + }), + o => o.Retry(10)); + } + catch (DocumentNotFoundException) { _logger.LogWarning("Stack {StackId} not found when marking as regressed", stackId); - return; } - - stack.Status = StackStatus.Regressed; - await SaveAsync(stack, o => o.Cache()); } public Task SoftDeleteByProjectIdAsync(string organizationId, string projectId) @@ -187,6 +239,309 @@ public Task SoftDeleteByProjectIdAsync(string organizationId, string proje ); } + public async Task> GetDuplicateSignaturesAsync(int maxResults = 10000) + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(maxResults); + + // ImmediateConsistency forces a segment refresh before the aggregation so that + // any stacks soft-deleted in a previous batch are excluded here. Cost: one refresh + // per batch (not per item), equivalent to the original explicit index refresh. + var result = await CountAsync( + q => q.AggregationsExpression($"terms:(duplicate_signature~{maxResults} @min:2)"), + o => o.ImmediateConsistency()); + + var buckets = result.Aggregations.Terms("terms_duplicate_signature")?.Buckets; + if (buckets is not { Count: > 0 }) + return []; + + return buckets.Select(b => b.Key).ToArray(); + } + + public async Task AddEventTagsAsync(string stackId, IEnumerable tags) + { + ArgumentException.ThrowIfNullOrEmpty(stackId); + ArgumentNullException.ThrowIfNull(tags); + var tagsToAdd = tags.ToArray(); + + string? redirectToStackId = null; + bool modified = await PatchAsync( + stackId, + new ActionPatch(stack => + { + if (!String.IsNullOrEmpty(stack.RedirectToStackId)) + { + redirectToStackId = stack.RedirectToStackId; + return false; + } + + stack.Tags ??= new TagSet(); + var originalTags = new TagSet(stack.Tags); + stack.Tags.UnionWith(tagsToAdd); + stack.Tags.RemoveExcessTags(); + return !stack.Tags.SetEquals(originalTags); + }), + o => o.Notifications(false).Retry(10)); + + if (String.IsNullOrEmpty(redirectToStackId)) + return modified; + + var canonicalStack = await GetCanonicalStackAsync(redirectToStackId); + return canonicalStack is not null && await AddEventTagsAsync(canonicalStack.Id, tagsToAdd); + } + + public Task MarkOpenAsync(IEnumerable stackIds) + { + ArgumentNullException.ThrowIfNull(stackIds); + var ids = new Ids(stackIds.Distinct(StringComparer.Ordinal)); + if (ids.Count == 0) + return Task.FromResult(0L); + + return PatchAsync( + ids, + new ActionPatch(stack => + { + if (stack is { Status: StackStatus.Open, DateFixed: null, FixedInVersion: null, SnoozeUntilUtc: null }) + return false; + + stack.MarkOpen(); + return true; + }), + o => o.Retry(10)); + } + + public async Task SetDuplicateStackRedirectAsync(Stack sourceStack, string targetStackId, bool isDeleted = false) + { + ArgumentNullException.ThrowIfNull(sourceStack); + ArgumentException.ThrowIfNullOrEmpty(sourceStack.Id); + ArgumentException.ThrowIfNullOrEmpty(targetStackId); + if (String.Equals(sourceStack.Id, targetStackId, StringComparison.Ordinal)) + throw new ArgumentException("Source and target stack ids must be different.", nameof(targetStackId)); + + var canonicalTarget = await GetCanonicalStackAsync(targetStackId) + ?? throw new DocumentNotFoundException(targetStackId); + if (String.Equals(sourceStack.Id, canonicalTarget.Id, StringComparison.Ordinal)) + throw new ArgumentException("A stack redirect cannot create a cycle.", nameof(targetStackId)); + + targetStackId = canonicalTarget.Id; + + if (isDeleted) + { + const string finalizeScript = @" +ctx._source.redirect_to_stack_id = params.targetStackId; +ctx._source.is_deleted = true; +ctx._source.needs_redirect_reconciliation = true; +ctx._source.title = ''; +ctx._source.remove('description'); +ctx._source.signature_info = new HashMap(); +ctx._source.tags = new ArrayList(); +ctx._source.references = new ArrayList(); +ctx._source.remove('fixed_in_version');"; + + await PatchAsync( + sourceStack.Id, + new ScriptPatch(finalizeScript.TrimScript()) + { + Params = new Dictionary { ["targetStackId"] = targetStackId } + }, + o => o.Notifications(false).Retry(10)); + } + else + { + await PatchAsync( + sourceStack.Id, + new PartialPatch(new { redirect_to_stack_id = targetStackId, is_deleted = false, needs_redirect_reconciliation = true }), + o => o.ImmediateConsistency().Notifications(false).Retry(10)); + } + + await Cache.RemoveAsync(GetStackSignatureCacheKey(sourceStack)); + } + + public Task MarkDuplicateStackReconciledAsync(Stack sourceStack) + { + ArgumentNullException.ThrowIfNull(sourceStack); + ArgumentException.ThrowIfNullOrEmpty(sourceStack.Id); + ArgumentException.ThrowIfNullOrEmpty(sourceStack.RedirectToStackId); + + const string script = @" +Instant parseDate(def dt) { + if (dt != null) { + try { + return Instant.parse(dt); + } catch(DateTimeParseException e) {} + } + return Instant.MIN; +} + +if (ctx._source.needs_redirect_reconciliation == true + && ctx._source.total_occurrences == params.expectedTotalOccurrences + && parseDate(ctx._source.updated_utc).equals(parseDate(params.expectedUpdatedUtc)) + && ctx._source.redirect_to_stack_id == params.expectedTargetStackId) { + ctx._source.needs_redirect_reconciliation = false; +} else { + ctx.op = 'noop'; +}"; + + return PatchAsync( + sourceStack.Id, + new ScriptPatch(script.TrimScript()) + { + Params = new Dictionary + { + ["expectedTotalOccurrences"] = sourceStack.TotalOccurrences, + ["expectedUpdatedUtc"] = sourceStack.UpdatedUtc, + ["expectedTargetStackId"] = sourceStack.RedirectToStackId + } + }, + o => o.Notifications(false).Retry(10)); + } + + public async Task MergeDuplicateStackAsync(string targetStackId, Stack sourceStack) + { + ArgumentException.ThrowIfNullOrEmpty(targetStackId); + ArgumentNullException.ThrowIfNull(sourceStack); + ArgumentException.ThrowIfNullOrEmpty(sourceStack.Id); + if (String.Equals(targetStackId, sourceStack.Id, StringComparison.Ordinal)) + throw new ArgumentException("Source and target stack ids must be different.", nameof(sourceStack)); + + long nestedOccurrenceTotal = sourceStack.MergedDuplicateStackTotals.Values.Sum(total => (long)total); + int sourceOccurrenceTotal = (int)Math.Clamp(sourceStack.TotalOccurrences - nestedOccurrenceTotal, 0, Int32.MaxValue); + var sourceContributions = new Dictionary(sourceStack.MergedDuplicateStackTotals, StringComparer.Ordinal) + { + [sourceStack.Id] = sourceOccurrenceTotal + }; + + // Track every transitive source contribution independently. This makes retries idempotent, + // allows late occurrence deltas, and prevents redirect chains from double-counting nested + // stacks that were already included in the source total. + const string script = @" +Instant parseDate(def dt) { + if (dt != null) { + try { + return Instant.parse(dt); + } catch(DateTimeParseException e) {} + } + return Instant.MIN; +} + +if (ctx._source.merged_duplicate_stack_totals == null) { + ctx._source.merged_duplicate_stack_totals = new HashMap(); +} + +def occurrenceDelta = 0; +for (def contribution : params.sourceContributions.entrySet()) { + def previousContribution = ctx._source.merged_duplicate_stack_totals.containsKey(contribution.getKey()) + ? ctx._source.merged_duplicate_stack_totals[contribution.getKey()] + : 0; + if (contribution.getValue() > previousContribution) { + occurrenceDelta += contribution.getValue() - previousContribution; + } +} + +if (occurrenceDelta <= 0 + && !parseDate(ctx._source.created_utc).isAfter(parseDate(params.createdUtc)) + && !parseDate(ctx._source.last_occurrence).isBefore(parseDate(params.lastOccurrence)) + && !parseDate(ctx._source.snooze_until_utc).isBefore(parseDate(params.snoozeUntilUtc)) + && !parseDate(ctx._source.date_fixed).isBefore(parseDate(params.dateFixed)) + && !(ctx._source.status == 'open' && params.status != 'open') + && (params.tags == null || ctx._source.tags != null && ctx._source.tags.containsAll(params.tags)) + && (params.references == null || ctx._source.references != null && ctx._source.references.containsAll(params.references)) + && (ctx._source.occurrences_are_critical == true || params.occurrencesAreCritical == false)) { + ctx.op = 'noop'; +} else { + def safeOccurrenceDelta = occurrenceDelta > 0 ? occurrenceDelta : 0; + for (def contribution : params.sourceContributions.entrySet()) { + def previousContribution = ctx._source.merged_duplicate_stack_totals.containsKey(contribution.getKey()) + ? ctx._source.merged_duplicate_stack_totals[contribution.getKey()] + : 0; + if (contribution.getValue() > previousContribution) { + ctx._source.merged_duplicate_stack_totals[contribution.getKey()] = contribution.getValue(); + } + } + def currentTotalOccurrences = ctx._source.total_occurrences == null ? 0 : ctx._source.total_occurrences; + ctx._source.total_occurrences = currentTotalOccurrences + safeOccurrenceDelta; + + if (parseDate(ctx._source.created_utc).isAfter(parseDate(params.createdUtc))) { + ctx._source.created_utc = params.createdUtc; + } + if (parseDate(ctx._source.last_occurrence).isBefore(parseDate(params.lastOccurrence))) { + ctx._source.last_occurrence = params.lastOccurrence; + } + if (parseDate(ctx._source.snooze_until_utc).isBefore(parseDate(params.snoozeUntilUtc))) { + ctx._source.snooze_until_utc = params.snoozeUntilUtc; + } + if (parseDate(ctx._source.date_fixed).isBefore(parseDate(params.dateFixed))) { + ctx._source.date_fixed = params.dateFixed; + } + if (ctx._source.status == 'open' && params.status != 'open') { + ctx._source.status = params.status; + } + + if (ctx._source.tags == null) { + ctx._source.tags = new ArrayList(); + } + for (int i = 0; i < params.tags.size(); i++) { + if (!ctx._source.tags.contains(params.tags[i])) { + ctx._source.tags.add(params.tags[i]); + } + } + + if (ctx._source.references == null) { + ctx._source.references = new ArrayList(); + } + for (int i = 0; i < params.references.size(); i++) { + if (!ctx._source.references.contains(params.references[i])) { + ctx._source.references.add(params.references[i]); + } + } + + ctx._source.occurrences_are_critical = ctx._source.occurrences_are_critical == true || params.occurrencesAreCritical; +}"; + + var operation = new ScriptPatch(script.TrimScript()) + { + Params = new Dictionary + { + ["sourceContributions"] = sourceContributions, + ["createdUtc"] = sourceStack.CreatedUtc, + ["lastOccurrence"] = sourceStack.LastOccurrence, + ["snoozeUntilUtc"] = sourceStack.SnoozeUntilUtc ?? DateTime.MinValue, + ["dateFixed"] = sourceStack.DateFixed ?? DateTime.MinValue, + ["status"] = sourceStack.Status.ToString().ToLowerInvariant(), + ["tags"] = sourceStack.Tags.ToArray(), + ["references"] = sourceStack.References.ToArray(), + ["occurrencesAreCritical"] = sourceStack.OccurrencesAreCritical + } + }; + + bool modified = await PatchAsync(targetStackId, operation, o => o.Notifications(false).Retry(10)); + if (!modified) + return false; + + var targetStack = await GetByIdAsync(targetStackId, o => o.SoftDeleteMode(SoftDeleteQueryMode.All)); + if (targetStack is not null) + await Cache.RemoveAsync(GetStackSignatureCacheKey(targetStack)); + + return true; + } + + private async Task ResolveCanonicalStackAsync(Stack stack) + { + var visitedStackIds = new HashSet(StringComparer.Ordinal) { stack.Id }; + + while (!String.IsNullOrEmpty(stack.RedirectToStackId)) + { + if (!visitedStackIds.Add(stack.RedirectToStackId)) + throw new DocumentException($"Circular stack redirect detected for stack {stack.Id}."); + + stack = await GetByIdAsync( + stack.RedirectToStackId, + o => o.Cache().SoftDeleteMode(SoftDeleteQueryMode.All)) + ?? throw new DocumentNotFoundException(stack.RedirectToStackId); + } + + return stack.IsDeleted ? null : stack; + } + protected override async Task AddDocumentsToCacheAsync(ICollection> findHits, ICommandOptions options, bool isDirtyRead) { await base.AddDocumentsToCacheAsync(findHits, options, isDirtyRead); diff --git a/src/Exceptionless.Core/Serialization/JsonIgnoreForExternalSerializationAttribute.cs b/src/Exceptionless.Core/Serialization/JsonIgnoreForExternalSerializationAttribute.cs new file mode 100644 index 0000000000..5dd8f030a2 --- /dev/null +++ b/src/Exceptionless.Core/Serialization/JsonIgnoreForExternalSerializationAttribute.cs @@ -0,0 +1,7 @@ +namespace Exceptionless.Core.Serialization; + +/// +/// Excludes an internal persistence property from HTTP API serialization. +/// +[AttributeUsage(AttributeTargets.Property)] +internal sealed class JsonIgnoreForExternalSerializationAttribute : Attribute; diff --git a/src/Exceptionless.Core/Serialization/JsonSerializerOptionsExtensions.cs b/src/Exceptionless.Core/Serialization/JsonSerializerOptionsExtensions.cs index cd27fb785e..872a1afb47 100644 --- a/src/Exceptionless.Core/Serialization/JsonSerializerOptionsExtensions.cs +++ b/src/Exceptionless.Core/Serialization/JsonSerializerOptionsExtensions.cs @@ -35,6 +35,10 @@ public static JsonSerializerOptions ConfigureExceptionlessDefaults(this JsonSeri public static JsonSerializerOptions ConfigureExceptionlessApiDefaults(this JsonSerializerOptions options) { ConfigureExceptionlessDefaults(options, skipEmptyCollections: false); + + if (options.TypeInfoResolver is DefaultJsonTypeInfoResolver resolver) + resolver.Modifiers.Add(RemoveInternalProperties); + return options; } @@ -73,4 +77,13 @@ private static JsonSerializerOptions ConfigureExceptionlessDefaults(JsonSerializ options.TypeInfoResolver = resolver; return options; } + + private static void RemoveInternalProperties(JsonTypeInfo typeInfo) + { + for (int i = typeInfo.Properties.Count - 1; i >= 0; i--) + { + if (typeInfo.Properties[i].AttributeProvider?.IsDefined(typeof(JsonIgnoreForExternalSerializationAttribute), inherit: true) == true) + typeInfo.Properties.RemoveAt(i); + } + } } diff --git a/tests/Exceptionless.Tests/Jobs/CleanupDataJobPaginationTests.cs b/tests/Exceptionless.Tests/Jobs/CleanupDataJobPaginationTests.cs new file mode 100644 index 0000000000..f0732d33f4 --- /dev/null +++ b/tests/Exceptionless.Tests/Jobs/CleanupDataJobPaginationTests.cs @@ -0,0 +1,160 @@ +using Exceptionless.Core.Models; +using Exceptionless.DateTimeExtensions; +using Exceptionless.Tests.Utility; +using Foundatio.Repositories; +using Foundatio.Repositories.Utility; +using Xunit; + +namespace Exceptionless.Tests.Jobs; + +public partial class CleanupDataJobTests +{ + [Fact] + public async Task CleanupSoftDeletedOrganizations_WithMultiplePages_RemovesAllData() + { + // Arrange + var organizations = new List(); + for (int i = 0; i < 12; i++) + { + var organization = _organizationData.GenerateOrganization(_billingManager, _plans, generateId: true); + organization.IsDeleted = true; + organizations.Add(organization); + } + + await _organizationRepository.AddAsync(organizations, o => o.ImmediateConsistency()); + + var project = await _projectRepository.AddAsync( + _projectData.GenerateProject(generateId: true, organizationId: organizations[0].Id), + o => o.ImmediateConsistency()); + var stack = await _stackRepository.AddAsync( + _stackData.GenerateStack(generateId: true, organizationId: organizations[0].Id, projectId: project.Id), + o => o.ImmediateConsistency()); + await _eventRepository.AddAsync( + _eventData.GenerateEvents(10, organizations[0].Id, project.Id, stack.Id), + o => o.ImmediateConsistency()); + + // Act + await _job.RunAsync(TestCancellationToken); + + // Assert + foreach (var organization in organizations) + Assert.Null(await _organizationRepository.GetByIdAsync(organization.Id, o => o.IncludeSoftDeletes())); + + Assert.Null(await _projectRepository.GetByIdAsync(project.Id, o => o.IncludeSoftDeletes())); + Assert.Null(await _stackRepository.GetByIdAsync(stack.Id, o => o.IncludeSoftDeletes())); + Assert.Equal(0, await _eventRepository.CountAsync(o => o.IncludeSoftDeletes())); + } + + [Fact] + public async Task EnforceRetention_WithMultipleOrganizations_RespectsPerOrganizationRetention() + { + // Arrange + // Retention enforcement uses the next plan above the organization's retention: + // FreePlan (3d) becomes 30d and SmallPlan (30d) becomes 90d. + var organization1 = _organizationData.GenerateOrganization(_billingManager, _plans, generateId: true); + _billingManager.ApplyBillingPlan(organization1, _plans.SmallPlan); + organization1.StripeCustomerId = "cust_test1"; + organization1.CardLast4 = "4242"; + organization1.SubscribeDate = DateTime.UtcNow; + organization1.BillingChangedByUserId = TestConstants.UserId; + + var organization2 = _organizationData.GenerateOrganization(_billingManager, _plans, generateId: true); + _billingManager.ApplyBillingPlan(organization2, _plans.FreePlan); + await _organizationRepository.AddAsync([organization1, organization2], o => o.ImmediateConsistency()); + + var project1 = _projectData.GenerateProject(generateId: true, organizationId: organization1.Id); + var project2 = _projectData.GenerateProject(generateId: true, organizationId: organization2.Id); + await _projectRepository.AddAsync([project1, project2], o => o.ImmediateConsistency()); + + var stack1 = _stackData.GenerateStack(generateId: true, organizationId: organization1.Id, projectId: project1.Id); + var stack2 = _stackData.GenerateStack(generateId: true, organizationId: organization2.Id, projectId: project2.Id); + await _stackRepository.AddAsync([stack1, stack2], o => o.ImmediateConsistency()); + + var recentStart = DateTimeOffset.UtcNow.AddDays(-2); + var recentEnd = DateTimeOffset.UtcNow.AddDays(-1); + await _eventRepository.AddAsync(_eventData.GenerateEvents(10, organization1.Id, project1.Id, stack1.Id, startDate: recentStart, endDate: recentEnd), o => o.ImmediateConsistency()); + await _eventRepository.AddAsync(_eventData.GenerateEvents(10, organization2.Id, project2.Id, stack2.Id, startDate: recentStart, endDate: recentEnd), o => o.ImmediateConsistency()); + + var olderStart = DateTimeOffset.UtcNow.AddDays(-37); + var olderEnd = DateTimeOffset.UtcNow.AddDays(-33); + await _eventRepository.AddAsync(_eventData.GenerateEvents(10, organization1.Id, project1.Id, stack1.Id, startDate: olderStart, endDate: olderEnd), o => o.ImmediateConsistency()); + await _eventRepository.AddAsync(_eventData.GenerateEvents(10, organization2.Id, project2.Id, stack2.Id, startDate: olderStart, endDate: olderEnd), o => o.ImmediateConsistency()); + + // Act + await _job.RunAsync(TestCancellationToken); + await RefreshDataAsync(); + + // Assert + Assert.Equal(20, await _eventRepository.CountAsync(q => q.FilterExpression($"organization:{organization1.Id}"))); + Assert.Equal(10, await _eventRepository.CountAsync(q => q.FilterExpression($"organization:{organization2.Id}"))); + } + + [Fact] + public async Task CleanupSoftDeletedStacks_WithMultiplePages_RemovesAllStacks() + { + // Arrange + var organization = await _organizationRepository.AddAsync( + _organizationData.GenerateOrganization(_billingManager, _plans, generateId: true), + o => o.ImmediateConsistency()); + var project = await _projectRepository.AddAsync( + _projectData.GenerateProject(generateId: true, organizationId: organization.Id), + o => o.ImmediateConsistency()); + + var stacks = new List(); + for (int i = 0; i < 600; i++) + { + var stack = _stackData.GenerateStack(generateId: true, organizationId: organization.Id, projectId: project.Id); + stack.IsDeleted = true; + stacks.Add(stack); + } + + await _stackRepository.AddAsync(stacks, o => o.ImmediateConsistency()); + + var validStack = await _stackRepository.AddAsync( + _stackData.GenerateStack(generateId: true, organizationId: organization.Id, projectId: project.Id), + o => o.ImmediateConsistency()); + await _eventRepository.AddAsync( + _eventData.GenerateEvents(5, organization.Id, project.Id, validStack.Id), + o => o.ImmediateConsistency()); + + // Act + await _job.RunAsync(TestCancellationToken); + await RefreshDataAsync(); + + // Assert + Assert.Equal(1, await _stackRepository.CountAsync(o => o.IncludeSoftDeletes())); + Assert.Equal(5, await _eventRepository.CountAsync(o => o.IncludeSoftDeletes())); + } + + [Fact] + public async Task EnforceRetention_WithEventsOutsideRetention_DeletesOnlyExpiredEvents() + { + // Arrange + // FreePlan's 3-day retention is enforced at the next plan threshold (30 days). + var organization = _organizationData.GenerateOrganization(_billingManager, _plans, generateId: true); + _billingManager.ApplyBillingPlan(organization, _plans.FreePlan); + await _organizationRepository.AddAsync(organization, o => o.ImmediateConsistency()); + + var project = await _projectRepository.AddAsync( + _projectData.GenerateProject(generateId: true, organizationId: organization.Id), + o => o.ImmediateConsistency()); + var stack = await _stackRepository.AddAsync( + _stackData.GenerateStack(generateId: true, organizationId: organization.Id, projectId: project.Id), + o => o.ImmediateConsistency()); + + var outsideRetentionStart = DateTimeOffset.UtcNow.SubtractDays(37); + var outsideRetentionEnd = DateTimeOffset.UtcNow.SubtractDays(33); + await _eventRepository.AddAsync(_eventData.GenerateEvents(10, organization.Id, project.Id, stack.Id, startDate: outsideRetentionStart, endDate: outsideRetentionEnd), o => o.ImmediateConsistency()); + + var insideRetentionStart = DateTimeOffset.UtcNow.SubtractDays(2); + var insideRetentionEnd = DateTimeOffset.UtcNow.SubtractDays(1); + await _eventRepository.AddAsync(_eventData.GenerateEvents(10, organization.Id, project.Id, stack.Id, startDate: insideRetentionStart, endDate: insideRetentionEnd), o => o.ImmediateConsistency()); + + // Act + await _job.RunAsync(TestCancellationToken); + await RefreshDataAsync(); + + // Assert + Assert.Equal(10, await _eventRepository.CountAsync(o => o.IncludeSoftDeletes())); + } +} diff --git a/tests/Exceptionless.Tests/Jobs/CleanupDataJobTests.cs b/tests/Exceptionless.Tests/Jobs/CleanupDataJobTests.cs index c7300641a2..786391f8bd 100644 --- a/tests/Exceptionless.Tests/Jobs/CleanupDataJobTests.cs +++ b/tests/Exceptionless.Tests/Jobs/CleanupDataJobTests.cs @@ -1,8 +1,8 @@ using Exceptionless.Core; using Exceptionless.Core.Authorization; using Exceptionless.Core.Billing; -using Exceptionless.Core.Jobs; using Exceptionless.Core.Extensions; +using Exceptionless.Core.Jobs; using Exceptionless.Core.Models; using Exceptionless.Core.Repositories; using Exceptionless.Core.Services; @@ -17,7 +17,7 @@ namespace Exceptionless.Tests.Jobs; -public class CleanupDataJobTests : IntegrationTestsBase +public partial class CleanupDataJobTests : IntegrationTestsBase { private readonly CleanupDataJob _job; private readonly UsageService _usageService; @@ -61,7 +61,7 @@ public CleanupDataJobTests(ITestOutputHelper output, AppWebHostFactory factory) } [Fact] - public async Task CanCleanupSuspendedTokens() + public async Task RunAsync_SuspendedOrganization_SuspendsRelatedTokens() { var organization = _organizationData.GenerateSampleOrganization(_billingManager, _plans); organization.IsSuspended = true; @@ -136,7 +136,7 @@ OAuthToken CreateOAuthToken(DateTime updatedUtc, bool isDisabled, string? refres } [Fact] - public async Task CanCleanupSoftDeletedOrganization() + public async Task RunAsync_SoftDeletedOrganization_RemovesAllRelatedData() { var organization = _organizationData.GenerateSampleOrganization(_billingManager, _plans); organization.IsDeleted = true; @@ -238,7 +238,7 @@ public async Task CleanupSyntheticUsersAsync_FreshMemberAndSimilarUsers_KeepsUse } [Fact] - public async Task CanCleanupSoftDeletedProject() + public async Task RunAsync_SoftDeletedProject_RemovesProjectAndEvents() { var organization = await _organizationRepository.AddAsync(_organizationData.GenerateSampleOrganization(_billingManager, _plans), o => o.ImmediateConsistency()); @@ -258,7 +258,7 @@ public async Task CanCleanupSoftDeletedProject() } [Fact] - public async Task CanCleanupSoftDeletedStack() + public async Task RunAsync_SoftDeletedStack_RemovesStackAndEvents() { var organization = await _organizationRepository.AddAsync(_organizationData.GenerateSampleOrganization(_billingManager, _plans), o => o.ImmediateConsistency()); var project = await _projectRepository.AddAsync(_projectData.GenerateSampleProject(), o => o.ImmediateConsistency()); @@ -278,7 +278,7 @@ public async Task CanCleanupSoftDeletedStack() } [Fact] - public async Task CanCleanupEventsOutsideOfRetentionPeriod() + public async Task RunAsync_EventsOutsideRetentionPeriod_RemovesExpiredEvents() { var organization = _organizationData.GenerateSampleOrganization(_billingManager, _plans); _billingManager.ApplyBillingPlan(organization, _plans.FreePlan); @@ -300,7 +300,7 @@ public async Task CanCleanupEventsOutsideOfRetentionPeriod() } [Fact] - public async Task CanDeleteOrphanedEventsByStack() + public async Task DeleteOrphanedEventsByStack_WithLargeDataset_DeletesAllOrphanedEvents() { var organization = _organizationData.GenerateSampleOrganization(_billingManager, _plans); await _organizationRepository.AddAsync(organization, o => o.ImmediateConsistency()); @@ -898,4 +898,5 @@ private static OAuthToken CreateUserOAuthToken(DateTime utcNow, string userId) UpdatedUtc = utcNow }; } + } diff --git a/tests/Exceptionless.Tests/Jobs/CleanupOrphanedDataJobTests.cs b/tests/Exceptionless.Tests/Jobs/CleanupOrphanedDataJobTests.cs index af3f405b16..70eeccfde7 100644 --- a/tests/Exceptionless.Tests/Jobs/CleanupOrphanedDataJobTests.cs +++ b/tests/Exceptionless.Tests/Jobs/CleanupOrphanedDataJobTests.cs @@ -3,9 +3,11 @@ using Exceptionless.Core.Jobs; using Exceptionless.Core.Models; using Exceptionless.Core.Repositories; +using Exceptionless.Core.Repositories.Configuration; using Exceptionless.Tests.Utility; using Foundatio.Repositories; using Foundatio.Repositories.Utility; +using Foundatio.Utility; using Xunit; namespace Exceptionless.Tests.Jobs; @@ -23,6 +25,7 @@ public class CleanupOrphanedDataJobTests : IntegrationTestsBase private readonly IEventRepository _eventRepository; private readonly BillingManager _billingManager; private readonly BillingPlans _plans; + private readonly ExceptionlessElasticConfiguration _configuration; public CleanupOrphanedDataJobTests(ITestOutputHelper output, AppWebHostFactory factory) : base(output, factory) { @@ -37,12 +40,13 @@ public CleanupOrphanedDataJobTests(ITestOutputHelper output, AppWebHostFactory f _eventRepository = GetService(); _billingManager = GetService(); _plans = GetService(); + _configuration = GetService(); } [Fact] public async Task DeleteOrphanedEventsByStack_WithValidStack_DoesNotDeleteEvents() { - // Arrange - Two tenants, each with valid stacks and events + // Arrange var organization1 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); var organization2 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId2); await _organizationRepository.AddAsync([organization1, organization2], o => o.ImmediateConsistency()); @@ -62,7 +66,7 @@ public async Task DeleteOrphanedEventsByStack_WithValidStack_DoesNotDeleteEvents // Act await _job.RunAsync(TestCancellationToken); - // Assert - All events should remain (no orphans) + // Assert var totalCount = await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency()); Assert.Equal(200, totalCount); } @@ -70,7 +74,7 @@ public async Task DeleteOrphanedEventsByStack_WithValidStack_DoesNotDeleteEvents [Fact] public async Task DeleteOrphanedEventsByStack_WithMixedOrphanedAndValid_OnlyDeletesOrphaned() { - // Arrange - Tenant 1 has valid events; Tenant 2 has orphaned events (stack doesn't exist) + // Arrange var organization1 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); var organization2 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId2); await _organizationRepository.AddAsync([organization1, organization2], o => o.ImmediateConsistency()); @@ -83,15 +87,10 @@ public async Task DeleteOrphanedEventsByStack_WithMixedOrphanedAndValid_OnlyDele var stack2 = _stackData.GenerateStack(generateId: true, organizationId: organization2.Id, projectId: project2.Id); await _stackRepository.AddAsync([stack1, stack2], o => o.ImmediateConsistency()); - // Valid events for both tenants var validEvents1 = _eventData.GenerateEvents(50, organization1.Id, project1.Id, stack1.Id).ToList(); var validEvents2 = _eventData.GenerateEvents(50, organization2.Id, project2.Id, stack2.Id).ToList(); - - // Orphaned events (stack IDs that don't exist) in both tenants - string fakeStackId1 = ObjectId.GenerateNewId().ToString(); - string fakeStackId2 = ObjectId.GenerateNewId().ToString(); - var orphanedEvents1 = _eventData.GenerateEvents(30, organization1.Id, project1.Id, fakeStackId1).ToList(); - var orphanedEvents2 = _eventData.GenerateEvents(20, organization2.Id, project2.Id, fakeStackId2).ToList(); + var orphanedEvents1 = _eventData.GenerateEvents(30, organization1.Id, project1.Id, ObjectId.GenerateNewId().ToString()).ToList(); + var orphanedEvents2 = _eventData.GenerateEvents(20, organization2.Id, project2.Id, ObjectId.GenerateNewId().ToString()).ToList(); await _eventRepository.AddAsync(validEvents1.Concat(validEvents2).Concat(orphanedEvents1).Concat(orphanedEvents2), o => o.ImmediateConsistency()); @@ -101,7 +100,7 @@ public async Task DeleteOrphanedEventsByStack_WithMixedOrphanedAndValid_OnlyDele // Act await _job.RunAsync(TestCancellationToken); - // Assert - Only valid events remain + // Assert var totalAfter = await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency()); Assert.Equal(100, totalAfter); } @@ -109,7 +108,7 @@ public async Task DeleteOrphanedEventsByStack_WithMixedOrphanedAndValid_OnlyDele [Fact] public async Task DeleteOrphanedEventsByStack_LargeVolume_PreservesAllValidEvents() { - // Arrange - Large volume across two tenants: 5000 valid + 10000 orphaned + // Arrange var organization = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); await _organizationRepository.AddAsync(organization, o => o.ImmediateConsistency()); @@ -119,10 +118,8 @@ public async Task DeleteOrphanedEventsByStack_LargeVolume_PreservesAllValidEvent var stack = _stackData.GenerateStack(id: TestConstants.StackId, organizationId: organization.Id, projectId: project.Id); await _stackRepository.AddAsync(stack, o => o.ImmediateConsistency()); - // 5000 valid events for existing stack await _eventRepository.AddAsync(_eventData.GenerateEvents(5000, organization.Id, project.Id, stack.Id), o => o.ImmediateConsistency()); - // 10000 orphaned events with many different fake stack IDs var orphanedEvents = _eventData.GenerateEvents(10000, organization.Id, project.Id).ToList(); orphanedEvents.ForEach(e => e.StackId = ObjectId.GenerateNewId().ToString()); await _eventRepository.AddAsync(orphanedEvents, o => o.ImmediateConsistency()); @@ -133,15 +130,42 @@ public async Task DeleteOrphanedEventsByStack_LargeVolume_PreservesAllValidEvent // Act await _job.RunAsync(TestCancellationToken); - // Assert - Only the 5000 valid events remain + // Assert var totalAfter = await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency()); Assert.Equal(5000, totalAfter); } + [Fact] + public async Task DeleteOrphanedEventsByStack_WithManyUniqueOrphanedStacks_DeletesAllOrphanedEvents() + { + // Arrange + var organization = _organizationData.GenerateSampleOrganization(_billingManager, _plans); + await _organizationRepository.AddAsync(organization, o => o.ImmediateConsistency()); + var project = await _projectRepository.AddAsync(_projectData.GenerateSampleProject(), o => o.ImmediateConsistency()); + + var stack = await _stackRepository.AddAsync(_stackData.GenerateSampleStack(), o => o.ImmediateConsistency()); + await _eventRepository.AddAsync(_eventData.GenerateEvents(50, organization.Id, project.Id, stack.Id), o => o.ImmediateConsistency()); + + var orphanedEvents = _eventData.GenerateEvents(200, organization.Id, project.Id).ToList(); + var uniqueStackIds = Enumerable.Range(0, 200).Select(_ => ObjectId.GenerateNewId().ToString()).ToList(); + for (int i = 0; i < orphanedEvents.Count; i++) + orphanedEvents[i].StackId = uniqueStackIds[i]; + + await _eventRepository.AddAsync(orphanedEvents, o => o.ImmediateConsistency()); + + Assert.Equal(250, await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency())); + + // Act + await _job.RunAsync(TestCancellationToken); + + // Assert + Assert.Equal(50, await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency())); + } + [Fact] public async Task DeleteOrphanedEventsByStack_MultipleValidStacks_PreservesAll() { - // Arrange - Multiple valid stacks in two organizations, no orphans + // Arrange var organization1 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); var organization2 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId2); await _organizationRepository.AddAsync([organization1, organization2], o => o.ImmediateConsistency()); @@ -150,7 +174,6 @@ public async Task DeleteOrphanedEventsByStack_MultipleValidStacks_PreservesAll() var project2 = _projectData.GenerateProject(generateId: true, organizationId: organization2.Id); await _projectRepository.AddAsync([project1, project2], o => o.ImmediateConsistency()); - // Multiple stacks per project var stacks = new List(); for (int i = 0; i < 10; i++) { @@ -159,7 +182,6 @@ public async Task DeleteOrphanedEventsByStack_MultipleValidStacks_PreservesAll() } await _stackRepository.AddAsync(stacks, o => o.ImmediateConsistency()); - // Events across all stacks var events = new List(); foreach (var stack in stacks) events.AddRange(_eventData.GenerateEvents(10, stack.OrganizationId, stack.ProjectId, stack.Id)); @@ -168,7 +190,7 @@ public async Task DeleteOrphanedEventsByStack_MultipleValidStacks_PreservesAll() // Act await _job.RunAsync(TestCancellationToken); - // Assert - All 200 events preserved + // Assert var totalAfter = await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency()); Assert.Equal(200, totalAfter); } @@ -176,7 +198,7 @@ public async Task DeleteOrphanedEventsByStack_MultipleValidStacks_PreservesAll() [Fact] public async Task DeleteOrphanedEventsByStack_OnlyOrphanedEventsInOneTenant_OtherTenantUnaffected() { - // Arrange - Tenant 1 has all orphaned events (will be deleted); Tenant 2 has all valid events + // Arrange var organization1 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); var organization2 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId2); await _organizationRepository.AddAsync([organization1, organization2], o => o.ImmediateConsistency()); @@ -185,15 +207,11 @@ public async Task DeleteOrphanedEventsByStack_OnlyOrphanedEventsInOneTenant_Othe var project2 = _projectData.GenerateProject(generateId: true, organizationId: organization2.Id); await _projectRepository.AddAsync([project1, project2], o => o.ImmediateConsistency()); - // Tenant 2 has a valid stack var validStack = _stackData.GenerateStack(generateId: true, organizationId: organization2.Id, projectId: project2.Id); await _stackRepository.AddAsync(validStack, o => o.ImmediateConsistency()); - // Tenant 1 events are all orphaned (fake stack IDs) var orphanedEvents = _eventData.GenerateEvents(100, organization1.Id, project1.Id).ToList(); orphanedEvents.ForEach(e => e.StackId = ObjectId.GenerateNewId().ToString()); - - // Tenant 2 events are all valid var validEvents = _eventData.GenerateEvents(100, organization2.Id, project2.Id, validStack.Id).ToList(); await _eventRepository.AddAsync(orphanedEvents.Concat(validEvents), o => o.ImmediateConsistency()); @@ -201,11 +219,36 @@ public async Task DeleteOrphanedEventsByStack_OnlyOrphanedEventsInOneTenant_Othe // Act await _job.RunAsync(TestCancellationToken); - // Assert - Only tenant 2's events remain + // Assert var totalAfter = await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency()); Assert.Equal(100, totalAfter); } + [Fact] + public async Task DeleteOrphanedEventsByStack_WithSoftDeletedStack_DeletesOrphanedEvents() + { + // Arrange + var organization = _organizationData.GenerateSampleOrganization(_billingManager, _plans); + await _organizationRepository.AddAsync(organization, o => o.ImmediateConsistency()); + var project = await _projectRepository.AddAsync(_projectData.GenerateSampleProject(), o => o.ImmediateConsistency()); + + var stack = await _stackRepository.AddAsync(_stackData.GenerateSampleStack(), o => o.ImmediateConsistency()); + await _eventRepository.AddAsync(_eventData.GenerateEvents(50, organization.Id, project.Id, stack.Id), o => o.ImmediateConsistency()); + + var softDeletedStack = _stackData.GenerateStack(generateId: true, organizationId: organization.Id, projectId: project.Id); + softDeletedStack.IsDeleted = true; + softDeletedStack = await _stackRepository.AddAsync(softDeletedStack, o => o.ImmediateConsistency()); + await _eventRepository.AddAsync(_eventData.GenerateEvents(100, organization.Id, project.Id, softDeletedStack.Id), o => o.ImmediateConsistency()); + + Assert.Equal(150, await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency())); + + // Act + await _job.RunAsync(TestCancellationToken); + + // Assert + Assert.Equal(50, await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency())); + } + [Fact] public async Task DeleteOrphanedEventsByProject_WithValidProjects_DoesNotDeleteEvents() { @@ -236,7 +279,7 @@ public async Task DeleteOrphanedEventsByProject_WithValidProjects_DoesNotDeleteE [Fact] public async Task DeleteOrphanedEventsByProject_WithOrphanedProject_DeletesEventsForMissingProject() { - // Arrange - Events reference a project that doesn't exist + // Arrange var organization1 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); var organization2 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId2); await _organizationRepository.AddAsync([organization1, organization2], o => o.ImmediateConsistency()); @@ -247,10 +290,7 @@ public async Task DeleteOrphanedEventsByProject_WithOrphanedProject_DeletesEvent var validStack = _stackData.GenerateStack(id: TestConstants.StackId, organizationId: organization1.Id, projectId: validProject.Id); await _stackRepository.AddAsync(validStack, o => o.ImmediateConsistency()); - // Valid events for existing project var validEvents = _eventData.GenerateEvents(75, organization1.Id, validProject.Id, validStack.Id).ToList(); - - // Orphaned events referencing a non-existent project in organization 2 string fakeProjectId = ObjectId.GenerateNewId().ToString(); string fakeStackId = ObjectId.GenerateNewId().ToString(); var orphanedEvents = _eventData.GenerateEvents(50, organization2.Id, fakeProjectId, fakeStackId).ToList(); @@ -260,30 +300,50 @@ public async Task DeleteOrphanedEventsByProject_WithOrphanedProject_DeletesEvent // Act await _job.RunAsync(TestCancellationToken); - // Assert - Orphaned events deleted, valid events preserved + // Assert var totalAfter = await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency()); Assert.Equal(75, totalAfter); } + [Fact] + public async Task DeleteOrphanedEventsByProject_WithMissingProjects_DeletesOrphanedEvents() + { + // Arrange + var organization = _organizationData.GenerateSampleOrganization(_billingManager, _plans); + await _organizationRepository.AddAsync(organization, o => o.ImmediateConsistency()); + var project = await _projectRepository.AddAsync(_projectData.GenerateSampleProject(), o => o.ImmediateConsistency()); + var stack = await _stackRepository.AddAsync(_stackData.GenerateSampleStack(), o => o.ImmediateConsistency()); + await _eventRepository.AddAsync(_eventData.GenerateEvents(50, organization.Id, project.Id, stack.Id), o => o.ImmediateConsistency()); + + string fakeProjectId = ObjectId.GenerateNewId().ToString(); + var orphanedEvents = _eventData.GenerateEvents(100, organization.Id, fakeProjectId).ToList(); + orphanedEvents.ForEach(e => e.StackId = stack.Id); + await _eventRepository.AddAsync(orphanedEvents, o => o.ImmediateConsistency()); + + Assert.Equal(150, await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency())); + + // Act + await _job.RunAsync(TestCancellationToken); + + // Assert + Assert.Equal(50, await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency())); + } + [Fact] public async Task DeleteOrphanedEventsByProject_MultiTenant_EachTenantIndependent() { - // Arrange - Tenant 1 has valid project, Tenant 2 has orphaned project + // Arrange var organization1 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); var organization2 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId2); await _organizationRepository.AddAsync([organization1, organization2], o => o.ImmediateConsistency()); - // Only Tenant 1 has a real project var project1 = _projectData.GenerateProject(id: TestConstants.ProjectId, organizationId: organization1.Id); await _projectRepository.AddAsync(project1, o => o.ImmediateConsistency()); var stack1 = _stackData.GenerateStack(id: TestConstants.StackId, organizationId: organization1.Id, projectId: project1.Id); await _stackRepository.AddAsync(stack1, o => o.ImmediateConsistency()); - // Tenant 1 valid events var validEvents = _eventData.GenerateEvents(60, organization1.Id, project1.Id, stack1.Id).ToList(); - - // Tenant 2 orphaned events (project doesn't exist) string nonExistentProjectId = ObjectId.GenerateNewId().ToString(); string fakeStackId = ObjectId.GenerateNewId().ToString(); var orphanedEvents = _eventData.GenerateEvents(40, organization2.Id, nonExistentProjectId, fakeStackId).ToList(); @@ -298,6 +358,33 @@ public async Task DeleteOrphanedEventsByProject_MultiTenant_EachTenantIndependen Assert.Equal(60, totalAfter); } + [Fact] + public async Task DeleteOrphanedEventsByProject_WithSoftDeletedProject_DeletesOrphanedEvents() + { + // Arrange + var organization = _organizationData.GenerateSampleOrganization(_billingManager, _plans); + await _organizationRepository.AddAsync(organization, o => o.ImmediateConsistency()); + + var project = await _projectRepository.AddAsync(_projectData.GenerateSampleProject(), o => o.ImmediateConsistency()); + var stack = await _stackRepository.AddAsync(_stackData.GenerateSampleStack(), o => o.ImmediateConsistency()); + await _eventRepository.AddAsync(_eventData.GenerateEvents(50, organization.Id, project.Id, stack.Id), o => o.ImmediateConsistency()); + + var softDeletedProject = _projectData.GenerateProject(generateId: true, organizationId: organization.Id); + softDeletedProject.IsDeleted = true; + softDeletedProject = await _projectRepository.AddAsync(softDeletedProject, o => o.ImmediateConsistency()); + var orphanedEvents = _eventData.GenerateEvents(100, organization.Id, softDeletedProject.Id).ToList(); + orphanedEvents.ForEach(e => e.StackId = stack.Id); + await _eventRepository.AddAsync(orphanedEvents, o => o.ImmediateConsistency()); + + Assert.Equal(150, await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency())); + + // Act + await _job.RunAsync(TestCancellationToken); + + // Assert + Assert.Equal(50, await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency())); + } + [Fact] public async Task DeleteOrphanedEventsByOrganization_WithValidOrganizations_DoesNotDeleteEvents() { @@ -321,7 +408,7 @@ public async Task DeleteOrphanedEventsByOrganization_WithValidOrganizations_Does // Act await _job.RunAsync(TestCancellationToken); - // Assert - All events preserved (both organizations exist) + // Assert var totalAfter = await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency()); Assert.Equal(160, totalAfter); } @@ -329,7 +416,7 @@ public async Task DeleteOrphanedEventsByOrganization_WithValidOrganizations_Does [Fact] public async Task DeleteOrphanedEventsByOrganization_WithOrphanedOrganization_DeletesEventsForMissingOrganization() { - // Arrange - Valid organization1 + events referencing non-existent organization + // Arrange var organization1 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); await _organizationRepository.AddAsync(organization1, o => o.ImmediateConsistency()); @@ -339,10 +426,7 @@ public async Task DeleteOrphanedEventsByOrganization_WithOrphanedOrganization_De var stack = _stackData.GenerateStack(id: TestConstants.StackId, organizationId: organization1.Id, projectId: project.Id); await _stackRepository.AddAsync(stack, o => o.ImmediateConsistency()); - // Valid events var validEvents = _eventData.GenerateEvents(100, organization1.Id, project.Id, stack.Id).ToList(); - - // Orphaned events referencing a non-existent organization string fakeOrganizationId = ObjectId.GenerateNewId().ToString(); string fakeProjectId = ObjectId.GenerateNewId().ToString(); string fakeStackId = ObjectId.GenerateNewId().ToString(); @@ -353,15 +437,39 @@ public async Task DeleteOrphanedEventsByOrganization_WithOrphanedOrganization_De // Act await _job.RunAsync(TestCancellationToken); - // Assert - Only valid events survive + // Assert var totalAfter = await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency()); Assert.Equal(100, totalAfter); } + [Fact] + public async Task DeleteOrphanedEventsByOrganization_WithMissingOrganizations_DeletesOrphanedEvents() + { + // Arrange + var organization = _organizationData.GenerateSampleOrganization(_billingManager, _plans); + await _organizationRepository.AddAsync(organization, o => o.ImmediateConsistency()); + var project = await _projectRepository.AddAsync(_projectData.GenerateSampleProject(), o => o.ImmediateConsistency()); + var stack = await _stackRepository.AddAsync(_stackData.GenerateSampleStack(), o => o.ImmediateConsistency()); + await _eventRepository.AddAsync(_eventData.GenerateEvents(50, organization.Id, project.Id, stack.Id), o => o.ImmediateConsistency()); + + string fakeOrganizationId = ObjectId.GenerateNewId().ToString(); + var orphanedEvents = _eventData.GenerateEvents(100, fakeOrganizationId, project.Id).ToList(); + orphanedEvents.ForEach(e => e.StackId = stack.Id); + await _eventRepository.AddAsync(orphanedEvents, o => o.ImmediateConsistency()); + + Assert.Equal(150, await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency())); + + // Act + await _job.RunAsync(TestCancellationToken); + + // Assert + Assert.Equal(50, await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency())); + } + [Fact] public async Task DeleteOrphanedEventsByOrganization_TwoTenantsOneDeleted_OnlyDeletesOrphanedTenantEvents() { - // Arrange - Organization 1 exists, Organization 2 does NOT exist (never created) but has events + // Arrange var organization1 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); await _organizationRepository.AddAsync(organization1, o => o.ImmediateConsistency()); @@ -371,10 +479,7 @@ public async Task DeleteOrphanedEventsByOrganization_TwoTenantsOneDeleted_OnlyDe var stack1 = _stackData.GenerateStack(id: TestConstants.StackId, organizationId: organization1.Id, projectId: project1.Id); await _stackRepository.AddAsync(stack1, o => o.ImmediateConsistency()); - // Tenant 1 valid events var validEvents = _eventData.GenerateEvents(120, organization1.Id, project1.Id, stack1.Id).ToList(); - - // Tenant 2 events (organization doesn't exist, simulates post-hard-delete orphans) string ghostOrganizationId = ObjectId.GenerateNewId().ToString(); string ghostProjectId = ObjectId.GenerateNewId().ToString(); string ghostStackId = ObjectId.GenerateNewId().ToString(); @@ -390,17 +495,43 @@ public async Task DeleteOrphanedEventsByOrganization_TwoTenantsOneDeleted_OnlyDe Assert.Equal(120, totalAfter); } + [Fact] + public async Task DeleteOrphanedEventsByOrganization_WithSoftDeletedOrganization_DeletesOrphanedEvents() + { + // Arrange + var validOrganization = _organizationData.GenerateSampleOrganization(_billingManager, _plans); + await _organizationRepository.AddAsync(validOrganization, o => o.ImmediateConsistency()); + var project = await _projectRepository.AddAsync(_projectData.GenerateSampleProject(), o => o.ImmediateConsistency()); + var stack = await _stackRepository.AddAsync(_stackData.GenerateSampleStack(), o => o.ImmediateConsistency()); + await _eventRepository.AddAsync(_eventData.GenerateEvents(50, validOrganization.Id, project.Id, stack.Id), o => o.ImmediateConsistency()); + + var softDeletedOrganization = _organizationData.GenerateOrganization(_billingManager, _plans, generateId: true); + softDeletedOrganization.IsDeleted = true; + softDeletedOrganization = await _organizationRepository.AddAsync(softDeletedOrganization, o => o.ImmediateConsistency()); + var orphanedEvents = _eventData.GenerateEvents(100, softDeletedOrganization.Id, project.Id).ToList(); + orphanedEvents.ForEach(e => e.StackId = stack.Id); + await _eventRepository.AddAsync(orphanedEvents, o => o.ImmediateConsistency()); + + Assert.Equal(150, await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency())); + + // Act + await _job.RunAsync(TestCancellationToken); + + // Assert + Assert.Equal(50, await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency())); + } + [Fact] public async Task FixDuplicateStacks_WithDuplicatesAcrossTenants_MergesCorrectly() { - // Arrange - Two stacks in the same project with the same signature (duplicate) + // Arrange var organization = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); await _organizationRepository.AddAsync(organization, o => o.ImmediateConsistency()); var project = _projectData.GenerateProject(id: TestConstants.ProjectId, organizationId: organization.Id); await _projectRepository.AddAsync(project, o => o.ImmediateConsistency()); - string signatureHash = "abc123def456"; + const string signatureHash = "abc123def456"; var stack1 = _stackData.GenerateStack(generateId: true, organizationId: organization.Id, projectId: project.Id, signatureHash: signatureHash); stack1.CreatedUtc = DateTime.UtcNow.AddDays(-10); stack1.TotalOccurrences = 5; @@ -409,7 +540,6 @@ public async Task FixDuplicateStacks_WithDuplicatesAcrossTenants_MergesCorrectly stack2.TotalOccurrences = 10; await _stackRepository.AddAsync([stack1, stack2], o => o.ImmediateConsistency()); - // Events on both stacks var events1 = _eventData.GenerateEvents(3, organization.Id, project.Id, stack1.Id).ToList(); var events2 = _eventData.GenerateEvents(7, organization.Id, project.Id, stack2.Id).ToList(); await _eventRepository.AddAsync(events1.Concat(events2), o => o.ImmediateConsistency()); @@ -417,7 +547,7 @@ public async Task FixDuplicateStacks_WithDuplicatesAcrossTenants_MergesCorrectly // Act await _job.RunAsync(TestCancellationToken); - // Assert - One stack should be deleted, all events should point to the surviving stack + // Assert await RefreshDataAsync(); var allStacks = await _stackRepository.GetAllAsync(o => o.IncludeSoftDeletes()); var activeStacks = allStacks.Documents.Where(s => !s.IsDeleted).ToList(); @@ -425,16 +555,176 @@ public async Task FixDuplicateStacks_WithDuplicatesAcrossTenants_MergesCorrectly Assert.Single(activeStacks); Assert.Single(deletedStacks); - // All events should now reference the surviving stack var allEvents = await _eventRepository.GetAllAsync(); Assert.Equal(10, allEvents.Total); Assert.All(allEvents.Documents, e => Assert.Equal(activeStacks[0].Id, e.StackId)); } + [Fact] + public async Task FixDuplicateStacks_WithDuplicateSignatures_MergesIntoMostPopularStack() + { + // Arrange + var organization = _organizationData.GenerateSampleOrganization(_billingManager, _plans); + await _organizationRepository.AddAsync(organization, o => o.ImmediateConsistency()); + var project = await _projectRepository.AddAsync(_projectData.GenerateSampleProject(), o => o.ImmediateConsistency()); + + var originalStack = _stackData.GenerateStack(generateId: true, organizationId: organization.Id, projectId: project.Id); + originalStack.TotalOccurrences = 100; + var duplicateStack = originalStack.DeepClone(); + duplicateStack.Id = ObjectId.GenerateNewId().ToString(); + duplicateStack.TotalOccurrences = 10; + + originalStack = await _stackRepository.AddAsync(originalStack, o => o.ImmediateConsistency()); + duplicateStack = await _stackRepository.AddAsync(duplicateStack, o => o.ImmediateConsistency()); + await _eventRepository.AddAsync(_eventData.GenerateEvents(100, organization.Id, project.Id, originalStack.Id), o => o.ImmediateConsistency()); + await _eventRepository.AddAsync(_eventData.GenerateEvents(10, organization.Id, project.Id, duplicateStack.Id), o => o.ImmediateConsistency()); + + // Act + await _job.RunAsync(TestCancellationToken); + await RefreshDataAsync(); + + // Assert + var updatedOriginal = await _stackRepository.GetByIdAsync(originalStack.Id, o => o.IncludeSoftDeletes()); + var updatedDuplicate = await _stackRepository.GetByIdAsync(duplicateStack.Id, o => o.IncludeSoftDeletes()); + + Assert.NotNull(updatedOriginal); + Assert.NotNull(updatedDuplicate); + Assert.False(updatedOriginal.IsDeleted); + Assert.True(updatedDuplicate.IsDeleted); + Assert.Equal(110, updatedOriginal.TotalOccurrences); + + Assert.Equal(110, await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency())); + Assert.Equal(110, await _eventRepository.CountAsync(q => q.Stack(originalStack.Id))); + Assert.Equal(0, await _eventRepository.CountAsync(q => q.Stack(duplicateStack.Id))); + } + + [Fact] + public async Task FixDuplicateStacks_AfterPartialTargetMerge_DoesNotDoubleApplyMetadata() + { + // Arrange + var organization = _organizationData.GenerateSampleOrganization(_billingManager, _plans); + await _organizationRepository.AddAsync(organization, o => o.ImmediateConsistency()); + var project = await _projectRepository.AddAsync(_projectData.GenerateSampleProject(), o => o.ImmediateConsistency()); + + var targetStack = _stackData.GenerateStack(generateId: true, organizationId: organization.Id, projectId: project.Id); + targetStack.TotalOccurrences = 100; + var sourceStack = targetStack.DeepClone(); + sourceStack.Id = ObjectId.GenerateNewId().ToString(); + sourceStack.TotalOccurrences = 10; + await _stackRepository.AddAsync([targetStack, sourceStack], o => o.ImmediateConsistency()); + await _eventRepository.AddAsync(_eventData.GenerateEvents(100, organization.Id, project.Id, targetStack.Id), o => o.ImmediateConsistency()); + await _eventRepository.AddAsync(_eventData.GenerateEvents(10, organization.Id, project.Id, sourceStack.Id), o => o.ImmediateConsistency()); + + // Simulate a prior run that merged the target but failed before hiding the source. + await _stackRepository.MergeDuplicateStackAsync(targetStack.Id, sourceStack); + + // Act + await _job.RunAsync(TestCancellationToken); + await RefreshDataAsync(); + + // Assert + var updatedTarget = await _stackRepository.GetByIdAsync(targetStack.Id, o => o.IncludeSoftDeletes()); + var updatedSource = await _stackRepository.GetByIdAsync(sourceStack.Id, o => o.IncludeSoftDeletes()); + Assert.NotNull(updatedTarget); + Assert.NotNull(updatedSource); + Assert.Equal(110, updatedTarget.TotalOccurrences); + Assert.False(updatedTarget.IsDeleted); + Assert.True(updatedSource.IsDeleted); + Assert.Equal(110, await _eventRepository.CountAsync(q => q.Stack(targetStack.Id), o => o.ImmediateConsistency())); + Assert.Equal(0, await _eventRepository.CountAsync(q => q.Stack(sourceStack.Id))); + } + + [Fact] + public async Task DeleteOrphanedEventsByStack_WithRedirectedSource_ReassignsLateEvents() + { + var organization = _organizationData.GenerateSampleOrganization(_billingManager, _plans); + await _organizationRepository.AddAsync(organization, o => o.ImmediateConsistency()); + var project = await _projectRepository.AddAsync(_projectData.GenerateSampleProject(), o => o.ImmediateConsistency()); + + var targetStack = _stackData.GenerateStack(generateId: true, organizationId: organization.Id, projectId: project.Id); + targetStack.TotalOccurrences = 100; + var sourceStack = targetStack.DeepClone(); + sourceStack.Id = ObjectId.GenerateNewId().ToString(); + sourceStack.TotalOccurrences = 10; + await _stackRepository.AddAsync([targetStack, sourceStack], o => o.ImmediateConsistency()); + await _stackRepository.SetDuplicateStackRedirectAsync(sourceStack, targetStack.Id, isDeleted: true); + + await _eventRepository.AddAsync( + _eventData.GenerateEvents(10, organization.Id, project.Id, sourceStack.Id), + o => o.ImmediateConsistency()); + + await RefreshDataAsync(); + await _job.RunAsync(TestCancellationToken); + await RefreshDataAsync(); + + Assert.Equal(10, await _eventRepository.CountAsync(q => q.Stack(targetStack.Id), o => o.ImmediateConsistency())); + Assert.Equal(0, await _eventRepository.CountAsync(q => q.Stack(sourceStack.Id), o => o.ImmediateConsistency())); + + var updatedTarget = await _stackRepository.GetByIdAsync(targetStack.Id, o => o.ImmediateConsistency()); + Assert.NotNull(updatedTarget); + Assert.Equal(110, updatedTarget.TotalOccurrences); + + var redirectTombstone = await _stackRepository.GetByIdAsync(sourceStack.Id, o => o.IncludeSoftDeletes()); + Assert.NotNull(redirectTombstone); + Assert.True(redirectTombstone.IsDeleted); + Assert.Equal(targetStack.Id, redirectTombstone.RedirectToStackId); + Assert.False(redirectTombstone.NeedsRedirectReconciliation); + } + + [Fact] + public async Task DeleteOrphanedEventsByStack_WithLateCounterAndNoSourceEvents_ReconcilesTombstone() + { + var organization = _organizationData.GenerateSampleOrganization(_billingManager, _plans); + await _organizationRepository.AddAsync(organization, o => o.ImmediateConsistency()); + var project = await _projectRepository.AddAsync(_projectData.GenerateSampleProject(), o => o.ImmediateConsistency()); + + var targetStack = _stackData.GenerateStack(generateId: true, organizationId: organization.Id, projectId: project.Id); + targetStack.TotalOccurrences = 100; + var sourceStack = targetStack.DeepClone(); + sourceStack.Id = ObjectId.GenerateNewId().ToString(); + sourceStack.TotalOccurrences = 10; + await _stackRepository.AddAsync([targetStack, sourceStack], o => o.ImmediateConsistency()); + await _stackRepository.MergeDuplicateStackAsync(targetStack.Id, sourceStack); + await _stackRepository.SetDuplicateStackRedirectAsync(sourceStack, targetStack.Id, isDeleted: true); + + await _stackRepository.IncrementEventCounterAsync( + sourceStack.OrganizationId, + sourceStack.ProjectId, + sourceStack.Id, + sourceStack.FirstOccurrence, + sourceStack.LastOccurrence.AddMinutes(1), + 5, + sendNotifications: false); + + var incrementedSource = await _stackRepository.GetByIdAsync(sourceStack.Id, o => o.IncludeSoftDeletes().ImmediateConsistency()); + Assert.NotNull(incrementedSource); + Assert.Equal(15, incrementedSource.TotalOccurrences); + + Assert.Equal(0, await _eventRepository.CountAsync(q => q.Stack(sourceStack.Id), o => o.ImmediateConsistency())); + + await RefreshDataAsync(); + await _job.RunAsync(TestCancellationToken); + + var updatedTarget = await _stackRepository.GetByIdAsync(targetStack.Id, o => o.ImmediateConsistency()); + Assert.NotNull(updatedTarget); + Assert.Equal(115, updatedTarget.TotalOccurrences); + + var reconciledSource = await _stackRepository.GetByIdAsync(sourceStack.Id, o => o.IncludeSoftDeletes().ImmediateConsistency()); + Assert.NotNull(reconciledSource); + Assert.False(reconciledSource.NeedsRedirectReconciliation); + Assert.Empty((await _stackRepository.GetRedirectedStacksNeedingReconciliationAsync()).Documents); + + DateTime targetUpdatedUtc = updatedTarget.UpdatedUtc; + await _job.RunAsync(TestCancellationToken); + updatedTarget = await _stackRepository.GetByIdAsync(targetStack.Id, o => o.ImmediateConsistency()); + Assert.NotNull(updatedTarget); + Assert.Equal(targetUpdatedUtc, updatedTarget.UpdatedUtc); + } + [Fact] public async Task FixDuplicateStacks_NoDuplicates_DoesNotModifyAnything() { - // Arrange - Two stacks with different signatures across two tenants + // Arrange var organization1 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); var organization2 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId2); await _organizationRepository.AddAsync([organization1, organization2], o => o.ImmediateConsistency()); @@ -454,7 +744,7 @@ public async Task FixDuplicateStacks_NoDuplicates_DoesNotModifyAnything() // Act await _job.RunAsync(TestCancellationToken); - // Assert - Nothing deleted + // Assert var allStacks = await _stackRepository.GetAllAsync(o => o.IncludeSoftDeletes()); Assert.Equal(2, allStacks.Total); Assert.All(allStacks.Documents, s => Assert.False(s.IsDeleted)); @@ -463,10 +753,83 @@ public async Task FixDuplicateStacks_NoDuplicates_DoesNotModifyAnything() Assert.Equal(40, totalEvents); } + [Fact] + public async Task FixDuplicateStacks_WithNoEvents_KeepsOldestStack() + { + // Arrange + var organization = _organizationData.GenerateSampleOrganization(_billingManager, _plans); + await _organizationRepository.AddAsync(organization, o => o.ImmediateConsistency()); + var project = await _projectRepository.AddAsync(_projectData.GenerateSampleProject(), o => o.ImmediateConsistency()); + + var originalStack = _stackData.GenerateStack(generateId: true, organizationId: organization.Id, projectId: project.Id); + originalStack.CreatedUtc = DateTime.UtcNow.AddMinutes(-10); + var duplicateStack = originalStack.DeepClone(); + duplicateStack.Id = ObjectId.GenerateNewId().ToString(); + duplicateStack.CreatedUtc = originalStack.CreatedUtc.AddMinutes(1); + + originalStack = await _stackRepository.AddAsync(originalStack, o => o.ImmediateConsistency()); + duplicateStack = await _stackRepository.AddAsync(duplicateStack, o => o.ImmediateConsistency()); + + // Act + await _job.RunAsync(TestCancellationToken); + await RefreshDataAsync(); + + // Assert + var updatedOriginal = await _stackRepository.GetByIdAsync(originalStack.Id, o => o.IncludeSoftDeletes()); + var updatedDuplicate = await _stackRepository.GetByIdAsync(duplicateStack.Id, o => o.IncludeSoftDeletes()); + + Assert.NotNull(updatedOriginal); + Assert.NotNull(updatedDuplicate); + Assert.False(updatedOriginal.IsDeleted); + Assert.True(updatedDuplicate.IsDeleted); + } + + [Fact] + public async Task FixDuplicateStacks_WithClosedEventIndex_KeepsAllStacksActive() + { + // Arrange + var organization = _organizationData.GenerateSampleOrganization(_billingManager, _plans); + await _organizationRepository.AddAsync(organization, o => o.ImmediateConsistency()); + var project = await _projectRepository.AddAsync(_projectData.GenerateSampleProject(), o => o.ImmediateConsistency()); + + var occurrenceDate = _configuration.TimeProvider.GetUtcNow().AddDays(-1); + var originalStack = _stackData.GenerateStack(generateId: true, organizationId: organization.Id, projectId: project.Id); + var duplicateStack = originalStack.DeepClone(); + duplicateStack.Id = ObjectId.GenerateNewId().ToString(); + await _stackRepository.AddAsync([originalStack, duplicateStack], o => o.ImmediateConsistency()); + await _eventRepository.AddAsync( + Enumerable.Range(0, 10).Select(_ => _eventData.GenerateEvent( + organization.Id, project.Id, duplicateStack.Id, occurrenceDate: occurrenceDate)), + o => o.ImmediateConsistency()); + + string eventIndex = _configuration.Events.GetVersionedIndex(occurrenceDate.UtcDateTime); + var closeResponse = await _configuration.Client.Indices.CloseAsync(eventIndex, TestContext.Current.CancellationToken); + Assert.True(closeResponse.IsValidResponse, closeResponse.DebugInformation); + + try + { + // Act + await _job.RunAsync(TestCancellationToken); + } + finally + { + var openResponse = await _configuration.Client.Indices.OpenAsync(eventIndex, TestContext.Current.CancellationToken); + Assert.True(openResponse.IsValidResponse, openResponse.DebugInformation); + } + + // Assert + var stacks = await _stackRepository.GetByIdsAsync( + [originalStack.Id, duplicateStack.Id], + o => o.IncludeSoftDeletes().ImmediateConsistency()); + Assert.Equal(2, stacks.Count); + Assert.All(stacks, stack => Assert.False(stack.IsDeleted)); + Assert.Equal(10, await _eventRepository.CountAsync(q => q.Stack(duplicateStack.Id), o => o.ImmediateConsistency())); + } + [Fact] public async Task RunAsync_AllOrphanTypes_CleansUpCorrectly() { - // Arrange - Complex scenario: valid data, orphaned by stack, orphaned by project, orphaned by organization + // Arrange var organization1 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); await _organizationRepository.AddAsync(organization1, o => o.ImmediateConsistency()); @@ -476,23 +839,10 @@ public async Task RunAsync_AllOrphanTypes_CleansUpCorrectly() var validStack = _stackData.GenerateStack(id: TestConstants.StackId, organizationId: organization1.Id, projectId: project1.Id); await _stackRepository.AddAsync(validStack, o => o.ImmediateConsistency()); - // 100 valid events var validEvents = _eventData.GenerateEvents(100, organization1.Id, project1.Id, validStack.Id).ToList(); - - // 25 orphaned by stack (stack doesn't exist) - string fakeStack = ObjectId.GenerateNewId().ToString(); - var orphanedByStack = _eventData.GenerateEvents(25, organization1.Id, project1.Id, fakeStack).ToList(); - - // 25 orphaned by project (project doesn't exist) - string fakeProject = ObjectId.GenerateNewId().ToString(); - string fakeStack2 = ObjectId.GenerateNewId().ToString(); - var orphanedByProject = _eventData.GenerateEvents(25, organization1.Id, fakeProject, fakeStack2).ToList(); - - // 25 orphaned by organization (organization doesn't exist) - string fakeOrganizationId = ObjectId.GenerateNewId().ToString(); - string fakeProject2 = ObjectId.GenerateNewId().ToString(); - string fakeStack3 = ObjectId.GenerateNewId().ToString(); - var orphanedByOrganization = _eventData.GenerateEvents(25, fakeOrganizationId, fakeProject2, fakeStack3).ToList(); + var orphanedByStack = _eventData.GenerateEvents(25, organization1.Id, project1.Id, ObjectId.GenerateNewId().ToString()).ToList(); + var orphanedByProject = _eventData.GenerateEvents(25, organization1.Id, ObjectId.GenerateNewId().ToString(), ObjectId.GenerateNewId().ToString()).ToList(); + var orphanedByOrganization = _eventData.GenerateEvents(25, ObjectId.GenerateNewId().ToString(), ObjectId.GenerateNewId().ToString(), ObjectId.GenerateNewId().ToString()).ToList(); await _eventRepository.AddAsync(validEvents.Concat(orphanedByStack).Concat(orphanedByProject).Concat(orphanedByOrganization), o => o.ImmediateConsistency()); @@ -502,7 +852,7 @@ public async Task RunAsync_AllOrphanTypes_CleansUpCorrectly() // Act await _job.RunAsync(TestCancellationToken); - // Assert - Only 100 valid events remain + // Assert var totalAfter = await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency()); Assert.Equal(100, totalAfter); } @@ -510,7 +860,7 @@ public async Task RunAsync_AllOrphanTypes_CleansUpCorrectly() [Fact] public async Task RunAsync_NoOrphans_PreservesEverything() { - // Arrange - Two complete tenants, no orphans anywhere + // Arrange var organization1 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId); var organization2 = _organizationData.GenerateOrganization(_billingManager, _plans, id: TestConstants.OrganizationId2); await _organizationRepository.AddAsync([organization1, organization2], o => o.ImmediateConsistency()); @@ -530,7 +880,7 @@ public async Task RunAsync_NoOrphans_PreservesEverything() // Act await _job.RunAsync(TestCancellationToken); - // Assert - All 400 events preserved + // Assert var totalAfter = await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency()); Assert.Equal(400, totalAfter); } @@ -538,11 +888,12 @@ public async Task RunAsync_NoOrphans_PreservesEverything() [Fact] public async Task RunAsync_EmptyDatabase_CompletesWithoutError() { - // Arrange - nothing + // Arrange - // Act & Assert - should not throw + // Act await _job.RunAsync(TestCancellationToken); + // Assert var totalAfter = await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency()); Assert.Equal(0, totalAfter); } diff --git a/tests/Exceptionless.Tests/Repositories/EventRepositoryTests.cs b/tests/Exceptionless.Tests/Repositories/EventRepositoryTests.cs index 95c5d2b628..dd4466729a 100644 --- a/tests/Exceptionless.Tests/Repositories/EventRepositoryTests.cs +++ b/tests/Exceptionless.Tests/Repositories/EventRepositoryTests.cs @@ -275,4 +275,195 @@ private async Task CreateDataAsync() _ids.Add(Tuple.Create(ev.Id, date)); } } + + [Fact] + public async Task GetDistinctStackIds_WithMultipleStacks_ReturnsAllUniqueIds() + { + // Arrange + var stack1 = await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: TestConstants.ProjectId), o => o.ImmediateConsistency()); + var stack2 = await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: TestConstants.ProjectId), o => o.ImmediateConsistency()); + + await _repository.AddAsync(_eventData.GenerateEvents(5, TestConstants.OrganizationId, TestConstants.ProjectId, stack1.Id), o => o.ImmediateConsistency()); + await _repository.AddAsync(_eventData.GenerateEvents(3, TestConstants.OrganizationId, TestConstants.ProjectId, stack2.Id), o => o.ImmediateConsistency()); + + // Act + var page = await _repository.GetDistinctStackIdsAsync(10000, cancellationToken: TestContext.Current.CancellationToken); + var stackIds = page.Values; + + // Assert + Assert.Equal(stackIds.Count, stackIds.Distinct(StringComparer.Ordinal).Count()); + Assert.Contains(stack1.Id, stackIds); + Assert.Contains(stack2.Id, stackIds); + } + + [Fact] + public async Task GetDistinctStackIds_WithPagination_ReturnsAllIds() + { + // Arrange + var stacks = new List(); + for (int i = 0; i < 5; i++) + stacks.Add(await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: TestConstants.ProjectId), o => o.ImmediateConsistency())); + + foreach (var stack in stacks) + await _repository.AddAsync(_eventData.GenerateEvents(2, TestConstants.OrganizationId, TestConstants.ProjectId, stack.Id), o => o.ImmediateConsistency()); + + // Act - page through with batch size of 2 + var allIds = new List(); + string? nextValue = null; + do + { + var page = await _repository.GetDistinctStackIdsAsync(2, nextValue, TestContext.Current.CancellationToken); + allIds.AddRange(page.Values); + nextValue = page.NextValue; + } while (!String.IsNullOrEmpty(nextValue)); + + // Assert + Assert.Equal(allIds.Count, allIds.Distinct(StringComparer.Ordinal).Count()); + foreach (var stack in stacks) + Assert.Contains(stack.Id, allIds); + } + + [Fact] + public async Task ReassignStack_WithSourceEvents_MovesAllEventsToTarget() + { + // Arrange + var stack1 = await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: TestConstants.ProjectId), o => o.ImmediateConsistency()); + var stack2 = await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: TestConstants.ProjectId), o => o.ImmediateConsistency()); + + await _repository.AddAsync(_eventData.GenerateEvents(10, TestConstants.OrganizationId, TestConstants.ProjectId, stack1.Id), o => o.ImmediateConsistency()); + await _repository.AddAsync(_eventData.GenerateEvents(5, TestConstants.OrganizationId, TestConstants.ProjectId, stack2.Id), o => o.ImmediateConsistency()); + + // Act + long affected = await _repository.ReassignStackAsync([stack1.Id], stack2.Id, TestContext.Current.CancellationToken); + + // Assert + Assert.Equal(10, affected); + + await RefreshDataAsync(); + + Assert.Equal(0, await _repository.CountAsync(q => q.Stack(stack1.Id))); + Assert.Equal(15, await _repository.CountAsync(q => q.Stack(stack2.Id))); + } + + [Fact] + public async Task RemoveAllByProjectIds_WithMixedEvents_RemovesOnlyMatchingEvents() + { + // Arrange + var stack = await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: TestConstants.ProjectId), o => o.ImmediateConsistency()); + string otherProjectId = ObjectId.GenerateNewId().ToString(); + await _repository.AddAsync(_eventData.GenerateEvents(10, TestConstants.OrganizationId, TestConstants.ProjectId, stack.Id), o => o.ImmediateConsistency()); + await _repository.AddAsync(_eventData.GenerateEvents(3, TestConstants.OrganizationId, otherProjectId, stack.Id), o => o.ImmediateConsistency()); + + // Act + long removed = await _repository.RemoveAllByProjectIdsAsync([TestConstants.ProjectId]); + + // Assert + Assert.Equal(10, removed); + + await RefreshDataAsync(); + Assert.Equal(0, await _repository.CountAsync(q => q.Project(TestConstants.ProjectId))); + Assert.Equal(3, await _repository.CountAsync(q => q.Project(otherProjectId))); + } + + [Fact] + public async Task RemoveAllByOrganizationIds_WithMixedEvents_RemovesOnlyMatchingEvents() + { + // Arrange + var stack = await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: TestConstants.ProjectId), o => o.ImmediateConsistency()); + string otherOrganizationId = ObjectId.GenerateNewId().ToString(); + await _repository.AddAsync(_eventData.GenerateEvents(10, TestConstants.OrganizationId, TestConstants.ProjectId, stack.Id), o => o.ImmediateConsistency()); + await _repository.AddAsync(_eventData.GenerateEvents(3, otherOrganizationId, TestConstants.ProjectId, stack.Id), o => o.ImmediateConsistency()); + + // Act + long removed = await _repository.RemoveAllByOrganizationIdsAsync([TestConstants.OrganizationId]); + + // Assert + Assert.Equal(10, removed); + + await RefreshDataAsync(); + Assert.Equal(0, await _repository.CountAsync(q => q.Organization(TestConstants.OrganizationId))); + Assert.Equal(3, await _repository.CountAsync(q => q.Organization(otherOrganizationId))); + } + + [Fact] + public async Task RemoveAllByStackIds_WithMixedEvents_RemovesOnlyMatchingEvents() + { + // Arrange + var stack = await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: TestConstants.ProjectId), o => o.ImmediateConsistency()); + var otherStack = await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: TestConstants.ProjectId), o => o.ImmediateConsistency()); + await _repository.AddAsync(_eventData.GenerateEvents(10, TestConstants.OrganizationId, TestConstants.ProjectId, stack.Id), o => o.ImmediateConsistency()); + await _repository.AddAsync(_eventData.GenerateEvents(3, TestConstants.OrganizationId, TestConstants.ProjectId, otherStack.Id), o => o.ImmediateConsistency()); + + // Act + long removed = await _repository.RemoveAllByStackIdsAsync([stack.Id]); + + // Assert + Assert.Equal(10, removed); + + await RefreshDataAsync(); + Assert.Equal(0, await _repository.CountAsync(q => q.Stack(stack.Id))); + Assert.Equal(3, await _repository.CountAsync(q => q.Stack(otherStack.Id))); + } + + [Fact] + public async Task ReassignStack_WithEmptySourceIds_ReturnsZeroWithoutModification() + { + // Arrange + var stack1 = await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: TestConstants.ProjectId), o => o.ImmediateConsistency()); + var stack2 = await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: TestConstants.ProjectId), o => o.ImmediateConsistency()); + await _repository.AddAsync(_eventData.GenerateEvents(10, TestConstants.OrganizationId, TestConstants.ProjectId, stack1.Id), o => o.ImmediateConsistency()); + + // Act - empty source list must be a no-op; an unchecked empty .Stack() filter would patch ALL events + long affected = await _repository.ReassignStackAsync([], stack2.Id, TestContext.Current.CancellationToken); + + // Assert + Assert.Equal(0, affected); + + await RefreshDataAsync(); + Assert.Equal(10, await _repository.CountAsync(q => q.Stack(stack1.Id))); + Assert.Equal(0, await _repository.CountAsync(q => q.Stack(stack2.Id))); + } + + [Fact] + public async Task GetDistinctProjectIds_WithMultipleProjects_ReturnsAllUniqueIds() + { + // Arrange + var stack = await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: TestConstants.ProjectId), o => o.ImmediateConsistency()); + string project2Id = ObjectId.GenerateNewId().ToString(); + string projectWithoutEventsId = ObjectId.GenerateNewId().ToString(); + await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: projectWithoutEventsId), o => o.ImmediateConsistency()); + + await _repository.AddAsync(_eventData.GenerateEvents(3, TestConstants.OrganizationId, TestConstants.ProjectId, stack.Id), o => o.ImmediateConsistency()); + await _repository.AddAsync(_eventData.GenerateEvents(2, TestConstants.OrganizationId, project2Id, stack.Id), o => o.ImmediateConsistency()); + + // Act + var page = await _repository.GetDistinctProjectIdsAsync(10000, cancellationToken: TestContext.Current.CancellationToken); + var projectIds = page.Values; + + // Assert + Assert.Equal(projectIds.Count, projectIds.Distinct(StringComparer.Ordinal).Count()); + Assert.Contains(TestConstants.ProjectId, projectIds); + Assert.Contains(project2Id, projectIds); + Assert.DoesNotContain(projectWithoutEventsId, projectIds); + } + + [Fact] + public async Task GetDistinctOrganizationIds_WithMultipleOrganizations_ReturnsAllUniqueIds() + { + // Arrange + var stack = await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: TestConstants.ProjectId), o => o.ImmediateConsistency()); + string organization2Id = ObjectId.GenerateNewId().ToString(); + + await _repository.AddAsync(_eventData.GenerateEvents(3, TestConstants.OrganizationId, TestConstants.ProjectId, stack.Id), o => o.ImmediateConsistency()); + await _repository.AddAsync(_eventData.GenerateEvents(2, organization2Id, TestConstants.ProjectId, stack.Id), o => o.ImmediateConsistency()); + + // Act + var page = await _repository.GetDistinctOrganizationIdsAsync(10000, cancellationToken: TestContext.Current.CancellationToken); + var organizationIds = page.Values; + + // Assert + Assert.Equal(organizationIds.Count, organizationIds.Distinct(StringComparer.Ordinal).Count()); + Assert.Contains(TestConstants.OrganizationId, organizationIds); + Assert.Contains(organization2Id, organizationIds); + } } diff --git a/tests/Exceptionless.Tests/Repositories/StackRepositoryTests.cs b/tests/Exceptionless.Tests/Repositories/StackRepositoryTests.cs index 54b4093187..8737c5a021 100644 --- a/tests/Exceptionless.Tests/Repositories/StackRepositoryTests.cs +++ b/tests/Exceptionless.Tests/Repositories/StackRepositoryTests.cs @@ -7,8 +7,11 @@ using Exceptionless.Tests.Utility; using Foundatio.Caching; using Foundatio.Repositories; +using Foundatio.Repositories.Exceptions; using Foundatio.Repositories.Options; +using Foundatio.Repositories.Utility; using Foundatio.Serializer; +using Foundatio.Utility; using Xunit; namespace Exceptionless.Tests.Repositories; @@ -279,4 +282,355 @@ await _repository.AddAsync( Assert.NotNull(stacks.Documents.SingleOrDefault(s => String.Equals(s.Id, TestConstants.StackId))); Assert.NotNull(stacks.Documents.SingleOrDefault(s => String.Equals(s.Id, TestConstants.StackId2))); } + + [Fact] + public async Task GetDuplicateSignatures_WithDuplicates_ReturnsSignatures() + { + string uniqueProjectId = ObjectId.GenerateNewId().ToString(); + var stack1 = _stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: uniqueProjectId); + stack1.DuplicateSignature = $"{uniqueProjectId}:dup_sig_test"; + + var stack2 = stack1.DeepClone(); + stack2.Id = ObjectId.GenerateNewId().ToString(); + + await _repository.AddAsync(new[] { stack1, stack2 }, o => o.ImmediateConsistency()); + + var duplicates = await _repository.GetDuplicateSignaturesAsync(); + Assert.Contains($"{uniqueProjectId}:dup_sig_test", duplicates); + } + + [Fact] + public async Task GetDuplicateSignatures_WithNoDuplicates_ReturnsEmpty() + { + // Use a unique project ID to avoid interference from pre-existing sample data + string uniqueProjectId = ObjectId.GenerateNewId().ToString(); + var stack1 = _stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: uniqueProjectId); + stack1.DuplicateSignature = $"{uniqueProjectId}:unique_sig_1"; + + var stack2 = _stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: uniqueProjectId); + stack2.DuplicateSignature = $"{uniqueProjectId}:unique_sig_2"; + + await _repository.AddAsync(new[] { stack1, stack2 }, o => o.ImmediateConsistency()); + + var duplicates = await _repository.GetDuplicateSignaturesAsync(); + // Should not contain our unique signatures since they each appear only once + Assert.DoesNotContain($"{uniqueProjectId}:unique_sig_1", duplicates); + Assert.DoesNotContain($"{uniqueProjectId}:unique_sig_2", duplicates); + } + + [Fact] + public async Task GetDuplicateSignatures_WithSoftDeletedStacks_ExcludesThem() + { + // Use a unique project ID to avoid interference from pre-existing sample data + string uniqueProjectId = ObjectId.GenerateNewId().ToString(); + var stack1 = _stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: uniqueProjectId); + stack1.DuplicateSignature = $"{uniqueProjectId}:softdelete_sig"; + + var stack2 = stack1.DeepClone(); + stack2.Id = ObjectId.GenerateNewId().ToString(); + stack2.IsDeleted = true; + + await _repository.AddAsync(new[] { stack1, stack2 }, o => o.ImmediateConsistency()); + + var duplicates = await _repository.GetDuplicateSignaturesAsync(); + // The soft-deleted stack should be excluded, leaving only 1 stack with this signature + Assert.DoesNotContain($"{uniqueProjectId}:softdelete_sig", duplicates); + } + + [Fact] + public async Task GetSoftDeleted_WithRedirect_ExcludesRedirectTombstone() + { + var source = _stackData.GenerateSampleStack(); + source.IsDeleted = true; + source.RedirectToStackId = ObjectId.GenerateNewId().ToString(); + source.NeedsRedirectReconciliation = true; + await _repository.AddAsync(source, o => o.ImmediateConsistency()); + + var softDeleted = await _repository.GetSoftDeleted(); + var redirected = await _repository.GetRedirectedStacksNeedingReconciliationAsync(); + + Assert.DoesNotContain(softDeleted.Documents, stack => stack.Id == source.Id); + Assert.Contains(redirected.Documents, stack => stack.Id == source.Id); + } + + [Fact] + public async Task GetCanonicalStack_WithRedirect_ReturnsActiveTarget() + { + var target = _stackData.GenerateSampleStack(); + var source = target.DeepClone(); + source.Id = ObjectId.GenerateNewId().ToString(); + source.IsDeleted = true; + source.RedirectToStackId = target.Id; + await _repository.AddAsync([target, source], o => o.ImmediateConsistency()); + + var canonical = await _repository.GetCanonicalStackAsync(source.Id); + await _repository.AddEventTagsAsync(source.Id, ["redirected-tag"]); + var updatedTarget = await _repository.GetByIdAsync(target.Id, o => o.ImmediateConsistency()); + + Assert.NotNull(canonical); + Assert.Equal(target.Id, canonical.Id); + Assert.NotNull(updatedTarget); + Assert.Contains("redirected-tag", updatedTarget.Tags); + } + + [Fact] + public async Task SetDuplicateStackRedirect_WithCycle_Throws() + { + var stackA = _stackData.GenerateSampleStack(); + var stackB = stackA.DeepClone(); + stackB.Id = ObjectId.GenerateNewId().ToString(); + await _repository.AddAsync([stackA, stackB], o => o.ImmediateConsistency()); + await _repository.SetDuplicateStackRedirectAsync(stackA, stackB.Id); + + await Assert.ThrowsAsync(() => + _repository.SetDuplicateStackRedirectAsync(stackB, stackA.Id)); + + var unchangedTarget = await _repository.GetByIdAsync(stackB.Id, o => o.ImmediateConsistency()); + Assert.NotNull(unchangedTarget); + Assert.Null(unchangedTarget.RedirectToStackId); + } + + [Fact] + public async Task Save_WithStaleVersion_CannotOverwriteDuplicateMerge() + { + var target = _stackData.GenerateSampleStack(); + target.TotalOccurrences = 100; + var source = target.DeepClone(); + source.Id = ObjectId.GenerateNewId().ToString(); + source.TotalOccurrences = 10; + await _repository.AddAsync([target, source], o => o.ImmediateConsistency()); + + var staleTarget = await _repository.GetByIdAsync(target.Id, o => o.ImmediateConsistency()); + Assert.NotNull(staleTarget); + await _repository.MergeDuplicateStackAsync(target.Id, source); + + staleTarget.Title = "stale write"; + await Assert.ThrowsAsync(() => + _repository.SaveAsync(staleTarget, o => o.ImmediateConsistency())); + + var mergedTarget = await _repository.GetByIdAsync(target.Id, o => o.ImmediateConsistency()); + Assert.NotNull(mergedTarget); + Assert.Equal(110, mergedTarget.TotalOccurrences); + Assert.Equal(10, mergedTarget.MergedDuplicateStackTotals[source.Id]); + } + + [Fact] + public async Task AddEventTags_WithConcurrentCounterUpdates_PreservesBothChanges() + { + var stack = _stackData.GenerateSampleStack(); + stack.TotalOccurrences = 0; + await _repository.AddAsync(stack, o => o.ImmediateConsistency()); + + await Task.WhenAll(Enumerable.Range(0, 20).Select(async index => + { + await Task.WhenAll( + _repository.AddEventTagsAsync(stack.Id, [$"tag-{index}"]), + _repository.IncrementEventCounterAsync( + stack.OrganizationId, + stack.ProjectId, + stack.Id, + stack.FirstOccurrence, + stack.LastOccurrence.AddMinutes(index + 1), + 1, + sendNotifications: false)); + })); + + var updated = await _repository.GetByIdAsync(stack.Id, o => o.ImmediateConsistency()); + Assert.NotNull(updated); + Assert.Equal(20, updated.TotalOccurrences); + Assert.All(Enumerable.Range(0, 20), index => Assert.Contains($"tag-{index}", updated.Tags)); + } + + [Fact] + public async Task MarkOpen_WithConcurrentCounterUpdate_PreservesBothChanges() + { + var stack = _stackData.GenerateSampleStack(); + stack.Status = StackStatus.Snoozed; + stack.SnoozeUntilUtc = DateTime.UtcNow.AddMinutes(-1); + stack.TotalOccurrences = 10; + await _repository.AddAsync(stack, o => o.ImmediateConsistency()); + + await Task.WhenAll( + _repository.MarkOpenAsync([stack.Id]), + _repository.IncrementEventCounterAsync( + stack.OrganizationId, + stack.ProjectId, + stack.Id, + stack.FirstOccurrence, + stack.LastOccurrence.AddMinutes(1), + 1, + sendNotifications: false)); + + var updated = await _repository.GetByIdAsync(stack.Id, o => o.ImmediateConsistency()); + Assert.NotNull(updated); + Assert.Equal(StackStatus.Open, updated.Status); + Assert.Null(updated.SnoozeUntilUtc); + Assert.Equal(11, updated.TotalOccurrences); + } + + [Fact] + public async Task MarkAsRegressed_WithConcurrentCounterUpdate_PreservesBothChanges() + { + var stack = _stackData.GenerateSampleStack(); + stack.Status = StackStatus.Fixed; + stack.TotalOccurrences = 10; + await _repository.AddAsync(stack, o => o.ImmediateConsistency()); + + await Task.WhenAll( + _repository.MarkAsRegressedAsync(stack.Id), + _repository.IncrementEventCounterAsync( + stack.OrganizationId, + stack.ProjectId, + stack.Id, + stack.FirstOccurrence, + stack.LastOccurrence.AddMinutes(1), + 1, + sendNotifications: false)); + + var updated = await _repository.GetByIdAsync(stack.Id, o => o.ImmediateConsistency()); + Assert.NotNull(updated); + Assert.Equal(StackStatus.Regressed, updated.Status); + Assert.Equal(11, updated.TotalOccurrences); + } + + [Fact] + public async Task MergeDuplicateStack_WithRedirectChain_AppliesOnlyLateDelta() + { + var stackA = _stackData.GenerateSampleStack(); + stackA.TotalOccurrences = 100; + var stackB = stackA.DeepClone(); + stackB.Id = ObjectId.GenerateNewId().ToString(); + stackB.TotalOccurrences = 10; + var stackC = stackA.DeepClone(); + stackC.Id = ObjectId.GenerateNewId().ToString(); + stackC.TotalOccurrences = 50; + await _repository.AddAsync([stackA, stackB, stackC], o => o.ImmediateConsistency()); + + await _repository.MergeDuplicateStackAsync(stackA.Id, stackB); + await _repository.SetDuplicateStackRedirectAsync(stackB, stackA.Id, isDeleted: true); + + stackA = await _repository.GetByIdAsync(stackA.Id, o => o.ImmediateConsistency()) ?? throw new InvalidOperationException(); + await _repository.MergeDuplicateStackAsync(stackC.Id, stackA); + await _repository.SetDuplicateStackRedirectAsync(stackA, stackC.Id, isDeleted: true); + + await _repository.IncrementEventCounterAsync( + stackB.OrganizationId, + stackB.ProjectId, + stackB.Id, + stackB.FirstOccurrence, + stackB.LastOccurrence.AddMinutes(1), + 5, + sendNotifications: false); + stackB = await _repository.GetByIdAsync(stackB.Id, o => o.IncludeSoftDeletes().ImmediateConsistency()) ?? throw new InvalidOperationException(); + await _repository.MergeDuplicateStackAsync(stackC.Id, stackB); + + var canonical = await _repository.GetCanonicalStackAsync(stackB.Id); + var mergedTarget = await _repository.GetByIdAsync(stackC.Id, o => o.ImmediateConsistency()); + Assert.NotNull(canonical); + Assert.Equal(stackC.Id, canonical.Id); + Assert.NotNull(mergedTarget); + Assert.Equal(165, mergedTarget.TotalOccurrences); + Assert.Equal(100, mergedTarget.MergedDuplicateStackTotals[stackA.Id]); + Assert.Equal(15, mergedTarget.MergedDuplicateStackTotals[stackB.Id]); + } + + [Fact] + public async Task RemoveAllByProjectId_WithRedirectTombstone_RemovesAllStacks() + { + string organizationId = ObjectId.GenerateNewId().ToString(); + string projectId = ObjectId.GenerateNewId().ToString(); + var target = _stackData.GenerateStack(generateId: true, organizationId: organizationId, projectId: projectId); + var source = target.DeepClone(); + source.Id = ObjectId.GenerateNewId().ToString(); + await _repository.AddAsync([target, source], o => o.ImmediateConsistency()); + await _repository.SetDuplicateStackRedirectAsync(source, target.Id, isDeleted: true); + + await _repository.RemoveAllByProjectIdAsync(organizationId, projectId); + + var remaining = await _repository.GetByIdsAsync([target.Id, source.Id], o => o.IncludeSoftDeletes().ImmediateConsistency()); + Assert.Empty(remaining); + } + + [Fact] + public async Task RemoveAllByOrganizationId_WithRedirectTombstone_RemovesAllStacks() + { + string organizationId = ObjectId.GenerateNewId().ToString(); + string projectId = ObjectId.GenerateNewId().ToString(); + var target = _stackData.GenerateStack(generateId: true, organizationId: organizationId, projectId: projectId); + var source = target.DeepClone(); + source.Id = ObjectId.GenerateNewId().ToString(); + await _repository.AddAsync([target, source], o => o.ImmediateConsistency()); + await _repository.SetDuplicateStackRedirectAsync(source, target.Id, isDeleted: true); + + await _repository.RemoveAllByOrganizationIdAsync(organizationId); + + var remaining = await _repository.GetByIdsAsync([target.Id, source.Id], o => o.IncludeSoftDeletes().ImmediateConsistency()); + Assert.Empty(remaining); + } + + [Fact] + public async Task MergeDuplicateStack_WithRepeatedSource_AppliesMetadataOnce() + { + // Arrange + var target = _stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: TestConstants.ProjectId); + target.CreatedUtc = new DateTime(2026, 1, 2, 0, 0, 0, DateTimeKind.Utc); + target.LastOccurrence = new DateTime(2026, 1, 2, 1, 0, 0, DateTimeKind.Utc); + target.TotalOccurrences = 100; + target.Tags.Add("target"); + target.References.Add("target-reference"); + + var source = _stackData.GenerateStack(generateId: true, organizationId: TestConstants.OrganizationId, projectId: TestConstants.ProjectId); + source.CreatedUtc = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc); + source.LastOccurrence = new DateTime(2026, 1, 3, 1, 0, 0, DateTimeKind.Utc); + source.TotalOccurrences = 10; + source.Status = StackStatus.Fixed; + source.Tags.Add("source"); + source.References.Add("source-reference"); + source.OccurrencesAreCritical = true; + + await _repository.AddAsync([target, source], o => o.ImmediateConsistency()); + + // Act + DateTime updatedBeforeMerge = target.UpdatedUtc; + await _repository.MergeDuplicateStackAsync(target.Id, source); + + var targetAfterMerge = await _repository.GetByIdAsync(target.Id, o => o.ImmediateConsistency()); + Assert.NotNull(targetAfterMerge); + Assert.True(targetAfterMerge.UpdatedUtc > updatedBeforeMerge); + + await _repository.MergeDuplicateStackAsync(target.Id, source); + var targetAfterNoOp = await _repository.GetByIdAsync(target.Id, o => o.ImmediateConsistency()); + Assert.NotNull(targetAfterNoOp); + Assert.Equal(targetAfterMerge.UpdatedUtc, targetAfterNoOp.UpdatedUtc); + + // A normal full save must preserve the internal retry ledger. + targetAfterMerge.Title = "updated after merge"; + await _repository.SaveAsync(targetAfterMerge, o => o.ImmediateConsistency()); + + await _repository.MergeDuplicateStackAsync(target.Id, source); + + // Assert + var merged = await _repository.GetByIdAsync(target.Id, o => o.ImmediateConsistency()); + Assert.NotNull(merged); + Assert.Equal(110, merged.TotalOccurrences); + Assert.Equal(source.CreatedUtc, merged.CreatedUtc); + Assert.Equal(source.LastOccurrence, merged.LastOccurrence); + Assert.Equal(StackStatus.Fixed, merged.Status); + Assert.Contains("target", merged.Tags); + Assert.Contains("source", merged.Tags); + Assert.Contains("target-reference", merged.References); + Assert.Contains("source-reference", merged.References); + Assert.True(merged.OccurrencesAreCritical); + + // Late in-flight occurrences on the redirected source are merged as a delta. + source.TotalOccurrences = 15; + source.LastOccurrence = source.LastOccurrence.AddMinutes(1); + await _repository.SaveAsync(source, o => o.ImmediateConsistency()); + await _repository.MergeDuplicateStackAsync(target.Id, source); + + merged = await _repository.GetByIdAsync(target.Id, o => o.ImmediateConsistency()); + Assert.NotNull(merged); + Assert.Equal(115, merged.TotalOccurrences); + Assert.Equal(source.LastOccurrence, merged.LastOccurrence); + } } diff --git a/tests/Exceptionless.Tests/Utility/PublicApiCompatibilityTests.cs b/tests/Exceptionless.Tests/Utility/PublicApiCompatibilityTests.cs index 05f47341d4..aa6ffd787d 100644 --- a/tests/Exceptionless.Tests/Utility/PublicApiCompatibilityTests.cs +++ b/tests/Exceptionless.Tests/Utility/PublicApiCompatibilityTests.cs @@ -59,4 +59,25 @@ public void DataDictionaryExtensions_GetValue_PreservesJsonSerializerOptionsOver Assert.Equal("compat-node", environment.MachineName); Assert.Equal("Linux", environment.OSName); } + + [Fact] + public void Stack_InternalCleanupState_IsNotSerializedByApi() + { + var stack = new Stack + { + RedirectToStackId = "target-stack", + MergedDuplicateStackTotals = new Dictionary { ["source-stack"] = 42 }, + NeedsRedirectReconciliation = true + }; + + string apiJson = JsonSerializer.Serialize(stack, new JsonSerializerOptions().ConfigureExceptionlessApiDefaults()); + string storageJson = JsonSerializer.Serialize(stack, new JsonSerializerOptions().ConfigureExceptionlessDefaults()); + + Assert.DoesNotContain("redirect_to_stack_id", apiJson); + Assert.DoesNotContain("merged_duplicate_stack_totals", apiJson); + Assert.DoesNotContain("needs_redirect_reconciliation", apiJson); + Assert.Contains("redirect_to_stack_id", storageJson); + Assert.Contains("merged_duplicate_stack_totals", storageJson); + Assert.Contains("needs_redirect_reconciliation", storageJson); + } }