diff --git a/src/Exceptionless.Core/Jobs/CleanupDataJob.cs b/src/Exceptionless.Core/Jobs/CleanupDataJob.cs index 8929aabe9e..22f3ff7f81 100644 --- a/src/Exceptionless.Core/Jobs/CleanupDataJob.cs +++ b/src/Exceptionless.Core/Jobs/CleanupDataJob.cs @@ -27,6 +27,7 @@ public class CleanupDataJob : JobWithLockBase, IHealthCheck private readonly ITokenRepository _tokenRepository; private readonly IWebHookRepository _webHookRepository; private readonly BillingManager _billingManager; + private readonly UsageService _usageService; private readonly AppOptions _appOptions; private readonly ILockProvider _lockProvider; private readonly ICacheClient _cacheClient; @@ -43,6 +44,7 @@ public CleanupDataJob( ILockProvider lockProvider, ICacheClient cacheClient, BillingManager billingManager, + UsageService usageService, AppOptions appOptions, TimeProvider timeProvider, IResiliencePolicyProvider resiliencePolicyProvider, @@ -57,6 +59,7 @@ ILoggerFactory loggerFactory _tokenRepository = tokenRepository; _webHookRepository = webHookRepository; _billingManager = billingManager; + _usageService = usageService; _appOptions = appOptions; _lockProvider = lockProvider; _cacheClient = cacheClient; @@ -164,7 +167,7 @@ private async Task CleanupSoftDeletedStacksAsync(JobContext context) { try { - await RemoveStacksAsync(stackResults.Documents, context); + await RemoveStacksAsync(stackResults.Documents, context, trackDeletedUsage: true); } catch (Exception ex) { @@ -206,6 +209,9 @@ private async Task RemoveProjectsAsync(Project project, JobContext context) await RenewLockAsync(context); long removedEvents = await _eventRepository.RemoveAllByProjectIdAsync(project.OrganizationId, project.Id); + if (removedEvents > 0) + await _usageService.IncrementDeletedAsync(project.OrganizationId, project.Id, removedEvents); + await RenewLockAsync(context); long removedStacks = await _stackRepository.RemoveAllByProjectIdAsync(project.OrganizationId, project.Id); @@ -213,17 +219,29 @@ private async Task RemoveProjectsAsync(Project project, JobContext context) _logger.RemoveProjectComplete(project.Name, project.Id, removedStacks, removedEvents); } - private async Task RemoveStacksAsync(IReadOnlyCollection stacks, JobContext context) + private async Task RemoveStacksAsync(IReadOnlyCollection stacks, JobContext context, bool trackDeletedUsage = false) { await RenewLockAsync(context); - string[] stackIds = stacks.Select(s => s.Id).ToArray(); - long removedEvents = await _eventRepository.RemoveAllByStackIdsAsync(stackIds); + var projectGroups = stacks.GroupBy(s => (s.OrganizationId, s.ProjectId)).ToList(); + + long totalRemovedEvents = 0; + foreach (var projectGroup in projectGroups) + { + string[] stackIds = projectGroup.Select(s => s.Id).ToArray(); + long removedEvents = await _eventRepository.RemoveAllByStackIdsAsync(stackIds); + totalRemovedEvents += removedEvents; + + if (trackDeletedUsage && removedEvents > 0) + await _usageService.IncrementDeletedAsync(projectGroup.Key.OrganizationId, projectGroup.Key.ProjectId, removedEvents); + } + await _stackRepository.RemoveAsync(stacks); - foreach (var orgGroup in stacks.GroupBy(s => (s.OrganizationId, s.ProjectId))) - await _cacheClient.RemoveByPrefixAsync(EventStackFilterQueryBuilder.GetScopedCachePrefix(orgGroup.Key.OrganizationId, orgGroup.Key.ProjectId)); - _logger.RemoveStacksComplete(stackIds.Length, removedEvents); + foreach (var projectGroup in projectGroups) + await _cacheClient.RemoveByPrefixAsync(EventStackFilterQueryBuilder.GetScopedCachePrefix(projectGroup.Key.OrganizationId, projectGroup.Key.ProjectId)); + + _logger.RemoveStacksComplete(stacks.Count, totalRemovedEvents); } private async Task EnforceRetentionAsync(JobContext context) diff --git a/src/Exceptionless.Core/Jobs/WorkItemHandlers/ResetProjectDataWorkItemHandler.cs b/src/Exceptionless.Core/Jobs/WorkItemHandlers/ResetProjectDataWorkItemHandler.cs index 3bed9eaeb3..ba265dd7bb 100644 --- a/src/Exceptionless.Core/Jobs/WorkItemHandlers/ResetProjectDataWorkItemHandler.cs +++ b/src/Exceptionless.Core/Jobs/WorkItemHandlers/ResetProjectDataWorkItemHandler.cs @@ -1,6 +1,7 @@ using Exceptionless.Core.Models.WorkItems; using Exceptionless.Core.Repositories; using Exceptionless.Core.Repositories.Queries; +using Exceptionless.Core.Services; using Foundatio.Caching; using Foundatio.Jobs; using Foundatio.Lock; @@ -14,13 +15,15 @@ public class ResetProjectDataWorkItemHandler : WorkItemHandlerBase private readonly IStackRepository _stackRepository; private readonly ICacheClient _cacheClient; private readonly ILockProvider _lockProvider; + private readonly UsageService _usageService; - public ResetProjectDataWorkItemHandler(IEventRepository eventRepository, IStackRepository stackRepository, ICacheClient cacheClient, ILockProvider lockProvider, ILoggerFactory loggerFactory) : base(loggerFactory) + public ResetProjectDataWorkItemHandler(IEventRepository eventRepository, IStackRepository stackRepository, ICacheClient cacheClient, ILockProvider lockProvider, UsageService usageService, ILoggerFactory loggerFactory) : base(loggerFactory) { _eventRepository = eventRepository; _stackRepository = stackRepository; _cacheClient = cacheClient; _lockProvider = lockProvider; + _usageService = usageService; } public override Task GetWorkItemLockAsync(object workItem, CancellationToken cancellationToken = default) @@ -41,6 +44,9 @@ public override async Task HandleItemAsync(WorkItemContext context) long removedEvents = await _eventRepository.RemoveAllByProjectIdAsync(workItem.OrganizationId, workItem.ProjectId); await context.ReportProgressAsync(50, $"Events removed: {removedEvents}"); + if (removedEvents > 0) + await _usageService.IncrementDeletedAsync(workItem.OrganizationId, workItem.ProjectId, removedEvents); + long removedStacks = await _stackRepository.RemoveAllByProjectIdAsync(workItem.OrganizationId, workItem.ProjectId); await _cacheClient.RemoveByPrefixAsync(EventStackFilterQueryBuilder.GetScopedCachePrefix(workItem.OrganizationId, workItem.ProjectId)); diff --git a/src/Exceptionless.Core/Models/UsageInfo.cs b/src/Exceptionless.Core/Models/UsageInfo.cs index 16ea919a61..a262486395 100644 --- a/src/Exceptionless.Core/Models/UsageInfo.cs +++ b/src/Exceptionless.Core/Models/UsageInfo.cs @@ -9,6 +9,7 @@ public record UsageInfo public int Blocked { get; set; } public int Discarded { get; set; } public int TooBig { get; set; } + public long Deleted { get; set; } } public record UsageHourInfo @@ -18,6 +19,7 @@ public record UsageHourInfo public int Blocked { get; set; } public int Discarded { get; set; } public int TooBig { get; set; } + public long Deleted { get; set; } } public record UsageInfoResponse diff --git a/src/Exceptionless.Core/Repositories/Configuration/Indexes/OrganizationIndex.cs b/src/Exceptionless.Core/Repositories/Configuration/Indexes/OrganizationIndex.cs index 11a487b319..0b134b7150 100644 --- a/src/Exceptionless.Core/Repositories/Configuration/Indexes/OrganizationIndex.cs +++ b/src/Exceptionless.Core/Repositories/Configuration/Indexes/OrganizationIndex.cs @@ -60,13 +60,15 @@ public static PropertiesDescriptor AddUsageMappings(this Propertie .Number(fu => fu.Name(i => i.Blocked)) .Number(fu => fu.Name(i => i.Discarded)) .Number(fu => fu.Name(i => i.Limit)) - .Number(fu => fu.Name(i => i.TooBig)))) + .Number(fu => fu.Name(i => i.TooBig)) + .Number(fu => fu.Name(i => i.Deleted)))) .Object(ui => ui.Name(o => o.UsageHours.First()).Properties(p => p .Date(fu => fu.Name(i => i.Date)) .Number(fu => fu.Name(i => i.Total)) .Number(fu => fu.Name(i => i.Blocked)) .Number(fu => fu.Name(i => i.Discarded)) .Number(fu => fu.Name(i => i.Limit)) - .Number(fu => fu.Name(i => i.TooBig)))); + .Number(fu => fu.Name(i => i.TooBig)) + .Number(fu => fu.Name(i => i.Deleted)))); } } diff --git a/src/Exceptionless.Core/Repositories/Configuration/Indexes/ProjectIndex.cs b/src/Exceptionless.Core/Repositories/Configuration/Indexes/ProjectIndex.cs index 38585e9061..8c7ce548ef 100644 --- a/src/Exceptionless.Core/Repositories/Configuration/Indexes/ProjectIndex.cs +++ b/src/Exceptionless.Core/Repositories/Configuration/Indexes/ProjectIndex.cs @@ -51,13 +51,15 @@ public static PropertiesDescriptor AddUsageMappings(this PropertiesDesc .Number(fu => fu.Name(i => i.Blocked)) .Number(fu => fu.Name(i => i.Discarded)) .Number(fu => fu.Name(i => i.Limit)) - .Number(fu => fu.Name(i => i.TooBig)))) + .Number(fu => fu.Name(i => i.TooBig)) + .Number(fu => fu.Name(i => i.Deleted)))) .Object(ui => ui.Name(o => o.UsageHours.First()).Properties(p => p .Date(fu => fu.Name(i => i.Date)) .Number(fu => fu.Name(i => i.Total)) .Number(fu => fu.Name(i => i.Blocked)) .Number(fu => fu.Name(i => i.Discarded)) .Number(fu => fu.Name(i => i.Limit)) - .Number(fu => fu.Name(i => i.TooBig)))); + .Number(fu => fu.Name(i => i.TooBig)) + .Number(fu => fu.Name(i => i.Deleted)))); } } diff --git a/src/Exceptionless.Core/Services/UsageService.cs b/src/Exceptionless.Core/Services/UsageService.cs index 52c1a9aeb2..9b61cadc45 100644 --- a/src/Exceptionless.Core/Services/UsageService.cs +++ b/src/Exceptionless.Core/Services/UsageService.cs @@ -81,6 +81,7 @@ private async Task SavePendingOrganizationUsageAsync(DateTime utcNow) var bucketBlocked = await _cache.GetAsync(GetBucketBlockedCacheKey(bucketUtc, organizationId)); var bucketDiscarded = await _cache.GetAsync(GetBucketDiscardedCacheKey(bucketUtc, organizationId)); var bucketTooBig = await _cache.GetAsync(GetBucketTooBigCacheKey(bucketUtc, organizationId)); + var bucketDeleted = await _cache.GetAsync(GetBucketDeletedCacheKey(bucketUtc, organizationId)); organization.LastEventDateUtc = _timeProvider.GetUtcNow().UtcDateTime; @@ -90,12 +91,14 @@ private async Task SavePendingOrganizationUsageAsync(DateTime utcNow) usage.Blocked += bucketBlocked?.Value ?? 0; usage.Discarded += bucketDiscarded?.Value ?? 0; usage.TooBig += bucketTooBig?.Value ?? 0; + usage.Deleted += bucketDeleted?.Value ?? 0; var hourlyUsage = organization.GetHourlyUsage(bucketUtc); hourlyUsage.Total += bucketTotal?.Value ?? 0; hourlyUsage.Blocked += bucketBlocked?.Value ?? 0; hourlyUsage.Discarded += bucketDiscarded?.Value ?? 0; hourlyUsage.TooBig += bucketTooBig?.Value ?? 0; + hourlyUsage.Deleted += bucketDeleted?.Value ?? 0; organization.TrimUsage(_timeProvider); @@ -104,6 +107,7 @@ await _cache.RemoveAllAsync(new[] { GetBucketBlockedCacheKey(bucketUtc, organizationId), GetBucketDiscardedCacheKey(bucketUtc, organizationId), GetBucketTooBigCacheKey(bucketUtc, organizationId), + GetBucketDeletedCacheKey(bucketUtc, organizationId), GetThrottledKey(bucketUtc, organizationId) }); @@ -159,6 +163,7 @@ private async Task SavePendingProjectUsageAsync(DateTime utcNow) var bucketBlocked = await _cache.GetAsync(GetBucketBlockedCacheKey(bucketUtc, project.OrganizationId, projectId)); var bucketDiscarded = await _cache.GetAsync(GetBucketDiscardedCacheKey(bucketUtc, project.OrganizationId, projectId)); var bucketTooBig = await _cache.GetAsync(GetBucketTooBigCacheKey(bucketUtc, project.OrganizationId, projectId)); + var bucketDeleted = await _cache.GetAsync(GetBucketDeletedCacheKey(bucketUtc, project.OrganizationId, projectId)); project.LastEventDateUtc = _timeProvider.GetUtcNow().UtcDateTime; @@ -171,12 +176,14 @@ private async Task SavePendingProjectUsageAsync(DateTime utcNow) usage.Blocked += bucketBlocked?.Value ?? 0; usage.Discarded += bucketDiscarded?.Value ?? 0; usage.TooBig += bucketTooBig?.Value ?? 0; + usage.Deleted += bucketDeleted?.Value ?? 0; var hourlyUsage = project.GetHourlyUsage(bucketUtc); hourlyUsage.Total += bucketTotal?.Value ?? 0; hourlyUsage.Blocked += bucketBlocked?.Value ?? 0; hourlyUsage.Discarded += bucketDiscarded?.Value ?? 0; hourlyUsage.TooBig += bucketTooBig?.Value ?? 0; + hourlyUsage.Deleted += bucketDeleted?.Value ?? 0; project.TrimUsage(_timeProvider); @@ -184,7 +191,8 @@ await _cache.RemoveAllAsync(new[] { GetBucketTotalCacheKey(bucketUtc, project.OrganizationId, projectId), GetBucketDiscardedCacheKey(bucketUtc, project.OrganizationId, projectId), GetBucketBlockedCacheKey(bucketUtc, project.OrganizationId, projectId), - GetBucketTooBigCacheKey(bucketUtc, project.OrganizationId, projectId) + GetBucketTooBigCacheKey(bucketUtc, project.OrganizationId, projectId), + GetBucketDeletedCacheKey(bucketUtc, project.OrganizationId, projectId) }); await _cache.SetAsync(GetTotalCacheKey(utcNow, project.OrganizationId, projectId), usage.Total, TimeSpan.FromHours(8)); @@ -332,6 +340,10 @@ public async Task GetUsageAsync(string organizationId, string usage.CurrentUsage.TooBig += bucketTooBig?.Value ?? 0; usage.CurrentHourUsage.TooBig += bucketTooBig?.Value ?? 0; + var bucketDeleted = await _cache.GetAsync(GetBucketDeletedCacheKey(bucketUtc, organizationId, projectId)); + usage.CurrentUsage.Deleted += bucketDeleted?.Value ?? 0; + usage.CurrentHourUsage.Deleted += bucketDeleted?.Value ?? 0; + bucketUtc = bucketUtc.Add(_bucketSize); } @@ -473,6 +485,29 @@ public async Task IncrementTooBigAsync(string organizationId, string? projectId) AppDiagnostics.PostTooBig.Add(1); } + public async Task IncrementDeletedAsync(string organizationId, string? projectId, long eventCount = 1) + { + if (eventCount <= 0) + return; + + var utcNow = _timeProvider.GetUtcNow().UtcDateTime; + + var tasks = new List(4) + { + _cache.IncrementAsync(GetBucketDeletedCacheKey(utcNow, organizationId), eventCount, TimeSpan.FromHours(8)), + _cache.ListAddAsync(GetOrganizationSetKey(utcNow), organizationId, TimeSpan.FromHours(8)) + }; + + if (!String.IsNullOrEmpty(projectId)) + { + tasks.Add(_cache.IncrementAsync(GetBucketDeletedCacheKey(utcNow, organizationId, projectId), eventCount, TimeSpan.FromHours(8))); + tasks.Add(_cache.ListAddAsync(GetProjectSetKey(utcNow), projectId, TimeSpan.FromHours(8))); + } + + await Task.WhenAll(tasks); + AppDiagnostics.EventsDeleted.Add(eventCount); + } + private int GetBucketEventLimit(int maxEventsPerMonth) { if (maxEventsPerMonth < 5000) @@ -539,6 +574,16 @@ private string GetBucketTooBigCacheKey(DateTime utcTime, string organizationId, return $"usage:{bucket}:{organizationId}:{projectId}:toobig"; } + private string GetBucketDeletedCacheKey(DateTime utcTime, string organizationId, string? projectId = null) + { + int bucket = GetCurrentBucket(utcTime); + + if (String.IsNullOrEmpty(projectId)) + return $"usage:{bucket}:{organizationId}:deleted"; + + return $"usage:{bucket}:{organizationId}:{projectId}:deleted"; + } + private string GetOrganizationSetKey(DateTime utcTime) { int bucket = GetCurrentBucket(utcTime); diff --git a/src/Exceptionless.Core/Utility/AppDiagnostics.cs b/src/Exceptionless.Core/Utility/AppDiagnostics.cs index 6ef85bc829..46aa58e979 100644 --- a/src/Exceptionless.Core/Utility/AppDiagnostics.cs +++ b/src/Exceptionless.Core/Utility/AppDiagnostics.cs @@ -97,6 +97,7 @@ public GaugeInfo(Meter meter, string name) internal static readonly Counter EventsDiscarded = Meter.CreateCounter("ex.events.discarded", description: "Events that were discarded"); internal static readonly Counter EventsBlocked = Meter.CreateCounter("ex.events.blocked", description: "Events that were blocked"); internal static readonly Counter EventsProcessCancelled = Meter.CreateCounter("ex.events.processing.cancelled", description: "Events that started processing and were cancelled"); + internal static readonly Counter EventsDeleted = Meter.CreateCounter("ex.events.deleted", description: "Events that were deleted"); internal static readonly Counter EventsRetryCount = Meter.CreateCounter("ex.events.retry.count", description: "Events where processing was retried"); internal static readonly Counter EventsRetryErrors = Meter.CreateCounter("ex.events.retry.errors", description: "Events where retry processing got an error"); internal static readonly Histogram EventsFieldCount = Meter.CreateHistogram("ex.events.field.count", description: "Number of fields per event"); diff --git a/src/Exceptionless.Web/ClientApp.angular/app/organization/manage/manage-controller.js b/src/Exceptionless.Web/ClientApp.angular/app/organization/manage/manage-controller.js index 804857e4e4..cca9a5b8fc 100644 --- a/src/Exceptionless.Web/ClientApp.angular/app/organization/manage/manage-controller.js +++ b/src/Exceptionless.Web/ClientApp.angular/app/organization/manage/manage-controller.js @@ -146,6 +146,10 @@ }); vm.chart.options.series[4].data = vm.organization.usage.map(function (item) { + return { x: moment.utc(item.date).unix(), y: item.deleted || 0, data: item }; + }); + + vm.chart.options.series[5].data = vm.organization.usage.map(function (item) { return { x: moment.utc(item.date).unix(), y: item.limit, data: item }; }); @@ -287,6 +291,11 @@ color: "#ccc", renderer: "stack", }, + { + name: translateService.T("Deleted"), + color: "#f0ad4e", + renderer: "stack", + }, { name: translateService.T("Limit"), color: "#a94442", diff --git a/src/Exceptionless.Web/ClientApp.angular/app/project/manage/manage-controller.js b/src/Exceptionless.Web/ClientApp.angular/app/project/manage/manage-controller.js index a98a1ed486..7c1ebb7cf6 100644 --- a/src/Exceptionless.Web/ClientApp.angular/app/project/manage/manage-controller.js +++ b/src/Exceptionless.Web/ClientApp.angular/app/project/manage/manage-controller.js @@ -261,7 +261,11 @@ return { x: moment.utc(item.date).unix(), y: item.too_big, data: item }; }); - vm.chart.options.series[5].data = vm.organization.usage.map(function (item) { + vm.chart.options.series[5].data = vm.project.usage.map(function (item) { + return { x: moment.utc(item.date).unix(), y: item.deleted || 0, data: item }; + }); + + vm.chart.options.series[6].data = vm.organization.usage.map(function (item) { return { x: moment.utc(item.date).unix(), y: item.limit, data: item }; }); @@ -788,6 +792,11 @@ color: "#ccc", renderer: "stack", }, + { + name: translateService.T("Deleted"), + color: "#f0ad4e", + renderer: "stack", + }, { name: translateService.T("Limit"), color: "#a94442", diff --git a/src/Exceptionless.Web/ClientApp/src/app.css b/src/Exceptionless.Web/ClientApp/src/app.css index 49c5cf7d8f..6fb6091220 100644 --- a/src/Exceptionless.Web/ClientApp/src/app.css +++ b/src/Exceptionless.Web/ClientApp/src/app.css @@ -50,6 +50,7 @@ --chart-4: #f7d34a; /* Ignored / too big: golden yellow */ --chart-5: #8a8f98; /* Organization total / snoozed: neutral gray */ --chart-6: #d9534f; /* Limit / destructive: classic red */ + --chart-7: #9b2d30; /* Deleted: dark rose */ } .dark { @@ -97,6 +98,7 @@ --chart-4: #ffe16a; /* Ignored / too big: golden yellow */ --chart-5: #a4abb5; /* Organization total / snoozed: neutral gray */ --chart-6: #ff746f; /* Limit / destructive: classic red */ + --chart-7: #c94f53; /* Deleted: dark rose */ } @theme inline { diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/auth/api.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/auth/api.svelte.ts index 758fe4656e..21b17a3afe 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/auth/api.svelte.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/auth/api.svelte.ts @@ -7,6 +7,7 @@ import { createQuery, type QueryClient } from '@tanstack/svelte-query'; import type { Login, TokenResult } from './models'; +import { endSession } from './exceptionless-session'; import { accessToken } from './index.svelte'; const queryKeys = { @@ -97,6 +98,7 @@ export async function login(email: string, password: string) { export async function logout(queryClient?: QueryClient, client = useFetchClient()) { await client.get('auth/logout', { expectedStatusCodes: [200, 401, 403] }); + endSession().catch(() => {}); await queryClient?.cancelQueries(); queryClient?.clear(); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/auth/exceptionless-session.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/auth/exceptionless-session.ts new file mode 100644 index 0000000000..80f9b55f72 --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/auth/exceptionless-session.ts @@ -0,0 +1,42 @@ +import { Exceptionless } from '@exceptionless/browser'; + +let _activeUserId: null | string = null; + +/** + * Ends the current Exceptionless session and clears user identity. + * Call on logout. + */ +export async function endSession(): Promise { + await Exceptionless.submitSessionEnd(); + Exceptionless.config.setUserIdentity('', ''); + _activeUserId = null; +} + +/** + * Sets the current user identity for Exceptionless error tracking. + * Starts a new session only when the identity changes (guards against repeated onSuccess calls from query refetches). + */ +export async function setUserIdentity(userId: string, userName?: string): Promise { + if (!userId) { + return; + } + + if (userName) { + Exceptionless.config.setUserIdentity(userId, userName); + } else { + Exceptionless.config.setUserIdentity(userId); + } + + if (_activeUserId !== userId) { + _activeUserId = userId; + await Exceptionless.submitSessionStart(); + } +} + +/** + * Submits a feature usage event for telemetry tracking. + * Mirrors the legacy Angular $ExceptionlessClient.submitFeatureUsage pattern. + */ +export async function submitFeatureUsage(feature: string): Promise { + await Exceptionless.submitFeatureUsage(feature); +} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/billing/components/change-plan-dialog.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/billing/components/change-plan-dialog.svelte index ec91cb5520..521c2c896c 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/billing/components/change-plan-dialog.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/billing/components/change-plan-dialog.svelte @@ -15,6 +15,7 @@ import { Skeleton } from '$comp/ui/skeleton'; import { Spinner } from '$comp/ui/spinner'; import * as Tabs from '$comp/ui/tabs'; + import { submitFeatureUsage } from '$features/auth/exceptionless-session'; import { FREE_PLAN_ID, isStripeEnabled, StripeProvider } from '$features/billing'; import { type ChangePlanFormData, ChangePlanSchema } from '$features/billing/schemas'; import { changePlanMutation, getPlansQuery } from '$features/organizations/api.svelte'; @@ -256,6 +257,7 @@ } toast.success(result.message ?? 'Your billing plan has been successfully changed.'); + submitFeatureUsage('organization.ChangePlan').catch(() => {}); onclose(true); return null; diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/users/api.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/users/api.svelte.ts index d5c67b3ee0..388e9b98e7 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/users/api.svelte.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/users/api.svelte.ts @@ -1,5 +1,6 @@ import type { WebSocketMessageValue } from '$features/websockets/models'; +import { setUserIdentity } from '$features/auth/exceptionless-session'; import { accessToken } from '$features/auth/index.svelte'; import { type FetchClientResponse, ProblemDetails, useFetchClient } from '@exceptionless/fetchclient'; import { createMutation, createQuery, QueryClient, useQueryClient } from '@tanstack/svelte-query'; @@ -68,6 +69,7 @@ export function getMeQuery() { enabled: () => !!accessToken.current, onSuccess: (data: ViewCurrentUser) => { queryClient.setQueryData(queryKeys.id(data.id!), data); + setUserIdentity(data.id, data.full_name); }, queryClient, queryFn: async ({ signal }: { signal: AbortSignal }) => { diff --git a/src/Exceptionless.Web/ClientApp/src/lib/generated/api.ts b/src/Exceptionless.Web/ClientApp/src/lib/generated/api.ts index 89cbe977b8..8ef1330255 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/generated/api.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/generated/api.ts @@ -407,6 +407,8 @@ export interface UsageHourInfo { discarded: number; /** @format int32 */ too_big: number; + /** @format int32 */ + deleted: number; } export interface UsageInfo { @@ -422,6 +424,8 @@ export interface UsageInfo { discarded: number; /** @format int32 */ too_big: number; + /** @format int32 */ + deleted: number; } export interface User { diff --git a/src/Exceptionless.Web/ClientApp/src/lib/generated/schemas.ts b/src/Exceptionless.Web/ClientApp/src/lib/generated/schemas.ts index e93216315a..d26a77f9a0 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/generated/schemas.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/generated/schemas.ts @@ -460,6 +460,7 @@ export const UsageHourInfoSchema = object({ blocked: int32(), discarded: int32(), too_big: int32(), + deleted: number(), }); export type UsageHourInfoFormData = Infer; @@ -470,6 +471,7 @@ export const UsageInfoSchema = object({ blocked: int32(), discarded: int32(), too_big: int32(), + deleted: number(), }); export type UsageInfoFormData = Infer; diff --git a/src/Exceptionless.Web/ClientApp/src/routes/(app)/organization/[organizationId]/usage/+page.svelte b/src/Exceptionless.Web/ClientApp/src/routes/(app)/organization/[organizationId]/usage/+page.svelte index 50476693bc..0c42001cd2 100644 --- a/src/Exceptionless.Web/ClientApp/src/routes/(app)/organization/[organizationId]/usage/+page.svelte +++ b/src/Exceptionless.Web/ClientApp/src/routes/(app)/organization/[organizationId]/usage/+page.svelte @@ -33,6 +33,7 @@ const chartConfig = { blocked: { color: 'var(--chart-2)', label: 'Blocked' }, + deleted: { color: 'var(--chart-7)', label: 'Deleted' }, discarded: { color: 'var(--chart-3)', label: 'Discarded' }, limit: { color: 'var(--chart-6)', label: 'Limit' }, too_big: { color: 'var(--chart-4)', label: 'Too Big' }, @@ -41,9 +42,8 @@ const chartData = $derived.by(() => { const organization = organizationQuery.data; - const org = organizationQuery.data; - if (!organization?.usage || !org?.usage) { + if (!organization?.usage) { return []; } @@ -60,6 +60,7 @@ { key: 'discarded', ...chartConfig.discarded }, { key: 'blocked', ...chartConfig.blocked }, { key: 'too_big', ...chartConfig.too_big }, + { key: 'deleted', ...chartConfig.deleted }, { key: 'limit', ...chartConfig.limit, @@ -118,7 +119,7 @@ data={chartData} x="date" xScale={scaleUtc()} - yDomain={[0, Math.max(1, ...chartData.map((d) => Math.max(d.total, d.limit, d.blocked, d.discarded, d.too_big)))]} + yDomain={[0, Math.max(1, ...chartData.map((d) => Math.max(d.total, d.limit, d.blocked, d.discarded, d.too_big, d.deleted)))]} {series} props={{ area: { diff --git a/src/Exceptionless.Web/ClientApp/src/routes/(app)/project/[projectId]/usage/+page.svelte b/src/Exceptionless.Web/ClientApp/src/routes/(app)/project/[projectId]/usage/+page.svelte index 152b213977..27f5590cde 100644 --- a/src/Exceptionless.Web/ClientApp/src/routes/(app)/project/[projectId]/usage/+page.svelte +++ b/src/Exceptionless.Web/ClientApp/src/routes/(app)/project/[projectId]/usage/+page.svelte @@ -46,6 +46,7 @@ const chartConfig = { blocked: { color: 'var(--chart-2)', label: 'Blocked' }, + deleted: { color: 'var(--chart-7)', label: 'Deleted' }, discarded: { color: 'var(--chart-3)', label: 'Discarded' }, limit: { color: 'var(--chart-6)', label: 'Limit' }, org_total: { color: 'var(--chart-5)', label: 'Total in Organization' }, @@ -73,6 +74,7 @@ return { blocked: projItem.blocked, date: new Date(projItem.date), + deleted: projItem.deleted, discarded: projItem.discarded, limit: orgItem?.limit || 0, org_total: orgItem?.total || 0, @@ -88,6 +90,7 @@ { key: 'discarded', ...chartConfig.discarded }, { key: 'blocked', ...chartConfig.blocked }, { key: 'too_big', ...chartConfig.too_big }, + { key: 'deleted', ...chartConfig.deleted }, { key: 'limit', ...chartConfig.limit, diff --git a/src/Exceptionless.Web/Controllers/EventController.cs b/src/Exceptionless.Web/Controllers/EventController.cs index 90100c092d..ef1434a95e 100644 --- a/src/Exceptionless.Web/Controllers/EventController.cs +++ b/src/Exceptionless.Web/Controllers/EventController.cs @@ -49,6 +49,7 @@ public class EventController : RepositoryApiController> GetUserCountByProjectIdsAsync(ICo return totals; } - protected override Task> DeleteModelsAsync(ICollection events) + protected override async Task> DeleteModelsAsync(ICollection events) { var user = CurrentUser; - foreach (var projectEvents in events.GroupBy(ev => ev.ProjectId)) + var projectGroups = events.GroupBy(ev => new { ev.OrganizationId, ev.ProjectId }).ToList(); + foreach (var projectGroup in projectGroups) { - var ev = projectEvents.First(); + var ev = projectGroup.First(); using var _ = _logger.BeginScope(new ExceptionlessState().Organization(ev.OrganizationId).Project(ev.ProjectId).Tag("Delete").Identity(user.EmailAddress).Property("User", user).SetHttpContext(HttpContext)); - _logger.LogInformation("User {User} deleted {RemovedCount} events in project ({ProjectId})", user.Id, projectEvents.Count(), ev.ProjectId); + _logger.LogInformation("User {User} deleted {RemovedCount} events in project ({ProjectId})", user.Id, projectGroup.Count(), ev.ProjectId); } - return base.DeleteModelsAsync(events); + var result = await base.DeleteModelsAsync(events); + + foreach (var projectGroup in projectGroups) + { + try + { + await _usageService.IncrementDeletedAsync(projectGroup.Key.OrganizationId, projectGroup.Key.ProjectId, projectGroup.Count()); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to increment deleted usage metrics for org {OrganizationId} project {ProjectId}", projectGroup.Key.OrganizationId, projectGroup.Key.ProjectId); + } + } + + return result; } } diff --git a/src/Exceptionless.Web/Controllers/OrganizationController.cs b/src/Exceptionless.Web/Controllers/OrganizationController.cs index 29911323d4..7d11666508 100644 --- a/src/Exceptionless.Web/Controllers/OrganizationController.cs +++ b/src/Exceptionless.Web/Controllers/OrganizationController.cs @@ -963,12 +963,14 @@ protected override async Task AfterResultMapAsync(ICollection(); _organizationRepository = GetService(); + _projectRepository = GetService(); _stackData = GetService(); _randomEventGenerator = GetService(); _eventData = GetService(); @@ -1990,4 +1992,104 @@ await SendRequestAsync(r => r Assert.Equal("legacy@exceptionless.test", userDescription.EmailAddress); Assert.Equal("Legacy description", userDescription.Description); } + + [Fact] + public async Task DeleteModelsAsync_SingleEvent_IncrementsDeletedUsage() + { + // Arrange + var usageService = GetService(); + + await SendRequestAsync(r => r + .Post() + .AsTestOrganizationClientUser() + .AppendPath("events") + .Content(new Event { Message = "test-delete-usage", Type = Event.KnownTypes.Log }) + .StatusCodeShouldBeAccepted()); + + var processEventsJob = GetService(); + await processEventsJob.RunAsync(TestCancellationToken); + await RefreshDataAsync(); + + var events = await _eventRepository.GetAllAsync(); + var targetEvent = events.Documents.Single(e => String.Equals(e.Message, "test-delete-usage", StringComparison.Ordinal)); + + // Act + await SendRequestAsync(r => r + .Delete() + .AsTestOrganizationUser() + .AppendPath($"events/{targetEvent.Id}") + .StatusCodeShouldBeAccepted()); + + await RefreshDataAsync(); + + // Assert + var remainingEvent = await _eventRepository.GetByIdAsync(targetEvent.Id); + Assert.Null(remainingEvent); + + var usageResponse = await usageService.GetUsageAsync(targetEvent.OrganizationId, targetEvent.ProjectId); + Assert.Equal(1, usageResponse.CurrentUsage.Deleted); + Assert.Equal(1, usageResponse.CurrentHourUsage.Deleted); + + TimeProvider.Advance(TimeSpan.FromMinutes(10)); + await usageService.SavePendingUsageAsync(); + + var organization = await _organizationRepository.GetByIdAsync(targetEvent.OrganizationId); + Assert.NotNull(organization); + var orgUsage = organization.Usage.FirstOrDefault(); + Assert.NotNull(orgUsage); + Assert.Equal(1, orgUsage.Deleted); + } + + [Fact] + public async Task DeleteModelsAsync_MultipleEvents_IncrementsDeletedUsageForAll() + { + // Arrange + var usageService = GetService(); + + for (int eventIndex = 0; eventIndex < 3; eventIndex++) + { + await SendRequestAsync(r => r + .Post() + .AsTestOrganizationClientUser() + .AppendPath("events") + .Content(new Event { Message = $"test-multi-delete-{eventIndex}", Type = Event.KnownTypes.Log }) + .StatusCodeShouldBeAccepted()); + } + + var processEventsJob = GetService(); + await processEventsJob.RunUntilEmptyAsync(TestCancellationToken); + await RefreshDataAsync(); + + var events = await _eventRepository.GetAllAsync(); + var targetEvents = events.Documents.Where(e => e.Message is not null && e.Message.StartsWith("test-multi-delete-", StringComparison.Ordinal)).ToList(); + Assert.Equal(3, targetEvents.Count); + + var ids = String.Join(",", targetEvents.Select(e => e.Id)); + + // Act + await SendRequestAsync(r => r + .Delete() + .AsTestOrganizationUser() + .AppendPath($"events/{ids}") + .StatusCodeShouldBeAccepted()); + + await RefreshDataAsync(); + + // Assert + var firstEvent = targetEvents.First(); + var usageResponse = await usageService.GetUsageAsync(firstEvent.OrganizationId, firstEvent.ProjectId); + Assert.Equal(3, usageResponse.CurrentUsage.Deleted); + Assert.Equal(3, usageResponse.CurrentHourUsage.Deleted); + + TimeProvider.Advance(TimeSpan.FromMinutes(10)); + await usageService.SavePendingUsageAsync(); + + var organization = await _organizationRepository.GetByIdAsync(firstEvent.OrganizationId); + Assert.NotNull(organization); + Assert.Equal(3, organization.Usage.Sum(u => u.Deleted)); + + var project = await _projectRepository.GetByIdAsync(firstEvent.ProjectId); + Assert.NotNull(project); + Assert.Equal(3, project.Usage.Sum(u => u.Deleted)); + } } diff --git a/tests/Exceptionless.Tests/Jobs/CleanupDataJobTests.cs b/tests/Exceptionless.Tests/Jobs/CleanupDataJobTests.cs index 091ca285de..3322012e9b 100644 --- a/tests/Exceptionless.Tests/Jobs/CleanupDataJobTests.cs +++ b/tests/Exceptionless.Tests/Jobs/CleanupDataJobTests.cs @@ -2,6 +2,7 @@ using Exceptionless.Core.Billing; using Exceptionless.Core.Jobs; using Exceptionless.Core.Repositories; +using Exceptionless.Core.Services; using Exceptionless.DateTimeExtensions; using Exceptionless.Tests.Utility; using Foundatio.Repositories; @@ -13,22 +14,24 @@ namespace Exceptionless.Tests.Jobs; public class CleanupDataJobTests : IntegrationTestsBase { private readonly CleanupDataJob _job; - private readonly OrganizationData _organizationData; + private readonly UsageService _usageService; private readonly IOrganizationRepository _organizationRepository; - private readonly ProjectData _projectData; + private readonly OrganizationData _organizationData; private readonly IProjectRepository _projectRepository; - private readonly StackData _stackData; + private readonly ProjectData _projectData; private readonly IStackRepository _stackRepository; - private readonly EventData _eventData; + private readonly StackData _stackData; private readonly IEventRepository _eventRepository; - private readonly TokenData _tokenData; + private readonly EventData _eventData; private readonly ITokenRepository _tokenRepository; + private readonly TokenData _tokenData; private readonly BillingManager _billingManager; private readonly BillingPlans _plans; public CleanupDataJobTests(ITestOutputHelper output, AppWebHostFactory factory) : base(output, factory) { _job = GetService(); + _usageService = GetService(); _organizationData = GetService(); _organizationRepository = GetService(); _projectData = GetService(); @@ -169,4 +172,152 @@ public async Task CanDeleteOrphanedEventsByStack() eventCount = await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency()); Assert.Equal(5000, eventCount); } + + [Fact] + public async Task RemoveProjectsAsync_SoftDeletedProjectWithEvents_IncrementsDeletedUsage() + { + // Arrange + var organization = await _organizationRepository.AddAsync(_organizationData.GenerateSampleOrganization(_billingManager, _plans), o => o.ImmediateConsistency()); + + var project = _projectData.GenerateSampleProject(); + project.IsDeleted = true; + await _projectRepository.AddAsync(project, o => o.ImmediateConsistency()); + + var stack = await _stackRepository.AddAsync(_stackData.GenerateSampleStack(), o => o.ImmediateConsistency()); + var events = _eventData.GenerateEvents(5, organization.Id, project.Id, stack.Id).ToList(); + await _eventRepository.AddAsync(events, o => o.ImmediateConsistency()); + + // Act + await _job.RunAsync(TestCancellationToken); + + // Assert + var orgUsage = await _usageService.GetUsageAsync(organization.Id, null); + Assert.Equal(5, orgUsage.CurrentUsage.Deleted); + Assert.Equal(5, orgUsage.CurrentHourUsage.Deleted); + + TimeProvider.Advance(TimeSpan.FromMinutes(10)); + await _usageService.SavePendingUsageAsync(); + + var savedOrg = await _organizationRepository.GetByIdAsync(organization.Id); + Assert.NotNull(savedOrg); + Assert.Equal(5, savedOrg.Usage.Sum(u => u.Deleted)); + + Assert.Null(await _projectRepository.GetByIdAsync(project.Id, o => o.IncludeSoftDeletes())); + var allEvents = await _eventRepository.GetAllAsync(); + Assert.DoesNotContain(allEvents.Documents, e => String.Equals(e.ProjectId, project.Id, StringComparison.Ordinal)); + } + + [Fact] + public async Task RemoveProjectsAsync_SoftDeletedEmptyProject_DoesNotIncrementDeletedUsage() + { + // Arrange + var organization = await _organizationRepository.AddAsync(_organizationData.GenerateSampleOrganization(_billingManager, _plans), o => o.ImmediateConsistency()); + + var project = _projectData.GenerateSampleProject(); + project.IsDeleted = true; + await _projectRepository.AddAsync(project, o => o.ImmediateConsistency()); + + // Act + await _job.RunAsync(TestCancellationToken); + + // Assert + var orgUsage = await _usageService.GetUsageAsync(organization.Id, null); + Assert.Equal(0, orgUsage.CurrentUsage.Deleted); + Assert.Equal(0, orgUsage.CurrentHourUsage.Deleted); + + Assert.Null(await _projectRepository.GetByIdAsync(project.Id, o => o.IncludeSoftDeletes())); + } + + [Fact] + public async Task RemoveStacksAsync_SoftDeletedStack_IncrementsDeletedUsage() + { + // Arrange + var organization = await _organizationRepository.AddAsync(_organizationData.GenerateSampleOrganization(_billingManager, _plans), o => o.ImmediateConsistency()); + var project = await _projectRepository.AddAsync(_projectData.GenerateSampleProject(), o => o.ImmediateConsistency()); + + var stack = _stackData.GenerateSampleStack(); + stack.IsDeleted = true; + await _stackRepository.AddAsync(stack, o => o.ImmediateConsistency()); + + var events = _eventData.GenerateEvents(3, organization.Id, project.Id, stack.Id).ToList(); + await _eventRepository.AddAsync(events, o => o.ImmediateConsistency()); + + // Act + await _job.RunAsync(TestCancellationToken); + + // Assert + var usageResponse = await _usageService.GetUsageAsync(organization.Id, project.Id); + Assert.Equal(3, usageResponse.CurrentUsage.Deleted); + Assert.Equal(3, usageResponse.CurrentHourUsage.Deleted); + + TimeProvider.Advance(TimeSpan.FromMinutes(10)); + await _usageService.SavePendingUsageAsync(); + + var savedOrg = await _organizationRepository.GetByIdAsync(organization.Id); + Assert.NotNull(savedOrg); + Assert.Equal(3, savedOrg.Usage.Sum(u => u.Deleted)); + + Assert.Null(await _stackRepository.GetByIdAsync(stack.Id, o => o.IncludeSoftDeletes())); + } + + [Fact] + public async Task RemoveStacksAsync_MultipleProjectsSoftDeleted_TracksExactDeletedUsagePerProject() + { + // Arrange + var organization = await _organizationRepository.AddAsync(_organizationData.GenerateSampleOrganization(_billingManager, _plans), o => o.ImmediateConsistency()); + var project1 = await _projectRepository.AddAsync(_projectData.GenerateSampleProject(), o => o.ImmediateConsistency()); + var project2 = await _projectRepository.AddAsync(_projectData.GenerateProject(generateId: true, organizationId: organization.Id), o => o.ImmediateConsistency()); + + var stack1a = _stackData.GenerateStack(generateId: true, organizationId: organization.Id, projectId: project1.Id); + stack1a.IsDeleted = true; + var stack1b = _stackData.GenerateStack(generateId: true, organizationId: organization.Id, projectId: project1.Id); + stack1b.IsDeleted = true; + var stack2 = _stackData.GenerateStack(generateId: true, organizationId: organization.Id, projectId: project2.Id); + stack2.IsDeleted = true; + await _stackRepository.AddAsync([stack1a, stack1b, stack2], o => o.ImmediateConsistency()); + + await _eventRepository.AddAsync(_eventData.GenerateEvents(4, organization.Id, project1.Id, stack1a.Id), o => o.ImmediateConsistency()); + await _eventRepository.AddAsync(_eventData.GenerateEvents(2, organization.Id, project1.Id, stack1b.Id), o => o.ImmediateConsistency()); + await _eventRepository.AddAsync(_eventData.GenerateEvents(3, organization.Id, project2.Id, stack2.Id), o => o.ImmediateConsistency()); + + // Act + await _job.RunAsync(TestCancellationToken); + + // Assert + var usageProject1 = await _usageService.GetUsageAsync(organization.Id, project1.Id); + var usageProject2 = await _usageService.GetUsageAsync(organization.Id, project2.Id); + Assert.Equal(6, usageProject1.CurrentUsage.Deleted); + Assert.Equal(3, usageProject2.CurrentUsage.Deleted); + + TimeProvider.Advance(TimeSpan.FromMinutes(10)); + await _usageService.SavePendingUsageAsync(); + + var savedOrg = await _organizationRepository.GetByIdAsync(organization.Id); + Assert.NotNull(savedOrg); + Assert.Equal(9, savedOrg.Usage.Sum(u => u.Deleted)); + } + + [Fact] + public async Task EnforceRetentionAsync_ExpiredEvents_DoesNotIncrementDeletedUsage() + { + // Arrange + var organization = _organizationData.GenerateSampleOrganization(_billingManager, _plans); + _billingManager.ApplyBillingPlan(organization, _plans.FreePlan); + await _organizationRepository.AddAsync(organization, o => o.ImmediateConsistency()); + var project = await _projectRepository.AddAsync(_projectData.GenerateSampleProject(), o => o.ImmediateConsistency()); + + var options = GetService(); + var expiredDate = DateTimeOffset.UtcNow.SubtractDays(options.MaximumRetentionDays); + + var stack = await _stackRepository.AddAsync(_stackData.GenerateSampleStack(), o => o.ImmediateConsistency()); + await _eventRepository.AddAsync(_eventData.GenerateEvent(organization.Id, project.Id, stack.Id, expiredDate, expiredDate, expiredDate), o => o.ImmediateConsistency()); + + // Act + await _job.RunAsync(TestCancellationToken); + + // Assert + var usageResponse = await _usageService.GetUsageAsync(organization.Id, project.Id); + Assert.Equal(0, usageResponse.CurrentUsage.Deleted); + Assert.Equal(0, usageResponse.CurrentHourUsage.Deleted); + } } diff --git a/tests/Exceptionless.Tests/Jobs/WorkItemHandlers/ResetProjectDataWorkItemHandlerTests.cs b/tests/Exceptionless.Tests/Jobs/WorkItemHandlers/ResetProjectDataWorkItemHandlerTests.cs new file mode 100644 index 0000000000..73b350a367 --- /dev/null +++ b/tests/Exceptionless.Tests/Jobs/WorkItemHandlers/ResetProjectDataWorkItemHandlerTests.cs @@ -0,0 +1,172 @@ +using Exceptionless.Core.Billing; +using Exceptionless.Core.Jobs; +using Exceptionless.Core.Models.WorkItems; +using Exceptionless.Core.Repositories; +using Exceptionless.Core.Services; +using Exceptionless.Tests.Utility; +using Foundatio.Jobs; +using Foundatio.Queues; +using Foundatio.Repositories; +using Xunit; + +namespace Exceptionless.Tests.Jobs.WorkItemHandlers; + +public class ResetProjectDataWorkItemHandlerTests : IntegrationTestsBase +{ + private readonly WorkItemJob _workItemJob; + private readonly IQueue _workItemQueue; + private readonly UsageService _usageService; + private readonly IOrganizationRepository _organizationRepository; + private readonly IProjectRepository _projectRepository; + private readonly IStackRepository _stackRepository; + private readonly IEventRepository _eventRepository; + private readonly OrganizationData _organizationData; + private readonly ProjectData _projectData; + private readonly StackData _stackData; + private readonly EventData _eventData; + private readonly BillingManager _billingManager; + private readonly BillingPlans _plans; + + public ResetProjectDataWorkItemHandlerTests(ITestOutputHelper output, AppWebHostFactory factory) : base(output, factory) + { + _workItemJob = GetService(); + _workItemQueue = GetService>(); + _usageService = GetService(); + _organizationData = GetService(); + _organizationRepository = GetService(); + _projectData = GetService(); + _projectRepository = GetService(); + _stackData = GetService(); + _stackRepository = GetService(); + _eventData = GetService(); + _eventRepository = GetService(); + _billingManager = GetService(); + _plans = GetService(); + } + + [Fact] + public async Task ResetProjectData_WithEvents_IncrementsDeletedUsageAndPersists() + { + // Arrange + var organization = await _organizationRepository.AddAsync(_organizationData.GenerateSampleOrganization(_billingManager, _plans), o => o.ImmediateConsistency()); + var project = await _projectRepository.AddAsync(_projectData.GenerateSampleProject(), o => o.ImmediateConsistency()); + var stack = await _stackRepository.AddAsync(_stackData.GenerateSampleStack(), o => o.ImmediateConsistency()); + + var events = _eventData.GenerateEvents(5, organization.Id, project.Id, stack.Id).ToList(); + await _eventRepository.AddAsync(events, o => o.ImmediateConsistency()); + + // Act + await _workItemQueue.EnqueueAsync(new ResetProjectDataWorkItem { OrganizationId = organization.Id, ProjectId = project.Id }); + await _workItemJob.RunUntilEmptyAsync(TestCancellationToken); + await RefreshDataAsync(); + + // Assert + var remaining = await _eventRepository.GetAllAsync(); + Assert.DoesNotContain(remaining.Documents, e => String.Equals(e.ProjectId, project.Id, StringComparison.Ordinal)); + Assert.Null(await _stackRepository.GetByIdAsync(stack.Id, o => o.IncludeSoftDeletes())); + + var usageResponse = await _usageService.GetUsageAsync(organization.Id, project.Id); + Assert.Equal(5, usageResponse.CurrentUsage.Deleted); + Assert.Equal(5, usageResponse.CurrentHourUsage.Deleted); + + TimeProvider.Advance(TimeSpan.FromMinutes(10)); + await _usageService.SavePendingUsageAsync(); + + var savedOrg = await _organizationRepository.GetByIdAsync(organization.Id); + Assert.NotNull(savedOrg); + Assert.Equal(5, savedOrg.Usage.Sum(u => u.Deleted)); + + var savedProject = await _projectRepository.GetByIdAsync(project.Id); + Assert.NotNull(savedProject); + Assert.Equal(5, savedProject.Usage.Sum(u => u.Deleted)); + } + + [Fact] + public async Task ResetProjectData_EmptyProject_DoesNotIncrementDeletedUsage() + { + // Arrange + var organization = await _organizationRepository.AddAsync(_organizationData.GenerateSampleOrganization(_billingManager, _plans), o => o.ImmediateConsistency()); + var project = await _projectRepository.AddAsync(_projectData.GenerateSampleProject(), o => o.ImmediateConsistency()); + + // Act + await _workItemQueue.EnqueueAsync(new ResetProjectDataWorkItem { OrganizationId = organization.Id, ProjectId = project.Id }); + await _workItemJob.RunUntilEmptyAsync(TestCancellationToken); + + // Assert + var usageResponse = await _usageService.GetUsageAsync(organization.Id, project.Id); + Assert.Equal(0, usageResponse.CurrentUsage.Deleted); + Assert.Equal(0, usageResponse.CurrentHourUsage.Deleted); + } + + [Fact] + public async Task ResetProjectData_MultipleStacks_IncrementsDeletedUsageForAllEvents() + { + // Arrange + var organization = await _organizationRepository.AddAsync(_organizationData.GenerateSampleOrganization(_billingManager, _plans), o => o.ImmediateConsistency()); + var project = await _projectRepository.AddAsync(_projectData.GenerateSampleProject(), o => o.ImmediateConsistency()); + + var stack1 = await _stackRepository.AddAsync(_stackData.GenerateSampleStack(), o => o.ImmediateConsistency()); + var stack2 = await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, organizationId: organization.Id, projectId: project.Id), o => o.ImmediateConsistency()); + + await _eventRepository.AddAsync(_eventData.GenerateEvents(3, organization.Id, project.Id, stack1.Id), o => o.ImmediateConsistency()); + await _eventRepository.AddAsync(_eventData.GenerateEvents(4, organization.Id, project.Id, stack2.Id), o => o.ImmediateConsistency()); + + // Act + await _workItemQueue.EnqueueAsync(new ResetProjectDataWorkItem { OrganizationId = organization.Id, ProjectId = project.Id }); + await _workItemJob.RunUntilEmptyAsync(TestCancellationToken); + + // Assert + var usageResponse = await _usageService.GetUsageAsync(organization.Id, project.Id); + Assert.Equal(7, usageResponse.CurrentUsage.Deleted); + } + + [Fact] + public async Task ResetProjectData_MultipleProjects_OnlyIncrementsTargetProject() + { + // Arrange + var organization = await _organizationRepository.AddAsync(_organizationData.GenerateSampleOrganization(_billingManager, _plans), o => o.ImmediateConsistency()); + var project1 = await _projectRepository.AddAsync(_projectData.GenerateSampleProject(), o => o.ImmediateConsistency()); + var project2 = await _projectRepository.AddAsync(_projectData.GenerateProject(generateId: true, organizationId: organization.Id), o => o.ImmediateConsistency()); + + var stack1 = await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, organizationId: organization.Id, projectId: project1.Id), o => o.ImmediateConsistency()); + var stack2 = await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, organizationId: organization.Id, projectId: project2.Id), o => o.ImmediateConsistency()); + + await _eventRepository.AddAsync(_eventData.GenerateEvents(5, organization.Id, project1.Id, stack1.Id), o => o.ImmediateConsistency()); + await _eventRepository.AddAsync(_eventData.GenerateEvents(3, organization.Id, project2.Id, stack2.Id), o => o.ImmediateConsistency()); + + // Act + await _workItemQueue.EnqueueAsync(new ResetProjectDataWorkItem { OrganizationId = organization.Id, ProjectId = project1.Id }); + await _workItemJob.RunUntilEmptyAsync(TestCancellationToken); + + // Assert + var usage1 = await _usageService.GetUsageAsync(organization.Id, project1.Id); + var usage2 = await _usageService.GetUsageAsync(organization.Id, project2.Id); + + Assert.Equal(5, usage1.CurrentUsage.Deleted); + Assert.Equal(0, usage2.CurrentUsage.Deleted); + + var project2Events = await _eventRepository.GetAllAsync(); + Assert.Equal(3, project2Events.Documents.Count(e => String.Equals(e.ProjectId, project2.Id, StringComparison.Ordinal))); + } + + [Fact] + public async Task ResetProjectData_DeletedUsage_DoesNotReduceEventsLeft() + { + // Arrange + var organization = await _organizationRepository.AddAsync(_organizationData.GenerateSampleOrganization(_billingManager, _plans), o => o.ImmediateConsistency()); + var project = await _projectRepository.AddAsync(_projectData.GenerateSampleProject(), o => o.ImmediateConsistency()); + var stack = await _stackRepository.AddAsync(_stackData.GenerateSampleStack(), o => o.ImmediateConsistency()); + + await _eventRepository.AddAsync(_eventData.GenerateEvents(10, organization.Id, project.Id, stack.Id), o => o.ImmediateConsistency()); + + int eventsLeftBefore = await _usageService.GetEventsLeftAsync(organization.Id); + + // Act + await _workItemQueue.EnqueueAsync(new ResetProjectDataWorkItem { OrganizationId = organization.Id, ProjectId = project.Id }); + await _workItemJob.RunUntilEmptyAsync(TestCancellationToken); + + // Assert + int eventsLeftAfter = await _usageService.GetEventsLeftAsync(organization.Id); + Assert.Equal(eventsLeftBefore, eventsLeftAfter); + } +} diff --git a/tests/Exceptionless.Tests/Services/UsageServiceTests.cs b/tests/Exceptionless.Tests/Services/UsageServiceTests.cs index 61b13936fc..a4ea26928c 100644 --- a/tests/Exceptionless.Tests/Services/UsageServiceTests.cs +++ b/tests/Exceptionless.Tests/Services/UsageServiceTests.cs @@ -76,6 +76,7 @@ await messageBus.SubscribeAsync(po => Assert.Equal(eventsLeftInBucket, usage.Total); Assert.Equal(0, usage.Blocked); Assert.Equal(0, usage.TooBig); + Assert.Equal(0, usage.Deleted); project = await _projectRepository.GetByIdAsync(project.Id); Assert.NotNull(project); @@ -84,6 +85,7 @@ await messageBus.SubscribeAsync(po => Assert.Equal(eventsLeftInBucket, usage.Total); Assert.Equal(0, usage.Blocked); Assert.Equal(0, usage.TooBig); + Assert.Equal(0, usage.Deleted); } [Fact] @@ -217,10 +219,12 @@ public async Task CanIncrementBlockedAsync() Assert.Equal(0, usage.Total); Assert.Equal(1, usage.Blocked); Assert.Equal(0, usage.TooBig); + Assert.Equal(0, usage.Deleted); var overage = organization.UsageHours.Single(); Assert.Equal(0, overage.Total); Assert.Equal(1, overage.Blocked); Assert.Equal(0, overage.TooBig); + Assert.Equal(0, overage.Deleted); project = await _projectRepository.GetByIdAsync(project.Id); Assert.NotNull(project); @@ -230,11 +234,13 @@ public async Task CanIncrementBlockedAsync() Assert.Equal(0, usage.Total); Assert.Equal(1, usage.Blocked); Assert.Equal(0, usage.TooBig); + Assert.Equal(0, usage.Deleted); overage = project.UsageHours.Single(); Assert.Equal(0, overage.Total); Assert.Equal(1, overage.Blocked); Assert.Equal(0, overage.TooBig); + Assert.Equal(0, overage.Deleted); } [Fact] @@ -257,10 +263,12 @@ public async Task CanIncrementDiscardedAsync() Assert.Equal(0, usage.Total); Assert.Equal(1, usage.Discarded); Assert.Equal(0, usage.TooBig); + Assert.Equal(0, usage.Deleted); var overage = organization.UsageHours.Single(); Assert.Equal(0, overage.Total); Assert.Equal(1, overage.Discarded); Assert.Equal(0, overage.TooBig); + Assert.Equal(0, overage.Deleted); project = await _projectRepository.GetByIdAsync(project.Id); Assert.NotNull(project); @@ -270,11 +278,13 @@ public async Task CanIncrementDiscardedAsync() Assert.Equal(0, usage.Total); Assert.Equal(1, usage.Discarded); Assert.Equal(0, usage.TooBig); + Assert.Equal(0, usage.Deleted); overage = project.UsageHours.Single(); Assert.Equal(0, overage.Total); Assert.Equal(1, overage.Discarded); Assert.Equal(0, overage.TooBig); + Assert.Equal(0, overage.Deleted); } [Fact] @@ -297,6 +307,7 @@ public async Task CanIncrementTooBigAsync() Assert.Equal(0, usage.Total); Assert.Equal(0, usage.Blocked); Assert.Equal(1, usage.TooBig); + Assert.Equal(0, usage.Deleted); project = await _projectRepository.GetByIdAsync(project.Id); Assert.NotNull(project); @@ -305,6 +316,141 @@ public async Task CanIncrementTooBigAsync() Assert.Equal(0, usage.Total); Assert.Equal(0, usage.Blocked); Assert.Equal(1, usage.TooBig); + Assert.Equal(0, usage.Deleted); + } + + [Fact] + public async Task IncrementDeletedAsync_WithProjectId_PersistsOrgAndProjectUsage() + { + // Arrange + var organization = await _organizationRepository.AddAsync(new Organization { Name = "Test", MaxEventsPerMonth = 750, PlanId = _plans.SmallPlan.Id }, o => o.ImmediateConsistency().Cache()); + var project = await _projectRepository.AddAsync(new Project { Name = "Test", OrganizationId = organization.Id, NextSummaryEndOfDayTicks = TimeProvider.GetUtcNow().UtcDateTime.Ticks }, o => o.ImmediateConsistency().Cache()); + + // Act + await _usageService.IncrementDeletedAsync(organization.Id, project.Id, 5); + TimeProvider.Advance(TimeSpan.FromMinutes(10)); + await _usageService.SavePendingUsageAsync(); + + // Assert + organization = await _organizationRepository.GetByIdAsync(organization.Id); + Assert.NotNull(organization); + Assert.Single(organization.UsageHours); + var usage = organization.Usage.Single(); + Assert.Equal(organization.MaxEventsPerMonth, usage.Limit); + Assert.Equal(0, usage.Total); + Assert.Equal(0, usage.Blocked); + Assert.Equal(0, usage.TooBig); + Assert.Equal(0, usage.Discarded); + Assert.Equal(5, usage.Deleted); + var hourUsage = organization.UsageHours.Single(); + Assert.Equal(0, hourUsage.Total); + Assert.Equal(0, hourUsage.Blocked); + Assert.Equal(0, hourUsage.TooBig); + Assert.Equal(0, hourUsage.Discarded); + Assert.Equal(5, hourUsage.Deleted); + + project = await _projectRepository.GetByIdAsync(project.Id); + Assert.NotNull(project); + Assert.Single(project.UsageHours); + usage = project.Usage.Single(); + Assert.Equal(0, usage.Total); + Assert.Equal(0, usage.Blocked); + Assert.Equal(0, usage.TooBig); + Assert.Equal(0, usage.Discarded); + Assert.Equal(5, usage.Deleted); + hourUsage = project.UsageHours.Single(); + Assert.Equal(0, hourUsage.Total); + Assert.Equal(0, hourUsage.Blocked); + Assert.Equal(0, hourUsage.TooBig); + Assert.Equal(0, hourUsage.Discarded); + Assert.Equal(5, hourUsage.Deleted); + } + + [Fact] + public async Task IncrementDeletedAsync_WithoutProjectId_OnlyIncrementsOrgUsage() + { + // Arrange + var organization = await _organizationRepository.AddAsync(new Organization { Name = "Test", MaxEventsPerMonth = 750, PlanId = _plans.SmallPlan.Id }, o => o.ImmediateConsistency().Cache()); + var project = await _projectRepository.AddAsync(new Project { Name = "Test", OrganizationId = organization.Id, NextSummaryEndOfDayTicks = TimeProvider.GetUtcNow().UtcDateTime.Ticks }, o => o.ImmediateConsistency().Cache()); + + // Act + await _usageService.IncrementDeletedAsync(organization.Id, null, 10); + TimeProvider.Advance(TimeSpan.FromMinutes(10)); + await _usageService.SavePendingUsageAsync(); + + // Assert + organization = await _organizationRepository.GetByIdAsync(organization.Id); + Assert.NotNull(organization); + Assert.Single(organization.UsageHours); + var usage = organization.Usage.Single(); + Assert.Equal(10, usage.Deleted); + Assert.Equal(0, usage.Total); + + project = await _projectRepository.GetByIdAsync(project.Id); + Assert.NotNull(project); + Assert.Empty(project.Usage); + } + + [Fact] + public async Task IncrementDeletedAsync_LargeCount_DoesNotReduceEventsLeft() + { + // Arrange + var organization = await _organizationRepository.AddAsync(new Organization { Name = "Test", MaxEventsPerMonth = 750, PlanId = _plans.SmallPlan.Id }, o => o.ImmediateConsistency().Cache()); + var project = await _projectRepository.AddAsync(new Project { Name = "Test", OrganizationId = organization.Id, NextSummaryEndOfDayTicks = TimeProvider.GetUtcNow().UtcDateTime.Ticks }, o => o.ImmediateConsistency().Cache()); + + int eventsLeftBefore = await _usageService.GetEventsLeftAsync(organization.Id); + + // Act + await _usageService.IncrementDeletedAsync(organization.Id, project.Id, 100); + + // Assert + int eventsLeftAfter = await _usageService.GetEventsLeftAsync(organization.Id); + Assert.Equal(eventsLeftBefore, eventsLeftAfter); + } + + [Fact] + public async Task IncrementDeletedAsync_MultipleCalls_AccumulatesCorrectly() + { + // Arrange + var organization = await _organizationRepository.AddAsync(new Organization { Name = "Test", MaxEventsPerMonth = 750, PlanId = _plans.SmallPlan.Id }, o => o.ImmediateConsistency().Cache()); + var project = await _projectRepository.AddAsync(new Project { Name = "Test", OrganizationId = organization.Id, NextSummaryEndOfDayTicks = TimeProvider.GetUtcNow().UtcDateTime.Ticks }, o => o.ImmediateConsistency().Cache()); + + // Act + await _usageService.IncrementDeletedAsync(organization.Id, project.Id, 3); + await _usageService.IncrementDeletedAsync(organization.Id, project.Id, 7); + TimeProvider.Advance(TimeSpan.FromMinutes(10)); + await _usageService.SavePendingUsageAsync(); + + // Assert + organization = await _organizationRepository.GetByIdAsync(organization.Id); + Assert.NotNull(organization); + var usage = organization.Usage.Single(); + Assert.Equal(10, usage.Deleted); + + project = await _projectRepository.GetByIdAsync(project.Id); + Assert.NotNull(project); + usage = project.Usage.Single(); + Assert.Equal(10, usage.Deleted); + } + + [Fact] + public async Task GetUsageAsync_PendingDeleted_IncludesInCurrentUsage() + { + // Arrange + var organization = await _organizationRepository.AddAsync(new Organization { Name = "Test", MaxEventsPerMonth = 750, PlanId = _plans.SmallPlan.Id }, o => o.ImmediateConsistency().Cache()); + var project = await _projectRepository.AddAsync(new Project { Name = "Test", OrganizationId = organization.Id, NextSummaryEndOfDayTicks = TimeProvider.GetUtcNow().UtcDateTime.Ticks }, o => o.ImmediateConsistency().Cache()); + + // Act + await _usageService.IncrementDeletedAsync(organization.Id, project.Id, 5); + + // Assert + var usageResponse = await _usageService.GetUsageAsync(organization.Id); + Assert.Equal(5, usageResponse.CurrentUsage.Deleted); + Assert.Equal(5, usageResponse.CurrentHourUsage.Deleted); + + var projectUsageResponse = await _usageService.GetUsageAsync(organization.Id, project.Id); + Assert.Equal(5, projectUsageResponse.CurrentUsage.Deleted); + Assert.Equal(5, projectUsageResponse.CurrentHourUsage.Deleted); } [Fact]