Fix predefined saved view synchronization

src/Exceptionless.Core/Bootstrapper.cs

@@ -87,6 +87,7 @@ public static void RegisterServices(IServiceCollection services, AppOptions appO
         {
             var handlers = new WorkItemHandlers();
             handlers.Register<FixStackStatsWorkItem>(s.GetRequiredService<FixStackStatsWorkItemHandler>);
+            handlers.Register<ForcePredefinedSavedViewsWorkItem>(s.GetRequiredService<ForcePredefinedSavedViewsWorkItemHandler>);
             handlers.Register<OrganizationMaintenanceWorkItem>(s.GetRequiredService<OrganizationMaintenanceWorkItemHandler>);
             handlers.Register<OrganizationNotificationWorkItem>(s.GetRequiredService<OrganizationNotificationWorkItemHandler>);
             handlers.Register<ProjectMaintenanceWorkItem>(s.GetRequiredService<ProjectMaintenanceWorkItemHandler>);

src/Exceptionless.Core/Jobs/WorkItemHandlers/ForcePredefinedSavedViewsWorkItemHandler.cs

@@ -0,0 +1,182 @@
+using System.Text.RegularExpressions;
+using Exceptionless.Core.Models;
+using Exceptionless.Core.Models.WorkItems;
+using Exceptionless.Core.Repositories;
+using Exceptionless.Core.Seed;
+using Foundatio.Jobs;
+using Foundatio.Lock;
+using Foundatio.Repositories;
+using Microsoft.Extensions.Logging;
+
+namespace Exceptionless.Core.Jobs.WorkItemHandlers;
+
+public class ForcePredefinedSavedViewsWorkItemHandler : WorkItemHandlerBase
+{
+    private const int SavedViewPageLimit = 1000;
+    private static readonly string[] ValidViewTypes = ["events", "stacks", "stream"];
+
+    private readonly ISavedViewRepository _savedViewRepository;
+    private readonly ILockProvider _lockProvider;
+
+    public ForcePredefinedSavedViewsWorkItemHandler(
+        ISavedViewRepository savedViewRepository,
+        ILockProvider lockProvider,
+        ILoggerFactory loggerFactory) : base(loggerFactory)
+    {
+        _savedViewRepository = savedViewRepository;
+        _lockProvider = lockProvider;
+    }
+
+    public override Task<ILock?> GetWorkItemLockAsync(object workItem, CancellationToken cancellationToken = default)
+    {
+        return _lockProvider.TryAcquireAsync(nameof(ForcePredefinedSavedViewsWorkItemHandler), TimeSpan.FromHours(1), cancellationToken);
+    }
+
+    public override async Task HandleItemAsync(WorkItemContext context)
+    {
+        var workItem = context.GetData<ForcePredefinedSavedViewsWorkItem>()!;
+        var predefinedSavedViewsByKey = await GetPredefinedSavedViewsByKeyAsync();
+
+        Log.LogInformation(
+            "Starting predefined saved view force update. Definitions: {DefinitionCount} InitiatedByUserId: {UserId}",
+            predefinedSavedViewsByKey.Count,
+            workItem.UserId);
+
+        int organizationsUpdated = 0;
+        int savedViewsUpdated = 0;
+        var savedViews = await _savedViewRepository.GetPredefinedForForceUpdateAsync(
+            PredefinedSavedViewsDataSeed.SystemOrganizationId,
+            predefinedSavedViewsByKey.Keys.ToArray(),
+            o => o.SearchAfterPaging().PageLimit(SavedViewPageLimit));
+
+        do
+        {
+            foreach (var savedViewsByOrganization in savedViews.Documents.GroupBy(savedView => savedView.OrganizationId))
+            {
+                if (context.CancellationToken.IsCancellationRequested)
+                    break;
+
+                var updateResult = await UpdateOrganizationSavedViewsAsync(
+                    savedViewsByOrganization.Key,
+                    savedViewsByOrganization.Select(savedView => savedView.Id),
+                    workItem.UserId,
+                    predefinedSavedViewsByKey);
+
+                int updatedForOrganization = updateResult;
+                if (updatedForOrganization > 0)
+                {
+                    organizationsUpdated++;
+                    savedViewsUpdated += updatedForOrganization;
+                }
+
+                await context.RenewLockAsync();
+            }
+        } while (!context.CancellationToken.IsCancellationRequested && await savedViews.NextPageAsync());
+
+        Log.LogInformation(
+            "Completed predefined saved view force update. OrganizationsUpdated: {OrganizationsUpdated} SavedViewsUpdated: {SavedViewsUpdated} InitiatedByUserId: {UserId}",
+            organizationsUpdated,
+            savedViewsUpdated,
+            workItem.UserId);
+    }
+
+    private async Task<Dictionary<string, SavedView>> GetPredefinedSavedViewsByKeyAsync()
+    {
+        var savedViewsByKey = new Dictionary<string, SavedView>(StringComparer.OrdinalIgnoreCase);
+
+        foreach (string viewType in ValidViewTypes)
+        {
+            var results = await _savedViewRepository.GetByViewAsync(PredefinedSavedViewsDataSeed.SystemOrganizationId, viewType, o => o.PageLimit(SavedViewPageLimit));
+            foreach (var savedView in results.Documents.Where(view => view.UserId is null && !String.IsNullOrWhiteSpace(view.PredefinedKey)))
+            {
+                if (!savedViewsByKey.TryAdd(savedView.PredefinedKey!, savedView))
+                    throw new InvalidOperationException($"Predefined saved view key '{savedView.PredefinedKey}' is duplicated.");
+            }
+        }
+
+        return savedViewsByKey;
+    }
+
+    private async Task<int> UpdateOrganizationSavedViewsAsync(
+        string organizationId,
+        IEnumerable<string> savedViewIds,
+        string userId,
+        IReadOnlyDictionary<string, SavedView> predefinedSavedViewsByKey)
+    {
+        int updatedSavedViews = 0;
+        bool lockAcquired = await _lockProvider.TryUsingAsync($"predefined-saved-views:{organizationId}", async () =>
+        {
+            updatedSavedViews = await UpdateOrganizationSavedViewsWithLockAsync(organizationId, savedViewIds, userId, predefinedSavedViewsByKey);
+        }, TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(15));
+
+        if (!lockAcquired)
+            Log.LogWarning("Skipping force update for organization {OrganizationId} because the saved view synchronization lock could not be acquired", organizationId);
+
+        return updatedSavedViews;
+    }
+
+    private async Task<int> UpdateOrganizationSavedViewsWithLockAsync(
+        string organizationId,
+        IEnumerable<string> savedViewIds,
+        string userId,
+        IReadOnlyDictionary<string, SavedView> predefinedSavedViewsByKey)
+    {
+        var savedViewsToSave = new List<SavedView>();
+        var savedViewIdsToUpdate = savedViewIds.ToHashSet(StringComparer.Ordinal);
+        var results = await _savedViewRepository.FindAsync(q => q.Organization(organizationId), o => o.PageLimit(SavedViewPageLimit));
+        var savedViewsByViewType = results.Documents.GroupBy(savedView => savedView.ViewType, StringComparer.OrdinalIgnoreCase)
+            .ToDictionary(group => group.Key, group => group.ToList(), StringComparer.OrdinalIgnoreCase);
+
+        foreach (var savedViews in savedViewsByViewType.Values)
+        {
+            foreach (var savedView in savedViews.Where(savedView => savedViewIdsToUpdate.Contains(savedView.Id)))
+            {
+                if (savedView.UserId is not null
+                    || String.IsNullOrWhiteSpace(savedView.PredefinedKey)
+                    || !predefinedSavedViewsByKey.TryGetValue(savedView.PredefinedKey, out var predefinedSavedView))
+                {
+                    continue;
+                }
+
+                string slug = GetUniqueSlug(predefinedSavedView.Slug, savedViews, savedView.Id);
+                if (PredefinedSavedViewConfiguration.Apply(savedView, predefinedSavedView, predefinedSavedView.PredefinedKey!, slug))
+                {
+                    savedView.UpdatedByUserId = userId;
+                    savedViewsToSave.Add(savedView);
+                }
+            }
+        }
+
+        if (savedViewsToSave.Count > 0)
+            await _savedViewRepository.SaveAsync(savedViewsToSave, o => o.Cache());
+
+        return savedViewsToSave.Count;
+    }
+
+    private static string GetUniqueSlug(string slug, IReadOnlyCollection<SavedView> existingViews, string? excludingId)
+    {
+        string baseSlug = ToSlug(slug);
+        if (String.IsNullOrWhiteSpace(baseSlug))
+            baseSlug = "saved-view";
+
+        baseSlug = baseSlug.Length > 100 ? baseSlug[..100].Trim('-') : baseSlug;
+        string candidate = baseSlug;
+        int suffix = 2;
+
+        while (existingViews.Any(view => view.Id != excludingId && String.Equals(view.Slug, candidate, StringComparison.OrdinalIgnoreCase)))
+        {
+            string suffixText = $"-{suffix}";
+            int maxBaseLength = 100 - suffixText.Length;
+            candidate = $"{baseSlug[..Math.Min(baseSlug.Length, maxBaseLength)].Trim('-')}{suffixText}";
+            suffix++;
+        }
+
+        return candidate;
+    }
+
+    private static string ToSlug(string value)
+    {
+        string slug = Regex.Replace(value.Trim().ToLowerInvariant(), "[^a-z0-9]+", "-").Trim('-');
+        return Regex.Replace(slug, "-+", "-");
+    }
+}

src/Exceptionless.Core/Models/SavedView.cs

@@ -61,6 +61,10 @@ public partial record SavedView : IOwnedByOrganizationWithIdentity, IHaveDates
     [MaxLength(150)]
     public string? PredefinedKey { get; set; }
 
+    /// <summary>Content hash of the predefined configuration last applied to this view.</summary>
+    [MaxLength(64)]
+    public string? PredefinedContentHash { get; set; }
+
     /// <summary>Display name shown in the sidebar and picker.</summary>
     [Required]
     [MaxLength(100)]

src/Exceptionless.Core/Models/WorkItems/ForcePredefinedSavedViewsWorkItem.cs

@@ -0,0 +1,12 @@
+using Foundatio.Queues;
+
+namespace Exceptionless.Core.Models.WorkItems;
+
+public record ForcePredefinedSavedViewsWorkItem : IHaveUniqueIdentifier
+{
+    public required string UserId { get; init; }
+
+    public Guid RunId { get; init; } = Guid.NewGuid();
+
+    public string UniqueIdentifier => $"{nameof(ForcePredefinedSavedViewsWorkItem)}:{RunId:N}";
+}

src/Exceptionless.Core/Repositories/Configuration/Indexes/SavedViewIndex.cs

@@ -12,7 +12,7 @@ public sealed class SavedViewIndex : VersionedIndex<Models.SavedView>
 
     private readonly ExceptionlessElasticConfiguration _configuration;
 
-    public SavedViewIndex(ExceptionlessElasticConfiguration configuration) : base(configuration, configuration.Options.ScopePrefix + "saved-views", 1)
+    public SavedViewIndex(ExceptionlessElasticConfiguration configuration) : base(configuration, configuration.Options.ScopePrefix + "saved-views", 3)
     {
         _configuration = configuration;
     }
@@ -27,6 +27,7 @@ public override void ConfigureIndexMapping(TypeMappingDescriptor<Models.SavedVie
                 .Keyword(e => e.UserId)
                 .Keyword(e => e.CreatedByUserId)
                 .Keyword(e => e.UpdatedByUserId)
+                .Text(e => e.PredefinedKey, t => t.Analyzer(KEYWORD_LOWERCASE_ANALYZER))
                 .Text(e => e.Name, t => t.Analyzer(KEYWORD_LOWERCASE_ANALYZER).AddKeywordField())
                 .Keyword(e => e.ViewType)
                 .IntegerNumber(e => e.Version));

src/Exceptionless.Core/Repositories/Interfaces/ISavedViewRepository.cs

@@ -9,6 +9,7 @@ public interface ISavedViewRepository : IRepositoryOwnedByOrganization<SavedView
     Task<FindResults<SavedView>> GetByViewAsync(string organizationId, string viewType, CommandOptionsDescriptor<SavedView>? options = null);
     Task<FindResults<SavedView>> GetByViewForUserAsync(string organizationId, string viewType, string userId, CommandOptionsDescriptor<SavedView>? options = null);
     Task<FindResults<SavedView>> GetByOrganizationForUserAsync(string organizationId, string userId, CommandOptionsDescriptor<SavedView>? options = null);
+    Task<FindResults<SavedView>> GetPredefinedForForceUpdateAsync(string systemOrganizationId, IReadOnlyCollection<string> predefinedKeys, CommandOptionsDescriptor<SavedView>? options = null);
     Task<long> CountByOrganizationIdAsync(string organizationId);
 
     /// <summary>Removes all private saved views belonging to a specific user within an organization.</summary>

src/Exceptionless.Core/Repositories/SavedViewRepository.cs

@@ -42,6 +42,31 @@ public Task<FindResults<SavedView>> GetByOrganizationForUserAsync(string organiz
                 .FieldEquals(e => e.UserId!, userId)), options);
     }
 
+    public Task<FindResults<SavedView>> GetPredefinedForForceUpdateAsync(
+        string systemOrganizationId,
+        IReadOnlyCollection<string> predefinedKeys,
+        CommandOptionsDescriptor<SavedView>? options = null)
+    {
+        ArgumentException.ThrowIfNullOrEmpty(systemOrganizationId);
+        ArgumentNullException.ThrowIfNull(predefinedKeys);
+
+        if (predefinedKeys.Count == 0)
+            return FindAsync(q => q.FieldEquals(e => e.Id, String.Empty), options);
+
+        return FindAsync(q =>
+        {
+            q.FieldEmpty(e => e.UserId!)
+                .FieldNotEquals(e => e.OrganizationId, systemOrganizationId)
+                .FieldOr(g =>
+                {
+                    foreach (string predefinedKey in predefinedKeys)
+                        g.FieldContains(e => e.PredefinedKey!, predefinedKey);
+                });
+
+            return q.SortAscending(e => e.OrganizationId).SortAscending(e => e.Id);
+        }, options);
+    }
+
     public async Task<long> RemovePrivateByUserIdAsync(string organizationId, string userId)
     {
         var results = await FindAsync(q => q

src/Exceptionless.Core/Seed/PredefinedSavedViewConfiguration.cs

@@ -0,0 +1,77 @@
+using Exceptionless.Core.Models;
+
+namespace Exceptionless.Core.Seed;
+
+public static class PredefinedSavedViewConfiguration
+{
+    public static bool Apply(SavedView destination, SavedView source, string key, string slug)
+    {
+        bool changed = false;
+        changed |= SetIfChanged(destination, null, static (view, value) => view.UserId = value, static view => view.UserId);
+        changed |= SetIfChanged(destination, key, static (view, value) => view.PredefinedKey = value, static view => view.PredefinedKey);
+        changed |= SetIfChanged(destination, source.Name, static (view, value) => view.Name = value, static view => view.Name);
+        changed |= SetIfChanged(destination, slug, static (view, value) => view.Slug = value, static view => view.Slug);
+        changed |= SetIfChanged(destination, source.ViewType, static (view, value) => view.ViewType = value, static view => view.ViewType);
+        changed |= SetIfChanged(destination, source.Filter, static (view, value) => view.Filter = value, static view => view.Filter);
+        changed |= SetIfChanged(destination, source.Time, static (view, value) => view.Time = value, static view => view.Time);
+        changed |= SetIfChanged(destination, source.Sort, static (view, value) => view.Sort = value, static view => view.Sort);
+        changed |= SetIfChanged(destination, source.FilterDefinitions, static (view, value) => view.FilterDefinitions = value, static view => view.FilterDefinitions);
+        changed |= SetDictionaryIfChanged(destination, source.Columns);
+        changed |= SetListIfChanged(destination, source.ColumnOrder);
+        changed |= SetIfChanged(destination, source.ShowStats, static (view, value) => view.ShowStats = value, static view => view.ShowStats);
+        changed |= SetIfChanged(destination, source.ShowChart, static (view, value) => view.ShowChart = value, static view => view.ShowChart);
+        changed |= SetIfChanged(destination, source.Version, static (view, value) => view.Version = value, static view => view.Version);
+        changed |= SetIfChanged(destination, PredefinedSavedViewContentHasher.GetContentHash(destination), static (view, value) => view.PredefinedContentHash = value, static view => view.PredefinedContentHash);
+
+        return changed;
+    }
+
+    private static bool SetDictionaryIfChanged(SavedView savedView, IReadOnlyDictionary<string, bool>? value)
+    {
+        if (DictionaryEquals(savedView.Columns, value))
+            return false;
+
+        savedView.Columns = value is null ? null : new Dictionary<string, bool>(value);
+        return true;
+    }
+
+    private static bool SetIfChanged<T>(SavedView savedView, T value, Action<SavedView, T> setter, Func<SavedView, T> getter)
+    {
+        if (EqualityComparer<T>.Default.Equals(getter(savedView), value))
+            return false;
+
+        setter(savedView, value);
+        return true;
+    }
+
+    private static bool SetListIfChanged(SavedView savedView, IReadOnlyCollection<string>? value)
+    {
+        if (CollectionEquals(savedView.ColumnOrder, value))
+            return false;
+
+        savedView.ColumnOrder = value is null ? null : [.. value];
+        return true;
+    }
+
+    private static bool CollectionEquals(IReadOnlyCollection<string>? left, IReadOnlyCollection<string>? right)
+    {
+        if (ReferenceEquals(left, right))
+            return true;
+
+        if (left is null || right is null || left.Count != right.Count)
+            return false;
+
+        return left.SequenceEqual(right, StringComparer.Ordinal);
+    }
+
+    private static bool DictionaryEquals(IReadOnlyDictionary<string, bool>? left, IReadOnlyDictionary<string, bool>? right)
+    {
+        if (ReferenceEquals(left, right))
+            return true;
+
+        if (left is null || right is null || left.Count != right.Count)
+            return false;
+
+        return left.All(kvp => right.TryGetValue(kvp.Key, out bool value) && value == kvp.Value);
+    }
+}