Only regenerate snapshot if Send/Receive works

backend/FwHeadless/FwHeadlessKernel.cs

@@ -21,13 +21,13 @@ public static IServiceCollection AddFwHeadless(this IServiceCollection services)
             .BindConfiguration("FwHeadlessConfig")
             .ValidateDataAnnotations()
             .ValidateOnStart();
-        services.AddSingleton<SyncJobStatusService>();
-        services.AddScoped<SendReceiveService>();
-        services.AddScoped<ProjectLookupService>();
+        services.AddSingleton<ISyncJobStatusService, SyncJobStatusService>();
+        services.AddScoped<ISendReceiveService, SendReceiveService>();
+        services.AddScoped<IProjectLookupService, ProjectLookupService>();
         services.AddScoped<ProjectDeletionService>();
         services.AddScoped<LogSanitizerService>();
         services.AddScoped<SafeLoggingProgress>();
-        services.AddScoped<ProjectMetadataService>();
+        services.AddScoped<IProjectMetadataService, ProjectMetadataService>();
         services
             .AddLcmCrdtClientCore()
             .AddFwDataBridge(ServiceLifetime.Scoped)

backend/FwHeadless/Media/MediaFileService.cs

@@ -1,6 +1,5 @@
 using System.Security.Cryptography;
 using FwHeadless.Services;
-using LcmCrdt;
 using LcmCrdt.MediaServer;
 using LexCore.Entities;
 using LexCore.Exceptions;
@@ -13,11 +12,11 @@
 
 namespace FwHeadless.Media;
 
-public class MediaFileService(LexBoxDbContext dbContext, IOptions<FwHeadlessConfig> config, SendReceiveService sendReceiveService)
+public class MediaFileService(LexBoxDbContext dbContext, IOptions<FwHeadlessConfig> config, ISendReceiveService sendReceiveService)
 {
     public record MediaFileSyncResult(List<MediaFile> Added, List<MediaFile> Removed);
     // TODO: This assumes FieldWorks is the source of truth, which is not true when FWL starts adding/deleting files
-    public async Task<MediaFileSyncResult> SyncMediaFiles(LcmCache cache)
+    public virtual async Task<MediaFileSyncResult> SyncMediaFiles(LcmCache cache)
     {
         var result = new MediaFileSyncResult([], []);
         var projectId = config.Value.LexboxProjectId(cache);
@@ -107,7 +106,7 @@ public string FilePath(MediaFile mediaFile)
         return Path.Join(config.Value.GetFwDataFolder(mediaFile.ProjectId), mediaFile.Filename);
     }
 
-    public async Task SyncMediaFiles(Guid projectId, LcmMediaService lcmMediaService)
+    public virtual async Task SyncMediaFiles(Guid projectId, LcmMediaService lcmMediaService)
     {
         var lcmResources = (await lcmMediaService.AllResources()).ToDictionary(r => r.Id);
         var existingDbFiles = dbContext.Files.Where(p => p.ProjectId == projectId).AsAsyncEnumerable();

backend/FwHeadless/Routes/MergeRoutes.cs

@@ -34,8 +34,8 @@ public static IEndpointConventionBuilder MapMergeRoutes(this WebApplication app)
 
     static async Task<Results<Ok, ProblemHttpResult>> ExecuteMergeRequest(
         SyncHostedService syncHostedService,
-        ProjectLookupService projectLookupService,
-        ProjectMetadataService metadataService,
+        IProjectLookupService projectLookupService,
+        IProjectMetadataService metadataService,
         ILogger<Program> logger,
         CrdtHttpSyncService crdtHttpSyncService,
         IHttpClientFactory httpClientFactory,
@@ -64,7 +64,7 @@ static async Task<Results<Ok, ProblemHttpResult>> ExecuteMergeRequest(
         }
 
         // Check if project is blocked from syncing
-        var blockInfo = await metadataService.GetSyncBlockInfoAsync(projectId);
+        var blockInfo = await metadataService.GetSyncBlockedInfoAsync(projectId);
         if (blockInfo?.IsBlocked == true)
         {
             logger.LogInformation("Project {projectId} is blocked from syncing. Reason: {Reason}", projectId, blockInfo.Reason);
@@ -79,7 +79,7 @@ static async Task<Results<Ok, ProblemHttpResult>> ExecuteMergeRequest(
 
     static async Task<Results<Ok, NotFound<string>>> SyncHarmonyProject(
         Guid projectId,
-        ProjectLookupService projectLookupService,
+        IProjectLookupService projectLookupService,
         CrdtSyncService crdtSyncService,
         IServiceProvider services,
         CancellationToken stoppingToken
@@ -104,8 +104,8 @@ CancellationToken stoppingToken
     static async Task<Results<Ok, NotFound<string>>> RegenerateProjectSnapshot(
         Guid projectId,
         CurrentProjectService projectContext,
-        ProjectLookupService projectLookupService,
-        CrdtFwdataProjectSyncService syncService,
+        IProjectLookupService projectLookupService,
+        ProjectSnapshotService projectSnapshotService,
         SnapshotAtCommitService snapshotAtCommitService,
         IOptions<FwHeadlessConfig> config,
         HttpContext context,
@@ -132,25 +132,25 @@ static async Task<Results<Ok, NotFound<string>>> RegenerateProjectSnapshot(
         var fwDataProject = config.Value.GetFwDataProject(projectId);
         if (commitId.HasValue)
         {
-            if (!await syncService.RegenerateProjectSnapshotAtCommit(snapshotAtCommitService, fwDataProject, commitId.Value, preserveAllFieldWorksCommits))
+            if (!await projectSnapshotService.RegenerateProjectSnapshotAtCommit(snapshotAtCommitService, fwDataProject, commitId.Value, preserveAllFieldWorksCommits))
             {
                 return TypedResults.NotFound($"Commit {commitId} not found");
             }
         }
         else
         {
             var miniLcmApi = context.RequestServices.GetRequiredService<IMiniLcmApi>();
-            await syncService.RegenerateProjectSnapshot(miniLcmApi, fwDataProject);
+            await projectSnapshotService.RegenerateProjectSnapshot(miniLcmApi, fwDataProject, keepBackup: true);
         }
         return TypedResults.Ok();
     }
 
     static async Task<Results<Ok<ProjectSyncStatus>, NotFound>> GetMergeStatus(
         CurrentProjectService projectContext,
-        ProjectLookupService projectLookupService,
-        SendReceiveService srService,
+        IProjectLookupService projectLookupService,
+        ISendReceiveService srService,
         IOptions<FwHeadlessConfig> config,
-        SyncJobStatusService syncJobStatusService,
+        ISyncJobStatusService syncJobStatusService,
         IServiceProvider services,
         LexBoxDbContext lexBoxDb,
         SyncHostedService syncHostedService,
@@ -199,7 +199,7 @@ static async Task<Results<Ok<ProjectSyncStatus>, NotFound>> GetMergeStatus(
 
     static async Task<SyncJobResult> AwaitSyncFinished(
         SyncHostedService syncHostedService,
-        SyncJobStatusService syncJobStatusService,
+        ISyncJobStatusService syncJobStatusService,
         CancellationToken cancellationToken,
         Guid projectId)
     {
@@ -241,8 +241,8 @@ static async Task<SyncJobResult> AwaitSyncFinished(
     }
 
     static async Task<Results<Ok, NotFound, BadRequest<string>>> BlockProject(
-        ProjectLookupService projectLookupService,
-        ProjectMetadataService metadataService,
+        IProjectLookupService projectLookupService,
+        IProjectMetadataService metadataService,
         ILogger<Program> logger,
         Guid projectId,
         string? reason = null)
@@ -273,8 +273,8 @@ static async Task<Results<Ok, NotFound, BadRequest<string>>> BlockProject(
     }
 
     static async Task<Results<Ok, NotFound, BadRequest<string>>> UnblockProject(
-        ProjectLookupService projectLookupService,
-        ProjectMetadataService metadataService,
+        IProjectLookupService projectLookupService,
+        IProjectMetadataService metadataService,
         ILogger<Program> logger,
         Guid projectId)
     {
@@ -304,8 +304,8 @@ static async Task<Results<Ok, NotFound, BadRequest<string>>> UnblockProject(
     }
 
     static async Task<Results<Ok<SyncBlockStatus>, NotFound, BadRequest<string>>> GetBlockStatus(
-        ProjectLookupService projectLookupService,
-        ProjectMetadataService metadataService,
+        IProjectLookupService projectLookupService,
+        IProjectMetadataService metadataService,
         ILogger<Program> logger,
         Guid projectId)
     {
@@ -319,7 +319,7 @@ static async Task<Results<Ok<SyncBlockStatus>, NotFound, BadRequest<string>>> Ge
             return TypedResults.NotFound();
         }
 
-        var blockInfo = await metadataService.GetSyncBlockInfoAsync(projectId);
+        var blockInfo = await metadataService.GetSyncBlockedInfoAsync(projectId);
 
         activity?.SetStatus(ActivityStatusCode.Ok, $"Block status retrieved: {(blockInfo?.IsBlocked == true ? "blocked" : "unblocked")}");
         return TypedResults.Ok(new SyncBlockStatus

backend/FwHeadless/Services/CrdtSyncService.cs

@@ -11,7 +11,7 @@ public class CrdtSyncService(
     DataModel dataModel,
     ILogger<CrdtSyncService> logger)
 {
-    public async Task SyncHarmonyProject()
+    public virtual async Task SyncHarmonyProject()
     {
         using var activity = FwHeadlessActivitySource.Value.StartActivity();
         activity?.SetTag("app.project_id", currentProjectService.ProjectData.Id);

backend/FwHeadless/Services/IProjectLookupService.cs

@@ -0,0 +1,9 @@
+namespace FwHeadless.Services;
+
+public interface IProjectLookupService
+{
+    ValueTask<string?> GetProjectCode(Guid projectId);
+    Task<Guid?> GetProjectId(string projectCode);
+    Task<bool> ProjectExists(Guid projectId);
+    Task<bool> IsCrdtProject(Guid projectId);
+}

backend/FwHeadless/Services/IProjectMetadataService.cs

@@ -0,0 +1,8 @@
+namespace FwHeadless.Services;
+
+public interface IProjectMetadataService
+{
+    Task BlockFromSyncAsync(Guid projectId, string? reason = null);
+    Task UnblockFromSyncAsync(Guid projectId);
+    Task<SyncBlockedInfo?> GetSyncBlockedInfoAsync(Guid projectId);
+}

backend/FwHeadless/Services/ISendReceiveService.cs

@@ -0,0 +1,11 @@
+using FwDataMiniLcmBridge;
+
+namespace FwHeadless.Services;
+
+public interface ISendReceiveService
+{
+    Task<SendReceiveHelpers.LfMergeBridgeResult> SendReceive(FwDataProject project, string? projectCode, string? commitMessage = null);
+    Task<SendReceiveHelpers.LfMergeBridgeResult> Clone(FwDataProject project, string? projectCode);
+    Task<int> PendingCommitCount(FwDataProject project, string? projectCode);
+    Task CommitFile(string filePath, string commitMessage);
+}

backend/FwHeadless/Services/ISyncJobStatusService.cs

@@ -0,0 +1,8 @@
+namespace FwHeadless.Services;
+
+public interface ISyncJobStatusService
+{
+    void StartSyncing(Guid projectId);
+    void StopSyncing(Guid projectId);
+    SyncJobStatus SyncStatus(Guid projectId);
+}

backend/FwHeadless/Services/ProjectContextFromIdService.cs

@@ -6,7 +6,7 @@
 namespace FwHeadless.Services;
 
 // TODO: Pick better name
-public class ProjectContextFromIdService(IOptions<FwHeadlessConfig> config, ProjectLookupService projectLookupService, ProjectDataCache projectDataCache)
+public class ProjectContextFromIdService(IOptions<FwHeadlessConfig> config, IProjectLookupService projectLookupService, ProjectDataCache projectDataCache)
 {
     public async Task PopulateProjectContext(HttpContext context, Func<Task> next)
     {

backend/FwHeadless/Services/ProjectDeletionService.cs

@@ -5,7 +5,7 @@ namespace FwHeadless.Services;
 
 public class ProjectDeletionService(
     IOptions<FwHeadlessConfig> config,
-    ProjectLookupService projectLookupService,
+    IProjectLookupService projectLookupService,
     SyncHostedService syncHostedService,
     ILogger<ProjectDeletionService> logger)
 {

backend/FwHeadless/Services/ProjectLookupService.cs

@@ -4,7 +4,7 @@
 
 namespace FwHeadless.Services;
 
-public class ProjectLookupService(LexBoxDbContext dbContext)
+public class ProjectLookupService(LexBoxDbContext dbContext) : IProjectLookupService
 {
     public virtual async ValueTask<string?> GetProjectCode(Guid projectId)
     {

backend/FwHeadless/Services/ProjectMetadataService.cs

@@ -3,29 +3,30 @@
 
 namespace FwHeadless.Services;
 
-public class ProjectMetadataService(IOptions<FwHeadlessConfig> config, ILogger<ProjectMetadataService> logger)
+public class ProjectMetadataService(IOptions<FwHeadlessConfig> config, ILogger<ProjectMetadataService> logger) : IProjectMetadataService
 {
     private readonly MetadataStore _store = new(config.Value, logger);
 
     public async Task BlockFromSyncAsync(Guid projectId, string? reason = null)
     {
+        var resolvedReason = reason ?? "Manual block";
         await _store.UpdateAsync(projectId, metadata =>
         {
-            metadata.SyncBlocked = new SyncBlockInfo
+            metadata.SyncBlocked = new SyncBlockedInfo
             {
                 IsBlocked = true,
-                Reason = reason ?? "Manual block",
+                Reason = resolvedReason,
                 BlockedAt = DateTime.UtcNow
             };
         });
-        logger.LogWarning("Project {projectId} blocked from sync. Reason: {reason}", projectId, reason);
+        logger.LogWarning("Project {projectId} blocked from sync. Reason: {reason}", projectId, resolvedReason);
     }
 
     public async Task UnblockFromSyncAsync(Guid projectId)
     {
         await _store.UpdateAsync(projectId, metadata =>
         {
-            metadata.SyncBlocked = new SyncBlockInfo
+            metadata.SyncBlocked = new SyncBlockedInfo
             {
                 IsBlocked = false,
                 BlockedAt = null,
@@ -35,7 +36,7 @@ await _store.UpdateAsync(projectId, metadata =>
         logger.LogInformation("Project {projectId} unblocked from sync", projectId);
     }
 
-    public Task<SyncBlockInfo?> GetSyncBlockInfoAsync(Guid projectId)
+    public Task<SyncBlockedInfo?> GetSyncBlockedInfoAsync(Guid projectId)
     {
         return _store.ReadAsync(projectId, metadata => metadata?.SyncBlocked);
     }
@@ -131,10 +132,10 @@ private void SaveMetadata(Guid projectId, ProjectMetadata metadata)
 
 public class ProjectMetadata
 {
-    public SyncBlockInfo? SyncBlocked { get; set; }
+    public SyncBlockedInfo? SyncBlocked { get; set; }
 }
 
-public class SyncBlockInfo
+public class SyncBlockedInfo
 {
     public bool IsBlocked { get; set; }
     public string? Reason { get; set; }

backend/FwHeadless/Services/ProjectSyncStatusService.cs

@@ -2,7 +2,7 @@
 
 namespace FwHeadless.Services;
 
-public class SyncJobStatusService()
+public class SyncJobStatusService() : ISyncJobStatusService
 {
     private ConcurrentDictionary<Guid, SyncJobStatus> Status { get; init; } = new();
 

backend/FwHeadless/Services/SendReceiveHelpers.cs

@@ -4,7 +4,11 @@
 
 namespace FwHeadless.Services;
 
-public class SendReceiveException(string? message) : Exception(message);
+public class SendReceiveException(string message, SendReceiveHelpers.LfMergeBridgeResult result)
+    : Exception($"{message}. Output: {result.Output}")
+{
+    public SendReceiveHelpers.LfMergeBridgeResult Result { get; } = result;
+}
 
 public static class SendReceiveHelpers
 {
@@ -24,6 +28,18 @@ public record LfMergeBridgeResult(string Output)
         private readonly IProgress? _progress = null;
         public bool ErrorEncountered => _progress?.ErrorEncountered ?? false;
 
+        /// <summary>
+        /// Substring emitted by Chorus when it has decided to rollback the operation.
+        /// We key off this because failures with rollback are special: it implies the local working
+        /// copy may be in a bad/unstable state and we should block further syncing.
+        ///
+        /// Chorus reference:
+        /// https://github.com/sillsdev/chorus/blob/master/src/LibChorus/sync/Synchronizer.cs#L651
+        /// </summary>
+        public const string RollbackIndicator = "Rolling back...";
+        public bool RollbackDetected => !string.IsNullOrEmpty(Output)
+                && Output.Contains(RollbackIndicator, StringComparison.Ordinal);
+
         /// <summary>
         /// This string in the output unambiguously indicates that the operation ultimately succeeded.
         /// </summary>

backend/FwHeadless/Services/SendReceiveService.cs

@@ -3,7 +3,7 @@
 
 namespace FwHeadless.Services;
 
-public class SendReceiveService(IOptions<FwHeadlessConfig> config, SafeLoggingProgress progress)
+public class SendReceiveService(IOptions<FwHeadlessConfig> config, SafeLoggingProgress progress) : ISendReceiveService
 {
     public async Task<SendReceiveHelpers.LfMergeBridgeResult> SendReceive(FwDataProject project, string? projectCode, string? commitMessage = null)
     {