ENG-1791 Obsidian sync does not recover from Concept upsert failure

apps/obsidian/src/utils/publishNode.ts

@@ -17,6 +17,7 @@ import {
   syncPublishedNodeAssets,
 } from "./syncDgNodesToSupabase";
 import { isProvisionalSchema } from "./typeUtils";
+import { intersection, difference } from "@repo/utils/setOperations";
 
 import type { DiscourseNodeInVault } from "./getDiscourseNodes";
 import type { SupabaseContext } from "./supabaseContext";
@@ -68,31 +69,6 @@ const publishSchema = async ({
   }
 };
 
-const intersection = <T>(set1: Set<T>, set2: Set<T>): Set<T> => {
-  // @ts-expect-error - Set.intersection is ES2025 feature
-  if (set1.intersection) return set1.intersection(set2); // eslint-disable-line
-  const r: Set<T> = new Set();
-  for (const x of set1) {
-    if (set2.has(x)) r.add(x);
-  }
-  return r;
-};
-
-const difference = <T>(set1: Set<T>, set2: Set<T>): Set<T> => {
-  // @ts-expect-error - Set.difference is ES2025 feature
-  if (set1.difference) return set1.difference(set2); // eslint-disable-line
-  const result = new Set(set1);
-  if (set1.size <= set2.size)
-    for (const e of set1) {
-      if (set2.has(e)) result.delete(e);
-    }
-  else
-    for (const e of set2) {
-      if (result.has(e)) result.delete(e);
-    }
-  return result;
-};
-
 export const publishNewRelation = async (
   plugin: DiscourseGraphPlugin,
   relation: RelationInstance,

apps/obsidian/src/utils/syncDgNodesToSupabase.ts

@@ -28,6 +28,7 @@ import {
 } from "./getDiscourseNodes";
 import { isAcceptedSchema } from "./typeUtils";
 import { getTemplatePluginInfo } from "./templates";
+import { difference } from "@repo/utils/setOperations";
 
 const DEFAULT_TIME = "1970-01-01";
 export type ChangeType = "title" | "content";
@@ -226,6 +227,7 @@ type BuildChangedNodesOptions = {
   supabaseClient: DGSupabaseClient;
   context: SupabaseContext;
   changeTypesByPath?: Map<string, ChangeType[]>;
+  fullSync?: boolean;
 };
 
 const mergeChangeTypes = (
@@ -322,6 +324,7 @@ const buildChangedNodesFromNodes = async ({
   supabaseClient,
   context,
   changeTypesByPath,
+  fullSync = false,
 }: BuildChangedNodesOptions): Promise<ObsidianDiscourseNodeData[]> => {
   if (nodes.length === 0) {
     return [];
@@ -339,6 +342,25 @@ const buildChangedNodesFromNodes = async ({
     context.spaceId,
   );
   const changedNodes: ObsidianDiscourseNodeData[] = [];
+  let missing: Set<string> | undefined;
+  if (fullSync) {
+    const existingConceptIds = await supabaseClient
+      .from("my_concepts")
+      .select("source_local_id")
+      .eq("space_id", context.spaceId)
+      .eq("arity", 0)
+      .eq("is_schema", false);
+    if (existingConceptIds.data) {
+      // fail silently otherwise, there will be other opportunities
+      const nodeIds = new Set(nodes.map((n) => n.nodeInstanceId));
+      const dbConceptIds = new Set(
+        existingConceptIds.data
+          .map((d) => d.source_local_id)
+          .filter((id) => id !== null),
+      );
+      missing = difference(nodeIds, dbConceptIds);
+    }
+  }
 
   for (const node of nodes) {
     if (node.frontmatter.importedFromRid) continue;
@@ -355,7 +377,7 @@ const buildChangedNodesFromNodes = async ({
         : detectedChangeTypes;
     const finalChangeTypes = mergedChangeTypes;
 
-    if (finalChangeTypes.length === 0) {
+    if (finalChangeTypes.length === 0 && !missing?.has(node.nodeInstanceId)) {
       continue;
     }
 
@@ -397,6 +419,7 @@ export const syncAllNodesAndRelations = async (
           nodes: allNodes,
           supabaseClient,
           context,
+          fullSync: true,
         });
 
     const accountLocalId = plugin.settings.accountLocalId;

packages/utils/src/setOperations.ts

@@ -0,0 +1,24 @@
+export const intersection = <T>(set1: Set<T>, set2: Set<T>): Set<T> => {
+  // @ts-expect-error - Set.intersection is ES2025 feature
+  if (set1.intersection) return set1.intersection(set2); // eslint-disable-line
+  const r: Set<T> = new Set();
+  for (const x of set1) {
+    if (set2.has(x)) r.add(x);
+  }
+  return r;
+};
+
+export const difference = <T>(set1: Set<T>, set2: Set<T>): Set<T> => {
+  // @ts-expect-error - Set.difference is ES2025 feature
+  if (set1.difference) return set1.difference(set2); // eslint-disable-line
+  const result = new Set(set1);
+  if (set1.size <= set2.size)
+    for (const e of set1) {
+      if (set2.has(e)) result.delete(e);
+    }
+  else
+    for (const e of set2) {
+      if (result.has(e)) result.delete(e);
+    }
+  return result;
+};