From bdd1c7e831954cfdf05e57499aae4f1d851119d7 Mon Sep 17 00:00:00 2001 From: Marc-Antoine Parent Date: Fri, 22 May 2026 18:10:19 -0400 Subject: [PATCH 1/2] ENG-1791 check missing concepts at full sync --- apps/obsidian/src/utils/publishNode.ts | 26 +------------------ apps/obsidian/src/utils/setOperations.ts | 24 +++++++++++++++++ .../src/utils/syncDgNodesToSupabase.ts | 25 +++++++++++++++++- 3 files changed, 49 insertions(+), 26 deletions(-) create mode 100644 apps/obsidian/src/utils/setOperations.ts diff --git a/apps/obsidian/src/utils/publishNode.ts b/apps/obsidian/src/utils/publishNode.ts index 6d9f6fa88..7952ea1a2 100644 --- a/apps/obsidian/src/utils/publishNode.ts +++ b/apps/obsidian/src/utils/publishNode.ts @@ -17,6 +17,7 @@ import { syncPublishedNodeAssets, } from "./syncDgNodesToSupabase"; import { isProvisionalSchema } from "./typeUtils"; +import { intersection, difference } from "./setOperations"; import type { DiscourseNodeInVault } from "./getDiscourseNodes"; import type { SupabaseContext } from "./supabaseContext"; @@ -68,31 +69,6 @@ const publishSchema = async ({ } }; -const intersection = (set1: Set, set2: Set): Set => { - // @ts-expect-error - Set.intersection is ES2025 feature - if (set1.intersection) return set1.intersection(set2); // eslint-disable-line - const r: Set = new Set(); - for (const x of set1) { - if (set2.has(x)) r.add(x); - } - return r; -}; - -const difference = (set1: Set, set2: Set): Set => { - // @ts-expect-error - Set.difference is ES2025 feature - if (set1.difference) return set1.difference(set2); // eslint-disable-line - const result = new Set(set1); - if (set1.size <= set2.size) - for (const e of set1) { - if (set2.has(e)) result.delete(e); - } - else - for (const e of set2) { - if (result.has(e)) result.delete(e); - } - return result; -}; - export const publishNewRelation = async ( plugin: DiscourseGraphPlugin, relation: RelationInstance, diff --git a/apps/obsidian/src/utils/setOperations.ts b/apps/obsidian/src/utils/setOperations.ts new file mode 100644 index 000000000..a228b5d7b --- /dev/null +++ b/apps/obsidian/src/utils/setOperations.ts @@ -0,0 +1,24 @@ +export const intersection = (set1: Set, set2: Set): Set => { + // @ts-expect-error - Set.intersection is ES2025 feature + if (set1.intersection) return set1.intersection(set2); // eslint-disable-line + const r: Set = new Set(); + for (const x of set1) { + if (set2.has(x)) r.add(x); + } + return r; +}; + +export const difference = (set1: Set, set2: Set): Set => { + // @ts-expect-error - Set.difference is ES2025 feature + if (set1.difference) return set1.difference(set2); // eslint-disable-line + const result = new Set(set1); + if (set1.size <= set2.size) + for (const e of set1) { + if (set2.has(e)) result.delete(e); + } + else + for (const e of set2) { + if (result.has(e)) result.delete(e); + } + return result; +}; diff --git a/apps/obsidian/src/utils/syncDgNodesToSupabase.ts b/apps/obsidian/src/utils/syncDgNodesToSupabase.ts index e0f75bd00..748d7b422 100644 --- a/apps/obsidian/src/utils/syncDgNodesToSupabase.ts +++ b/apps/obsidian/src/utils/syncDgNodesToSupabase.ts @@ -28,6 +28,7 @@ import { } from "./getDiscourseNodes"; import { isAcceptedSchema } from "./typeUtils"; import { getTemplatePluginInfo } from "./templates"; +import { difference } from "./setOperations"; const DEFAULT_TIME = "1970-01-01"; export type ChangeType = "title" | "content"; @@ -226,6 +227,7 @@ type BuildChangedNodesOptions = { supabaseClient: DGSupabaseClient; context: SupabaseContext; changeTypesByPath?: Map; + fullSync?: boolean; }; const mergeChangeTypes = ( @@ -322,6 +324,7 @@ const buildChangedNodesFromNodes = async ({ supabaseClient, context, changeTypesByPath, + fullSync = false, }: BuildChangedNodesOptions): Promise => { if (nodes.length === 0) { return []; @@ -339,6 +342,25 @@ const buildChangedNodesFromNodes = async ({ context.spaceId, ); const changedNodes: ObsidianDiscourseNodeData[] = []; + let missing: Set | undefined; + if (fullSync) { + const existingConceptIds = await supabaseClient + .from("my_concepts") + .select("source_local_id") + .eq("space_id", context.spaceId) + .eq("arity", 0) + .eq("is_schema", false); + if (existingConceptIds.data) { + // fail silently otherwise, there will be other opportunities + const nodeIds = new Set(nodes.map((n) => n.nodeInstanceId)); + const dbConceptIds = new Set( + existingConceptIds.data + .map((d) => d.source_local_id) + .filter((id) => id !== null), + ); + missing = difference(nodeIds, dbConceptIds); + } + } for (const node of nodes) { if (node.frontmatter.importedFromRid) continue; @@ -355,7 +377,7 @@ const buildChangedNodesFromNodes = async ({ : detectedChangeTypes; const finalChangeTypes = mergedChangeTypes; - if (finalChangeTypes.length === 0) { + if (finalChangeTypes.length === 0 && !missing?.has(node.nodeInstanceId)) { continue; } @@ -397,6 +419,7 @@ export const syncAllNodesAndRelations = async ( nodes: allNodes, supabaseClient, context, + fullSync: true, }); const accountLocalId = plugin.settings.accountLocalId; From fb2fc57c7afa94edb6da9590fa5bd7e1c75419fe Mon Sep 17 00:00:00 2001 From: Marc-Antoine Parent Date: Sun, 24 May 2026 10:49:21 -0400 Subject: [PATCH 2/2] moved setOperations to utils --- apps/obsidian/src/utils/publishNode.ts | 2 +- apps/obsidian/src/utils/syncDgNodesToSupabase.ts | 2 +- .../obsidian/src/utils => packages/utils/src}/setOperations.ts | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename {apps/obsidian/src/utils => packages/utils/src}/setOperations.ts (100%) diff --git a/apps/obsidian/src/utils/publishNode.ts b/apps/obsidian/src/utils/publishNode.ts index 7952ea1a2..78e0c86f8 100644 --- a/apps/obsidian/src/utils/publishNode.ts +++ b/apps/obsidian/src/utils/publishNode.ts @@ -17,7 +17,7 @@ import { syncPublishedNodeAssets, } from "./syncDgNodesToSupabase"; import { isProvisionalSchema } from "./typeUtils"; -import { intersection, difference } from "./setOperations"; +import { intersection, difference } from "@repo/utils/setOperations"; import type { DiscourseNodeInVault } from "./getDiscourseNodes"; import type { SupabaseContext } from "./supabaseContext"; diff --git a/apps/obsidian/src/utils/syncDgNodesToSupabase.ts b/apps/obsidian/src/utils/syncDgNodesToSupabase.ts index 748d7b422..03fc53232 100644 --- a/apps/obsidian/src/utils/syncDgNodesToSupabase.ts +++ b/apps/obsidian/src/utils/syncDgNodesToSupabase.ts @@ -28,7 +28,7 @@ import { } from "./getDiscourseNodes"; import { isAcceptedSchema } from "./typeUtils"; import { getTemplatePluginInfo } from "./templates"; -import { difference } from "./setOperations"; +import { difference } from "@repo/utils/setOperations"; const DEFAULT_TIME = "1970-01-01"; export type ChangeType = "title" | "content"; diff --git a/apps/obsidian/src/utils/setOperations.ts b/packages/utils/src/setOperations.ts similarity index 100% rename from apps/obsidian/src/utils/setOperations.ts rename to packages/utils/src/setOperations.ts