From 07a5ec2d85922d89b28f115fec19e326732bdb66 Mon Sep 17 00:00:00 2001 From: Marc-Antoine Parent Date: Thu, 26 Mar 2026 19:03:49 -0400 Subject: [PATCH 1/4] eng-1485 rescan unpublished relations --- apps/obsidian/src/utils/publishNode.ts | 128 ++++++++++++++++++ .../src/utils/syncDgNodesToSupabase.ts | 13 +- 2 files changed, 140 insertions(+), 1 deletion(-) diff --git a/apps/obsidian/src/utils/publishNode.ts b/apps/obsidian/src/utils/publishNode.ts index 6a8355aa7..200dae951 100644 --- a/apps/obsidian/src/utils/publishNode.ts +++ b/apps/obsidian/src/utils/publishNode.ts @@ -15,6 +15,9 @@ import { import type { RelationInstance } from "~/types"; import { getAvailableGroupIds } from "./importNodes"; import { syncAllNodesAndRelations } from "./syncDgNodesToSupabase"; +import type { DiscourseNodeInVault } from "./getDiscourseNodes"; +import type { SupabaseContext } from "./supabaseContext"; +import type { TablesInsert } from "@repo/database/dbTypes"; const publishSchema = async ({ client, @@ -74,6 +77,21 @@ const intersection = (set1: Set, set2: Set): Set => { return r; }; +const difference = (set1: Set, set2: Set): Set => { + // @ts-expect-error - Set.difference is ES2025 feature + if (set1.difference) return set1.difference(set2); // eslint-disable-line + const result = new Set(set1); + if (set1.size <= set2.size) + for (const e of set1) { + if (set2.has(e)) result.delete(e); + } + else + for (const e of set2) { + if (result.has(e)) result.delete(e); + } + return result; +}; + export const publishNewRelation = async ( plugin: DiscourseGraphPlugin, relation: RelationInstance, @@ -249,6 +267,116 @@ export const publishNode = async ({ return await publishNodeToGroup({ plugin, file, frontmatter, myGroup }); }; +export const ensurePublishedRelationsAccuracy = async ({ + client, + context, + allNodesById, + relationInstances, +}: { + client: DGSupabaseClient; + context: SupabaseContext; + allNodesById: Record; + relationInstances: RelationInstance[]; +}): Promise => { + const myGroups = await getAvailableGroupIds(client); + const syncedRelationIdsResult = await client + .from("Concept") + .select("source_local_id") + .eq("space_id", context.spaceId) + .eq("is_schema", false) + .gt("arity", 0); + if (syncedRelationIdsResult.error) { + console.error( + "Could not get synced relation ids", + syncedRelationIdsResult.error, + ); + return; + } + const syncedRelationIds = new Set( + (syncedRelationIdsResult.data || []).map((x) => x.source_local_id!), + ); + // Also a good time to look at orphan relations + const existingRelationIds = new Set(relationInstances.map((r) => r.id)); + const orphanRelationIds = difference(syncedRelationIds, existingRelationIds); + if (orphanRelationIds.size) { + const r = await client + .from("Concept") + .delete() + .eq("space_id", context.spaceId) + .in("source_local_id", [...orphanRelationIds]); + if (!r.error) { + for (const id of orphanRelationIds) { + syncedRelationIds.delete(id); + } + } + } + const missingPublishRecords: TablesInsert<"ResourceAccess">[] = []; + for (const group of myGroups) { + const publishableRelations = relationInstances.filter( + (r) => + !r.importedFromRid && + ( + (allNodesById[r.source]?.frontmatter?.publishedToGroups as + | string[] + | undefined) || [] + ).indexOf(group) >= 0 && + ( + (allNodesById[r.destination]?.frontmatter?.publishedToGroups as + | string[] + | undefined) || [] + ).indexOf(group) >= 0, + ); + const publishableRelationIds = new Set( + publishableRelations.map((x) => x.id), + ); + const publishedIds = await client + .from("ResourceAccess") + .select("source_local_id") + .eq("account_uid", group) + .eq("space_id", context.spaceId); + if (publishedIds.error) { + console.error("Could not get synced relation ids", publishedIds.error); + continue; + } + const publishedRelationIds = intersection( + syncedRelationIds, + new Set((publishedIds.data || []).map((x) => x.source_local_id)), + ); + const missingPublishableIds = difference( + publishableRelationIds, + publishedRelationIds, + ); + if (missingPublishableIds.size > 0) { + missingPublishRecords.push( + /* eslint-disable @typescript-eslint/naming-convention */ + ...[...missingPublishableIds].map((source_local_id) => ({ + source_local_id, + space_id: context.spaceId, + account_uid: group, + })), + /* eslint-enable @typescript-eslint/naming-convention */ + ); + } + const extraPublishableIds = difference( + publishedRelationIds, + publishableRelationIds, + ); + if (extraPublishableIds.size > 0) { + const r = await client + .from("ResourceAccess") + .delete() + .eq("account_uid", group) + .eq("space_id", context.spaceId) + .in("source_local_id", [...extraPublishableIds]); + if (r.error) console.error(r.error); + } + } + if (missingPublishRecords.length > 0) { + const r = await client.from("ResourceAccess").upsert(missingPublishRecords); + if (r.error) console.error(r.error); + } +}; + export const publishNodeToGroup = async ({ plugin, file, diff --git a/apps/obsidian/src/utils/syncDgNodesToSupabase.ts b/apps/obsidian/src/utils/syncDgNodesToSupabase.ts index 8e1f206f5..9f77f4f90 100644 --- a/apps/obsidian/src/utils/syncDgNodesToSupabase.ts +++ b/apps/obsidian/src/utils/syncDgNodesToSupabase.ts @@ -9,7 +9,7 @@ import { type SupabaseContext, } from "./supabaseContext"; import { default as DiscourseGraphPlugin } from "~/index"; -import { publishNode } from "./publishNode"; +import { publishNode, ensurePublishedRelationsAccuracy } from "./publishNode"; import { upsertNodesToSupabaseAsContentWithEmbeddings } from "./upsertNodesAsContentWithEmbeddings"; import { orderConceptsByDependency, @@ -416,6 +416,7 @@ export const syncAllNodesAndRelations = async ( accountLocalId, plugin, allNodes, + startupRun: true, }); // When synced nodes are already published, ensure non-text assets are in storage. @@ -433,6 +434,7 @@ const convertDgToSupabaseConcepts = async ({ accountLocalId, plugin, allNodes, + startupRun, }: { nodesSince: ObsidianDiscourseNodeData[]; supabaseClient: DGSupabaseClient; @@ -440,6 +442,7 @@ const convertDgToSupabaseConcepts = async ({ accountLocalId: string; plugin: DiscourseGraphPlugin; allNodes?: DiscourseNodeInVault[]; + startupRun?: boolean; }): Promise => { const lastNodeSchemaSync = ( await getLastNodeSchemaSyncTime(supabaseClient, context.spaceId) @@ -562,6 +565,14 @@ const convertDgToSupabaseConcepts = async ({ ? error : JSON.stringify(error, null, 2); throw new Error(`upsert_concepts failed: ${errorMessage}`); + } else if (startupRun === true) { + // occasional extra work: Make sure relations that should be published are. + await ensurePublishedRelationsAccuracy({ + client: supabaseClient, + context, + allNodesById, + relationInstances: Object.values(relationInstancesData.relations), + }); } } }; From e98c6025f7c8cf1262445cb89da0bf815993988e Mon Sep 17 00:00:00 2001 From: Marc-Antoine Parent Date: Thu, 26 Mar 2026 19:46:01 -0400 Subject: [PATCH 2/4] also set publishedToGroup --- apps/obsidian/src/utils/publishNode.ts | 35 +++++++++++++++++-- .../src/utils/syncDgNodesToSupabase.ts | 3 +- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/apps/obsidian/src/utils/publishNode.ts b/apps/obsidian/src/utils/publishNode.ts index 200dae951..feed118ea 100644 --- a/apps/obsidian/src/utils/publishNode.ts +++ b/apps/obsidian/src/utils/publishNode.ts @@ -11,6 +11,7 @@ import { getFileForNodeInstanceIds, loadRelations, saveRelations, + type RelationsFile, } from "./relationsStore"; import type { RelationInstance } from "~/types"; import { getAvailableGroupIds } from "./importNodes"; @@ -270,15 +271,18 @@ export const publishNode = async ({ export const ensurePublishedRelationsAccuracy = async ({ client, context, + plugin, allNodesById, - relationInstances, + relationInstancesData, }: { client: DGSupabaseClient; context: SupabaseContext; + plugin: DiscourseGraphPlugin; allNodesById: Record; - relationInstances: RelationInstance[]; + relationInstancesData: RelationsFile; }): Promise => { const myGroups = await getAvailableGroupIds(client); + const relationInstances = Object.values(relationInstancesData.relations); const syncedRelationIdsResult = await client .from("Concept") .select("source_local_id") @@ -310,11 +314,13 @@ export const ensurePublishedRelationsAccuracy = async ({ } } } + let changed = false; const missingPublishRecords: TablesInsert<"ResourceAccess">[] = []; for (const group of myGroups) { const publishableRelations = relationInstances.filter( (r) => !r.importedFromRid && + syncedRelationIds.has(r.id) && ( (allNodesById[r.source]?.frontmatter?.publishedToGroups as | string[] @@ -369,11 +375,36 @@ export const ensurePublishedRelationsAccuracy = async ({ .eq("space_id", context.spaceId) .in("source_local_id", [...extraPublishableIds]); if (r.error) console.error(r.error); + else { + for (const id of extraPublishableIds) { + const rel = relationInstancesData.relations[id]; + const pos = (rel?.publishedToGroupId || []).indexOf(group); + if (pos >= 0) { + rel!.publishedToGroupId!.splice(pos, 1); + changed = true; + } + } + } } } if (missingPublishRecords.length > 0) { const r = await client.from("ResourceAccess").upsert(missingPublishRecords); if (r.error) console.error(r.error); + else { + for (const record of missingPublishRecords) { + const rel = relationInstancesData.relations[record.source_local_id]; + const group = record.account_uid; + const pos = (rel?.publishedToGroupId || []).indexOf(group); + if (rel && pos < 1) { + if (rel.publishedToGroupId === undefined) rel.publishedToGroupId = []; + rel.publishedToGroupId.push(group); + changed = true; + } + } + } + } + if (changed) { + await saveRelations(plugin, relationInstancesData); } }; diff --git a/apps/obsidian/src/utils/syncDgNodesToSupabase.ts b/apps/obsidian/src/utils/syncDgNodesToSupabase.ts index 9f77f4f90..71701f9ba 100644 --- a/apps/obsidian/src/utils/syncDgNodesToSupabase.ts +++ b/apps/obsidian/src/utils/syncDgNodesToSupabase.ts @@ -570,8 +570,9 @@ const convertDgToSupabaseConcepts = async ({ await ensurePublishedRelationsAccuracy({ client: supabaseClient, context, + plugin, allNodesById, - relationInstances: Object.values(relationInstancesData.relations), + relationInstancesData, }); } } From eb1fd8aa2ff6429da889f7d1912eb060ddfb32ae Mon Sep 17 00:00:00 2001 From: Marc-Antoine Parent Date: Thu, 26 Mar 2026 20:03:50 -0400 Subject: [PATCH 3/4] address devin comment --- .../src/utils/syncDgNodesToSupabase.ts | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/apps/obsidian/src/utils/syncDgNodesToSupabase.ts b/apps/obsidian/src/utils/syncDgNodesToSupabase.ts index 71701f9ba..cff579898 100644 --- a/apps/obsidian/src/utils/syncDgNodesToSupabase.ts +++ b/apps/obsidian/src/utils/syncDgNodesToSupabase.ts @@ -565,17 +565,18 @@ const convertDgToSupabaseConcepts = async ({ ? error : JSON.stringify(error, null, 2); throw new Error(`upsert_concepts failed: ${errorMessage}`); - } else if (startupRun === true) { - // occasional extra work: Make sure relations that should be published are. - await ensurePublishedRelationsAccuracy({ - client: supabaseClient, - context, - plugin, - allNodesById, - relationInstancesData, - }); } } + if (startupRun === true) { + // occasional extra work: Make sure relations that should be published are. + await ensurePublishedRelationsAccuracy({ + client: supabaseClient, + context, + plugin, + allNodesById, + relationInstancesData, + }); + } }; /** From 5423f040b373f860498482dfb9a455eec55a4e90 Mon Sep 17 00:00:00 2001 From: Marc-Antoine Parent Date: Thu, 26 Mar 2026 20:08:38 -0400 Subject: [PATCH 4/4] graphite comment --- apps/obsidian/src/utils/publishNode.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/obsidian/src/utils/publishNode.ts b/apps/obsidian/src/utils/publishNode.ts index feed118ea..d50ca3089 100644 --- a/apps/obsidian/src/utils/publishNode.ts +++ b/apps/obsidian/src/utils/publishNode.ts @@ -395,7 +395,7 @@ export const ensurePublishedRelationsAccuracy = async ({ const rel = relationInstancesData.relations[record.source_local_id]; const group = record.account_uid; const pos = (rel?.publishedToGroupId || []).indexOf(group); - if (rel && pos < 1) { + if (rel && pos < 0) { if (rel.publishedToGroupId === undefined) rel.publishedToGroupId = []; rel.publishedToGroupId.push(group); changed = true;