From 17a85f8fffab252732caed5cc986bf67ef0a392d Mon Sep 17 00:00:00 2001 From: Marc-Antoine Parent Date: Fri, 10 Jul 2026 17:00:10 -0400 Subject: [PATCH] eng-1865 sync and publish relations related to published nodes, WIP --- apps/roam/src/utils/createReifiedBlock.ts | 30 +++ apps/roam/src/utils/publishNodesToGroups.ts | 219 +++++++++++++++++- .../src/utils/roamToCrossAppConverters.ts | 89 ++++++- packages/database/src/inputTypes.ts | 2 +- 4 files changed, 337 insertions(+), 3 deletions(-) diff --git a/apps/roam/src/utils/createReifiedBlock.ts b/apps/roam/src/utils/createReifiedBlock.ts index 642ddb12a..ad78975f7 100644 --- a/apps/roam/src/utils/createReifiedBlock.ts +++ b/apps/roam/src/utils/createReifiedBlock.ts @@ -102,6 +102,36 @@ export const countReifiedRelations = async (): Promise => { return (r[0] || [0])[0] as number; }; +export type ReifiedRelationData = { + sourceUid: string; + destinationUid: string; + hasSchema: string; + importedFromRid?: string; +}; + +export type ReifiedRelationDataWithRelId = ReifiedRelationData & { + relationId: string; +}; + +export const getReifiedRelations = async (): Promise< + ReifiedRelationDataWithRelId[] +> => { + const pageUid = getExistingRelationPageUid(); + if (pageUid === undefined) return []; + const r = await window.roamAlphaAPI.data.async.q( + `[:find ?ruid ?rdata :where + [?p :block/uid "${pageUid}"] + [?p :block/children ?c] + [?c :block/uid ?ruid] + [?c :block/props ?pr] + [(get ?pr :${DISCOURSE_GRAPH_PROP_NAME}) ?rdata] ]`, + ); + return r.map((x) => ({ + relationId: x[0] as string, + ...(x[1] as ReifiedRelationData), + })); +}; + export const createReifiedRelation = async ({ sourceUid, relationBlockUid, diff --git a/apps/roam/src/utils/publishNodesToGroups.ts b/apps/roam/src/utils/publishNodesToGroups.ts index b873fc2be..ab74f285e 100644 --- a/apps/roam/src/utils/publishNodesToGroups.ts +++ b/apps/roam/src/utils/publishNodesToGroups.ts @@ -2,6 +2,18 @@ import { CrossAppNode } from "@repo/database/crossAppContracts"; import type { DGSupabaseClient } from "@repo/database/lib/client"; import { getAvailableGroupIds } from "@repo/database/lib/groups"; import { nodeUidsWithTypeToCrossApp } from "./roamToCrossAppConverters"; +import { + reifiedRelationToCrossApp, + relationTripleSchemaToCrossApp, +} from "./roamToCrossAppConverters"; +import getDiscourseRelations from "./getDiscourseRelations"; +import { getReifiedRelations } from "./createReifiedBlock"; +import { + crossAppRelationToDbConcept, + crossAppRelationTripleSchemaToDbConcept, +} from "@repo/database/lib/crossAppConverters"; +import type { LocalConceptDataInput } from "@repo/database/inputTypes"; +import type { TablesInsert } from "@repo/database/dbTypes"; export type NodeUidWithType = { uid: string; @@ -15,6 +27,196 @@ type PublishNodesResult = { failedGroupIds: string[]; }; +const getAllPublishedIdsByGroup = async ( + client: DGSupabaseClient, + spaceId: number, + groupIds: string[], +): Promise>> => { + const response = await client + .from("ResourceAccess") + .select("account_uid, source_local_id") + .eq("space_id", spaceId) + .in("account_uid", groupIds); + if (response.error) throw response.error; + const publishedIdsByGroupId = Object.fromEntries( + groupIds.map((gid) => [gid, new Set()]), + ); + response.data.forEach(({ account_uid, source_local_id }) => { + publishedIdsByGroupId[account_uid].add(source_local_id); + }); + + return publishedIdsByGroupId; +}; + +const getSpaceIdAndUrlsByGroupId = async ( + client: DGSupabaseClient, + groupIds: string[], +): Promise<{ + spaceUrlById: Record; + spaceIdsByGroupId: Record>; +}> => { + const response = await client + .from("SpaceAccess") + .select("account_uid, space_id") + .in("account_uid", groupIds); + if (response.error) throw response.error; + const spaceIds = response.data.map((r) => r.space_id); + const response2 = await client + .from("Space") + .select("id, url") + .in("id", spaceIds); + if (response2.error) throw response2.error; + const spaceUrlById = Object.fromEntries( + response2.data.map(({ id, url }) => [id, url]), + ); + const spaceIdsByGroupId = Object.fromEntries( + groupIds.map((gid) => [gid, new Set()]), + ); + response.data.forEach(({ account_uid, space_id }) => { + spaceIdsByGroupId[account_uid].add(space_id); + }); + return { + spaceUrlById, + spaceIdsByGroupId, + }; +}; + +// Use readImportedSourceIdentity from eng-1859 when it's merged. +// eslint-disable-next-line @typescript-eslint/no-unused-vars +const importedFromSpaceId = (nodeId: string): number | undefined => undefined; + +export const publishCorrespondingRelations = async ({ + client, + spaceId, + groupIds, + syncedUids, + forNodeIds, +}: { + client: DGSupabaseClient; + spaceId: number; + groupIds: string[]; + syncedUids: Set; + forNodeIds?: Set; +}): Promise<{ + upsertConcepts: LocalConceptDataInput[]; + publishInfo: TablesInsert<"ResourceAccess">[]; +}> => { + const allRelationsSchemas = getDiscourseRelations(); + const allRelationSchemasById = Object.fromEntries( + allRelationsSchemas.map((s) => [s.id, s]), + ); + // Should we even handle non-reified relations? Assuming not. + // I need a way to know if a relation is imported, see importedFromSpaceId + const allRelations = await getReifiedRelations(); + const spaceIdOfNodes: Record = {}; + const isImportedFrom = (nodeLocalId: string): number => { + let cached = spaceIdOfNodes[nodeLocalId]; + if (cached === undefined) { + cached = spaceIdOfNodes[nodeLocalId] = + importedFromSpaceId(nodeLocalId) || spaceId; + } + return cached === spaceId ? 0 : cached; + }; + const relations = + forNodeIds !== undefined + ? allRelations.filter( + (r) => + r.importedFromRid === undefined && + (forNodeIds.has(r.sourceUid) || forNodeIds.has(r.destinationUid)), + ) + : allRelations.filter((r) => r.importedFromRid === undefined); + const { spaceIdsByGroupId, spaceUrlById } = await getSpaceIdAndUrlsByGroupId( + client, + groupIds, + ); + const isImportedFromSpaceUri = (uid: string) => + spaceUrlById[isImportedFrom(uid) || 0]; + const publishedIdsByGroup = await getAllPublishedIdsByGroup( + client, + spaceId, + groupIds, + ); + // calculate separately to avoid case of a relation between nodes published to or from different groups + const relevantRelationIdsPerGroupId = Object.fromEntries( + groupIds.map((groupId) => { + const groupSpaceIds = spaceIdsByGroupId[groupId]; + const publishedIds = publishedIdsByGroup[groupId]; + return [ + groupId, + relations + .filter( + (r) => + (publishedIds.has(r.sourceUid) || + groupSpaceIds.has(isImportedFrom(r.sourceUid) || 0)) && + (publishedIds.has(r.destinationUid) || + groupSpaceIds.has(isImportedFrom(r.destinationUid) || 0)), + ) + .map((r) => r.relationId), + ]; + }), + ); + const allRelevantRelationIds = new Set( + Object.values(relevantRelationIdsPerGroupId).flat(), + ); + let allRelevantRelations = relations.filter((r) => + allRelevantRelationIds.has(r.relationId), + ); + const relationSchemaIds = new Set( + allRelevantRelations + .map((r) => r.hasSchema) + // filter out deleted schemas + .filter((id) => id in allRelationSchemasById), + ); + allRelevantRelations = allRelevantRelations.filter((r) => + relationSchemaIds.has(r.hasSchema), + ); + const missingRelationSchemaTriples = allRelationsSchemas.filter( + (r) => relationSchemaIds.has(r.id) && !syncedUids.has(r.id), + ); + const missingRelations = allRelevantRelations.filter( + (r) => !syncedUids.has(r.relationId), + ); + const upsertConcepts = [ + ...missingRelationSchemaTriples + .map((rs3) => relationTripleSchemaToCrossApp(rs3)) + .filter((rs3) => rs3 !== null) + .map((rs3) => crossAppRelationTripleSchemaToDbConcept(rs3)), + ...missingRelations + .map((r) => reifiedRelationToCrossApp(r, isImportedFromSpaceUri)) + .filter((r) => r !== null) + .map((r) => crossAppRelationToDbConcept(r)), + ].filter((r) => r !== undefined); + + const publishInfo = []; + for (const groupId of groupIds) { + const groupRelationIds = new Set(relevantRelationIdsPerGroupId[groupId]); + const groupMissingRelations = missingRelations.filter((r) => + groupRelationIds.has(r.relationId), + ); + const groupMissingRelationIds = groupMissingRelations.map( + (r) => r.relationId, + ); + const groupSchemaIds = new Set( + groupMissingRelations.map((r) => r.hasSchema), + ); + const groupMissingSchemaIds = missingRelationSchemaTriples + .filter((rs3) => groupSchemaIds.has(rs3.id)) + .map((rs3) => rs3.id); + const groupMissingIds = [ + ...groupMissingRelationIds, + ...groupMissingSchemaIds, + ]; + publishInfo.push( + ...groupMissingIds.map((sourceLocalId) => ({ + account_uid: groupId, + source_local_id: sourceLocalId, + space_id: spaceId, + })), + ); + } + return { upsertConcepts, publishInfo }; +}; + // 23505 = unique_violation: the grant already exists, which counts as success. const isIgnorableUpsertError = (error: { code?: string } | null): boolean => !error || error.code === "23505"; @@ -65,7 +267,6 @@ export const publishNodesToGroups = async ({ .from("my_concepts") .select("source_local_id") .eq("space_id", spaceId) - .eq("is_schema", false) .in("source_local_id", uids); if (syncedRes.error) throw syncedRes.error; const syncedUids = new Set( @@ -125,6 +326,22 @@ export const publishNodesToGroups = async ({ result.okGroupIds.push(groupId); } + const { upsertConcepts, publishInfo } = await publishCorrespondingRelations({ + client, + spaceId, + groupIds: targetGroupIds, + syncedUids, + forNodeIds: new Set(syncedNodeUids), + }); + const response = await client.rpc("upsert_concepts", { + v_space_id: spaceId, + data: upsertConcepts, + }); + if (response.error) throw response.error; + const grantRes = await client + .from("ResourceAccess") + .upsert(publishInfo, { ignoreDuplicates: true }); + if (!isIgnorableUpsertError(grantRes.error)) throw grantRes.error; result.publishedNodeUids = result.okGroupIds.length > 0 ? syncedNodeUids : []; return result; diff --git a/apps/roam/src/utils/roamToCrossAppConverters.ts b/apps/roam/src/utils/roamToCrossAppConverters.ts index bc7b9b77d..7cc8b6ba7 100644 --- a/apps/roam/src/utils/roamToCrossAppConverters.ts +++ b/apps/roam/src/utils/roamToCrossAppConverters.ts @@ -1,13 +1,21 @@ -import type { CrossAppNode } from "@repo/database/crossAppContracts"; +import type { + CrossAppNode, + CrossAppNodeSchema, + CrossAppRelation, + CrossAppRelationTripleSchema, +} from "@repo/database/crossAppContracts"; import type { RoamFullContentNode } from "./convertRoamNodeToFullContent"; import type { DiscourseNode } from "./getDiscourseNodes"; import type { TreeNode, ViewType } from "roamjs-components/types"; import type { NodeUidWithType } from "~/utils/publishNodesToGroups"; import type { Json } from "@repo/database/dbTypes"; +import type { ReifiedRelationDataWithRelId } from "./createReifiedBlock"; +import type { DiscourseRelation } from "./getDiscourseRelations"; import { toMarkdown } from "./pageToMarkdown"; import getFullTreeByParentUid from "roamjs-components/queries/getFullTreeByParentUid"; import getPageViewType from "roamjs-components/queries/getPageViewType"; import { contentTypes } from "@repo/content-model"; +import { spaceUriAndLocalIdToRid } from "@repo/database/lib/rid"; const FULL_MARKDOWN_OPTS = { refs: true, @@ -116,3 +124,82 @@ export const nodeUidsWithTypeToCrossApp = async ( }); return results; }; + +export const reifiedRelationToCrossApp = ( + r: ReifiedRelationDataWithRelId, + isImportedFromSpaceUri: (nodeUid: string) => string | undefined, +): CrossAppRelation | null => { + const sourceSpaceUri = isImportedFromSpaceUri(r.sourceUid); + const destinationSpaceUri = isImportedFromSpaceUri(r.destinationUid); + const sourceId = + sourceSpaceUri === undefined + ? r.sourceUid + : spaceUriAndLocalIdToRid(sourceSpaceUri, r.sourceUid); + const destinationId = + destinationSpaceUri === undefined + ? r.destinationUid + : spaceUriAndLocalIdToRid(destinationSpaceUri, r.destinationUid); + const relData = window.roamAlphaAPI.pull( + "[:create/time :edit/time {:create/user [:user/uid]}]", + `[:block/uid "${r.relationId}"]`, + ) as Record; + if (relData == undefined || !relData[":create/user"]) return null; + const userUid = (relData[":create/user"] as Record)[ + ":user/uid" + ]; + + return { + localId: r.relationId, + relationType: r.hasSchema, + source: sourceId, + destination: destinationId, + authorId: userUid, + createdAt: new Date(relData[":create/time"] as number), + modifiedAt: new Date(relData[":edit/time"] as number), + }; +}; + +export const relationTripleSchemaToCrossApp = ( + r: DiscourseRelation, +): CrossAppRelationTripleSchema | null => { + const relData = window.roamAlphaAPI.pull( + "[:create/time :edit/time {:create/user [:user/uid]}]", + `[:block/uid "${r.id}"]`, + ) as Record; + if (!relData) return null; + const userUid = (relData[":create/user"] as Record)[ + ":user/uid" + ]; + + return { + localId: r.id, + sourceType: r.source, + destinationType: r.destination, + label: r.label, + complement: r.complement, + authorId: userUid, + createdAt: new Date(relData[":create/time"] as number), + modifiedAt: new Date(relData[":edit/time"] as number), + }; +}; + +export const nodeSchemaToCrossApp = ( + s: DiscourseNode, +): CrossAppNodeSchema | null => { + const relData = window.roamAlphaAPI.pull( + "[:create/time :edit/time {:create/user [:user/uid]}]", + `[:block/uid "${s.type}"]`, + ) as unknown as { + ":create/time": number; + ":edit/time": number; + ":create/user": { ":user/uid": string }; + }; + if (!relData) return null; + const userUid = relData[":create/user"][":user/uid"]; + return { + localId: s.type, + label: s.text, + authorId: userUid, + createdAt: new Date(relData[":create/time"]), + }; +}; diff --git a/packages/database/src/inputTypes.ts b/packages/database/src/inputTypes.ts index 5ff4b76a8..64f69c635 100644 --- a/packages/database/src/inputTypes.ts +++ b/packages/database/src/inputTypes.ts @@ -6,7 +6,7 @@ export type LocalAccountDataInput = Partial< export type LocalDocumentDataInput = Partial< Omit< Database["public"]["CompositeTypes"]["document_local_input"], - "author_inline" + "author_inline" | "contents" > & { author_inline: LocalAccountDataInput } >; export type LocalContentDataInput = Partial<