diff --git a/packages/database/features/addGeneric.feature b/packages/database/features/addGeneric.feature new file mode 100644 index 000000000..6856a6d0c --- /dev/null +++ b/packages/database/features/addGeneric.feature @@ -0,0 +1,134 @@ +Feature: Concept upsert + User story: + * As a user of the Roam plugin + * Logged in through a given space's anonymous account + * I want to be able to upsert concepts to that space + + Acceptance criteria: + * The upsert should succeed + + Background: + Given the database is blank + And the user user1 opens the Roam plugin in space s1 + + Scenario Outline: Calling the upsert steps together + When user user1 upserts this cross-app data to space s1: + """ + { + "nodeSchemas": [ + { + "label": "Claim", + "authorId": "user1", + "localId": "ns1", + "createdAt": "2000/01/01", + "modifiedAt": "2001/01/02" + } + ], + "relationTripleSchemas": [ + { + "authorId": "user1", + "localId": "r3s1", + "label": "supports", + "complement": "supportedBy", + "sourceType": "ns1", + "destinationType": "ns1", + "createdAt": "2000/01/01", + "modifiedAt": "2001/01/02" + } + ], + "nodes": [ + { + "title": "Some first claim", + "authorId": "user1", + "localId": "n1", + "nodeType": "ns1", + "createdAt": "2000/01/01", + "modifiedAt": "2001/01/02", + "content": { + "direct": { + "value": "Some text" + } + } + }, + { + "title": "Some second claim", + "authorId": "user1", + "localId": "n2", + "nodeType": "ns1", + "createdAt": "2000/01/03", + "modifiedAt": "2001/01/04", + "content": { + "direct": { + "value": "Some subtext", + "embedding": { + "model": "openai_text_embedding_3_small_1536", + "value": [ + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + ] + } + } + } + } + ], + "relations": [ + { + "name": "a supports relation", + "authorId": "user1", + "nodeType": "r3s1", + "createdAt": "2000/01/04", + "modifiedAt": "2001/01/05", + "source": "n1", + "destination": "n2" + } + ] + } + """ + Then a user logged in space s1 should see 5 Concept in the database + And a user logged in space s1 should see 2 Content in the database + And a user logged in space s1 should see 1 ContentEmbedding_openai_text_embedding_3_small_1536 in the database diff --git a/packages/database/features/step-definitions/stepdefs.ts b/packages/database/features/step-definitions/stepdefs.ts index 4f0f087e8..ecbd7ef3e 100644 --- a/packages/database/features/step-definitions/stepdefs.ts +++ b/packages/database/features/step-definitions/stepdefs.ts @@ -11,8 +11,10 @@ import { type Enums, type Json, } from "@repo/database/dbTypes"; +import type { DGSupabaseClient } from "~/lib/client"; import { getVariant, config } from "@repo/database/dbDotEnv"; import { getConcepts, initNodeSchemaCache } from "@repo/database/lib/queries"; +import axios from "axios"; import { spaceAnonUserEmail, @@ -268,6 +270,21 @@ const getLoggedinDatabase = async (spaceId: number) => { return client; }; +const getApiHeader = async (supabase: DGSupabaseClient) => { + const session = await supabase.auth.getSession(); + const sessionData = session.data?.session; + assert(sessionData, "Cannot find session token"); + const token = btoa(JSON.stringify(sessionData)); + return { + Cookie: `sb-127-auth-token=base64-${token}`, + }; +}; + +const getApiHeaderForSpace = async (spaceId: number) => { + const supabase = await getLoggedinDatabase(spaceId); + return await getApiHeader(supabase); +}; + const getLoggedinDatabaseForUsername = async (userName: string) => { assert.notStrictEqual(userName, undefined); const localRefs = (world.localRefs || {}) as LocalRefsType; @@ -421,6 +438,24 @@ Given( }, ); +// invoke the vercel upsert function, expects json +Given( + "user {word} upserts this cross-app data to space {word}:", + async (userName: string, spaceName: string, docString: string) => { + const data = JSON.parse(docString) as Record; + const localRefs = (world.localRefs || {}) as LocalRefsType; + const spaceId = localRefs[spaceName]; + if (typeof spaceId !== "number") assert.fail("spaceId not a number"); + const headers = await getApiHeaderForSpace(spaceId); + const response = await axios.post( + `http://localhost:3000/api/internal/space/${spaceId}`, + data, + { headers }, + ); + assert.equal(response.status, 200); + }, +); + Given( "a user logged in space {word} and calling getConcepts with these parameters: {string}", async (spaceName: string, paramsJ: string) => { diff --git a/packages/database/package.json b/packages/database/package.json index 5653b1eda..96dfe2575 100644 --- a/packages/database/package.json +++ b/packages/database/package.json @@ -58,6 +58,7 @@ "@repo/typescript-config": "workspace:*", "@types/node": "catalog:", "@vercel/sdk": "^1.19.40", + "axios": "^0.27.2", "dotenv": "^16.6.1", "eslint": "catalog:", "prettier-plugin-gherkin": "^3.1.3", diff --git a/packages/database/scripts/serveAndTest.ts b/packages/database/scripts/serveAndTest.ts index 032f05ad7..ccfafde77 100644 --- a/packages/database/scripts/serveAndTest.ts +++ b/packages/database/scripts/serveAndTest.ts @@ -1,5 +1,6 @@ -import { spawn, execSync } from "node:child_process"; +import { spawn, execSync, ChildProcessByStdio } from "node:child_process"; import { join, dirname } from "path"; +import type { Readable } from "node:stream"; const scriptDir = dirname(__filename); const projectRoot = join(scriptDir, ".."); @@ -16,58 +17,96 @@ if (process.env.SUPABASE_PROJECT_ID !== "test") { process.exit(2); } -const serve = spawn("supabase", ["functions", "serve"], { - cwd: projectRoot, - detached: true, - stdio: ["ignore", "pipe", "inherit"], -}); +process.env["SUPABASE_USE_DB"] = "local"; -let resolveCallback: ((value: unknown) => void) | undefined = undefined; -let rejectCallback: ((reason: unknown) => void) | undefined = undefined; -let serveSuccess = false; -let timeoutClear: NodeJS.Timeout | undefined = undefined; +class ParallelServer { + process: ChildProcessByStdio; + promise: Promise; + serveSuccess = false; + resolveCallback: + | ((value: boolean | PromiseLike) => void) + | undefined; + rejectCallback: ((reason: unknown) => void) | undefined; + timeoutClear: NodeJS.Timeout | undefined; + constructor({ + cmd, + args, + readySignal, + dir, + }: { + cmd: string; + args: string[]; + readySignal: string; + dir?: string; + }) { + this.process = spawn(cmd, args, { + cwd: dir || projectRoot, + detached: true, + stdio: ["ignore", "pipe", "inherit"], + }); + this.promise = new Promise((rsc, rjc) => { + this.resolveCallback = rsc; + this.rejectCallback = rjc; -const servingReady = new Promise((rsc, rjc) => { - resolveCallback = rsc; - rejectCallback = rjc; + // Add timeout + this.timeoutClear = setTimeout(() => { + rjc(new Error("Timeout waiting for functions to serve")); + }, 30000); // 30 second timeout + }); + this.process.stdout.on("data", (data: Buffer) => { + const output = data.toString(); + console.log(`stdout: ${output}`); + if (output.includes(readySignal)) { + console.log("Found serving functions"); + this.serveSuccess = true; + clearTimeout(this.timeoutClear); + if (this.resolveCallback === undefined) + throw new Error("did not get callback"); + this.resolveCallback(true); + } + }); + this.process.on("close", () => { + if (!this.serveSuccess && this.rejectCallback) + this.rejectCallback(new Error("serve closed without being ready")); + }); + this.process.on("error", (err) => { + if (this.rejectCallback) this.rejectCallback(err); + }); + } +} - // Add timeout - timeoutClear = setTimeout(() => { - rjc(new Error("Timeout waiting for functions to serve")); - }, 30000); // 30 second timeout +const supabaseServer = new ParallelServer({ + cmd: "supabase", + args: ["functions", "serve"], + readySignal: "Serving functions ", }); -serve.stdout.on("data", (data: Buffer) => { - const output = data.toString(); - console.log(`stdout: ${output}`); - if (output.includes("Serving functions ")) { - console.log("Found serving functions"); - serveSuccess = true; - clearTimeout(timeoutClear); - if (resolveCallback === undefined) throw new Error("did not get callback"); - resolveCallback(null); - } -}); -serve.on("close", () => { - if (!serveSuccess && rejectCallback) - rejectCallback(new Error("serve closed without being ready")); -}); -serve.on("error", (err) => { - if (rejectCallback) rejectCallback(err); +const nextServer = new ParallelServer({ + cmd: "npm", + args: ["run", "dev"], + readySignal: "Ready in ", + dir: join(scriptDir, "../../../apps/website"), }); const doTest = async () => { try { - await servingReady; + await Promise.all([supabaseServer.promise, nextServer.promise]); execSync("cucumber-js", { cwd: projectRoot, stdio: "inherit", }); // will throw on failure } finally { - if (serve.pid) + if (supabaseServer.process.pid) + try { + process.kill(-supabaseServer.process.pid); + } catch (e) { + console.error("Could not kill the process"); + // maybe it just ended on its own. + } + if (nextServer.process.pid) try { - process.kill(-serve.pid); + process.kill(-nextServer.process.pid); } catch (e) { console.error("Could not kill the process"); // maybe it just ended on its own. @@ -78,10 +117,12 @@ const doTest = async () => { doTest() .then(() => { console.log("success"); - clearTimeout(timeoutClear); + clearTimeout(supabaseServer.timeoutClear); + clearTimeout(nextServer.timeoutClear); }) .catch((err) => { console.error(err); - clearTimeout(timeoutClear); + clearTimeout(supabaseServer.timeoutClear); + clearTimeout(nextServer.timeoutClear); process.exit(1); }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d3d0aeba4..0b77e56b6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -618,6 +618,9 @@ importers: '@vercel/sdk': specifier: ^1.19.40 version: 1.19.40 + axios: + specifier: ^0.27.2 + version: 0.27.2(debug@4.4.3) dotenv: specifier: ^16.6.1 version: 16.6.1 @@ -5089,9 +5092,6 @@ packages: '@types/node@20.11.0': resolution: {integrity: sha512-o9bjXmDNcF7GbM4CNQpmi+TutCgap/K3w1JyKgxAjqx41zp9qlIAVFi0IhCNsJcXolEqLWhbFbEeL0PvYm4pcQ==} - '@types/node@20.19.13': - resolution: {integrity: sha512-yCAeZl7a0DxgNVteXFHt9+uyFbqXGy/ShC4BlcHkoE0AfGXYv/BUiplV72DjMYXHDBXFjhvr6DD1NiRVfB4j8g==} - '@types/node@22.20.0': resolution: {integrity: sha512-QWlFW2wf3nTjC13/DqRnBpR4ZO36VJH/JVBkA/vcnmbTBNQIlnObqyqZE1tUR7+Ni23Lda8R1BxMfbXRpCUx5g==} @@ -5871,7 +5871,7 @@ packages: basic-ftp@5.0.5: resolution: {integrity: sha512-4Bcg1P8xhUuqcii/S0Z9wiHIrQVPMermM1any+MX5GeGD7faD3/msQUDGLol9wOcz4/jbg/WJnGqoJF6LiBdtg==} engines: {node: '>=10.0.0'} - deprecated: Security vulnerability fixed in 5.2.0, please upgrade + deprecated: Security vulnerability fixed in 5.2.1, please upgrade before-after-hook@3.0.2: resolution: {integrity: sha512-Nik3Sc0ncrMK4UUdXQmAnRtzmNQTAAXmXIopizwZ1W1t8QmfJj+zL4OA2I7XPTPW5z5TDqv4hRo/JzouDJnX3A==} @@ -14055,7 +14055,7 @@ snapshots: '@playwright/test@1.29.0': dependencies: - '@types/node': 20.19.13 + '@types/node': 22.20.0 playwright-core: 1.29.0 '@popperjs/core@2.11.8': {} @@ -17060,7 +17060,7 @@ snapshots: '@types/jsdom@20.0.1': dependencies: - '@types/node': 20.19.13 + '@types/node': 22.20.0 '@types/tough-cookie': 4.0.5 parse5: 7.3.0 @@ -17122,10 +17122,6 @@ snapshots: dependencies: undici-types: 5.26.5 - '@types/node@20.19.13': - dependencies: - undici-types: 6.21.0 - '@types/node@22.20.0': dependencies: undici-types: 6.21.0