diff --git a/packages/software-factory/.gitattributes b/packages/software-factory/.gitattributes new file mode 100644 index 00000000000..1e7e7a296e0 --- /dev/null +++ b/packages/software-factory/.gitattributes @@ -0,0 +1 @@ +db-snapshots/template.pgdump binary diff --git a/packages/software-factory/db-snapshots/.gitkeep b/packages/software-factory/db-snapshots/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/packages/software-factory/db-snapshots/fingerprint.json b/packages/software-factory/db-snapshots/fingerprint.json new file mode 100644 index 00000000000..1079e27483e --- /dev/null +++ b/packages/software-factory/db-snapshots/fingerprint.json @@ -0,0 +1,7 @@ +{ + "fingerprint": "d4bb1aae0dbe17d2c7794f9983649c6590554abaf6370d0569300a977d46416d", + "cacheVersion": 8, + "realmServerURL": "http://localhost:41761/", + "generatedAt": "2026-04-10T16:03:26.733Z", + "pgDumpVersion": "pg_dump (PostgreSQL) 16.13 (Ubuntu 16.13-0ubuntu0.24.04.1)" +} diff --git a/packages/software-factory/db-snapshots/template.pgdump b/packages/software-factory/db-snapshots/template.pgdump new file mode 100644 index 00000000000..769019a9195 Binary files /dev/null and b/packages/software-factory/db-snapshots/template.pgdump differ diff --git a/packages/software-factory/package.json b/packages/software-factory/package.json index 296156f5576..e82c0182ef5 100644 --- a/packages/software-factory/package.json +++ b/packages/software-factory/package.json @@ -20,6 +20,7 @@ "lint:js": "eslint . --report-unused-disable-directives --cache", "lint:js:fix": "eslint . --report-unused-disable-directives --fix", "lint:format": "prettier --check .", + "lint:snapshot-freshness": "NODE_NO_WARNINGS=1 ts-node --transpileOnly scripts/lint-snapshot-freshness.ts", "lint:format:fix": "prettier --write .", "lint:types": "ember-tsc --noEmit", "serve:realm": "NODE_NO_WARNINGS=1 ts-node --transpileOnly src/cli/serve-realm.ts", diff --git a/packages/software-factory/playwright.global-setup.ts b/packages/software-factory/playwright.global-setup.ts index 7ce49f4410b..0611482546d 100644 --- a/packages/software-factory/playwright.global-setup.ts +++ b/packages/software-factory/playwright.global-setup.ts @@ -20,10 +20,6 @@ const fallbackRealmDir = resolve( packageRoot, 'test-fixtures/darkfactory-adopter', ); -const testSourceRealmDir = resolve( - packageRoot, - 'test-fixtures/public-software-factory-source', -); const bootstrapTargetRealmDir = resolve( packageRoot, 'test-fixtures/bootstrap-target', @@ -200,7 +196,6 @@ async function prepareTemplatesForRealms( ...process.env, SOFTWARE_FACTORY_CONTEXT: JSON.stringify(context), SOFTWARE_FACTORY_METADATA_FILE: metadataFile, - SOFTWARE_FACTORY_SOURCE_REALM_DIR: testSourceRealmDir, }, }); @@ -274,7 +269,6 @@ export default async function globalSetup() { ...process.env, NODE_NO_WARNINGS: '1', SOFTWARE_FACTORY_SUPPORT_METADATA_FILE: metadataFile, - SOFTWARE_FACTORY_SOURCE_REALM_DIR: testSourceRealmDir, }, }, ); @@ -300,12 +294,7 @@ export default async function globalSetup() { ); let preparedRealmDirs = [ - ...new Set([ - realmDir, - bootstrapTargetRealmDir, - testRealmRunnerDir, - testSourceRealmDir, - ]), + ...new Set([realmDir, bootstrapTargetRealmDir, testRealmRunnerDir]), ]; let preparedTemplates = await prepareTemplatesForRealms( preparedRealmDirs, diff --git a/packages/software-factory/scripts/lint-snapshot-freshness.ts b/packages/software-factory/scripts/lint-snapshot-freshness.ts new file mode 100644 index 00000000000..6a64faf67cd --- /dev/null +++ b/packages/software-factory/scripts/lint-snapshot-freshness.ts @@ -0,0 +1,95 @@ +import '../src/setup-logger'; +import { existsSync } from 'node:fs'; +import { + computeSnapshotFingerprint, + readSnapshotFingerprint, + DEFAULT_SNAPSHOT_FIXTURES, + DUMP_FILE, + FINGERPRINT_FILE, +} from '../src/harness/db-snapshot'; + +function printError(headline: string, detail: string): void { + console.error(''); + console.error(`lint:snapshot-freshness — ERROR`); + console.error(''); + console.error(` ${headline}`); + if (detail) { + console.error(''); + for (let line of detail.split('\n')) { + console.error(` ${line}`); + } + } + console.error(''); + console.error(' To fix this, run the following from the repo root:'); + console.error(''); + console.error( + ' cd packages/software-factory && pnpm cache:prepare --update-snapshot', + ); + console.error(''); + console.error( + ' This rebuilds the Playwright test database snapshot (~10 min on first run).', + ); + console.error( + ' Then commit the updated files in packages/software-factory/db-snapshots/.', + ); + console.error(''); + console.error( + ' NOTE: This requires a running PostgreSQL instance on the port configured', + ); + console.error( + ' for software-factory tests (default: 127.0.0.1:55436). If you do not have', + ); + console.error( + ' the test database infrastructure set up, ask someone on the team who works', + ); + console.error( + ' on the software-factory package to regenerate the snapshot.', + ); + console.error(''); +} + +function main(): void { + if (!existsSync(DUMP_FILE) || !existsSync(FINGERPRINT_FILE)) { + printError( + 'The software-factory database snapshot files are missing.', + 'This usually means the snapshot has not been generated yet.', + ); + process.exitCode = 1; + return; + } + + let committed = readSnapshotFingerprint(); + if (!committed) { + printError( + 'The software-factory snapshot fingerprint file is corrupt or unreadable.', + 'The file exists but could not be parsed as JSON.', + ); + process.exitCode = 1; + return; + } + + let currentFingerprint = computeSnapshotFingerprint( + DEFAULT_SNAPSHOT_FIXTURES, + ); + + if (committed.fingerprint !== currentFingerprint) { + printError( + 'The software-factory database snapshot is out of date.', + [ + 'Files in one of these directories have changed since the snapshot was last built:', + ' - packages/base/ (base realm)', + ' - packages/software-factory/realm/ (source realm)', + ' - packages/software-factory/test-fixtures/ (test fixtures)', + '', + ` Current source fingerprint: ${currentFingerprint}`, + ` Committed fingerprint: ${committed.fingerprint}`, + ].join('\n'), + ); + process.exitCode = 1; + return; + } + + console.log('snapshot-freshness: OK'); +} + +main(); diff --git a/packages/software-factory/src/cli/cache-realm.ts b/packages/software-factory/src/cli/cache-realm.ts index 913450adf55..69e0ec6714e 100644 --- a/packages/software-factory/src/cli/cache-realm.ts +++ b/packages/software-factory/src/cli/cache-realm.ts @@ -9,20 +9,29 @@ import { ensureFactoryRealmTemplate, } from '../harness'; import { isFactorySupportContext } from '../harness/shared'; +import { DEFAULT_SNAPSHOT_FIXTURES } from '../harness/db-snapshot'; import { readSupportContext } from '../runtime-metadata'; import { logger } from '../logger'; let log = logger('cache-realm'); async function main(): Promise { + let flags = process.argv.slice(2).filter((arg) => arg.startsWith('--')); let args = process.argv.slice(2).filter((arg) => !arg.startsWith('--')); - let realmDirs = [ - ...new Set( - (args.length > 0 ? args : ['test-fixtures/darkfactory-adopter']).map( - (realmDir) => resolve(process.cwd(), realmDir), + let useSnapshotFixtures = flags.includes('--update-snapshot'); + + let realmDirs: string[]; + if (useSnapshotFixtures) { + realmDirs = DEFAULT_SNAPSHOT_FIXTURES.map((f) => f.realmDir); + } else { + realmDirs = [ + ...new Set( + (args.length > 0 ? args : ['test-fixtures/darkfactory-adopter']).map( + (realmDir) => resolve(process.cwd(), realmDir), + ), ), - ), - ]; + ]; + } let serializedSupportContext = process.env.SOFTWARE_FACTORY_CONTEXT; let parsedEnvContext = serializedSupportContext diff --git a/packages/software-factory/src/harness/api.ts b/packages/software-factory/src/harness/api.ts index 321472e3b70..4559a9072a1 100644 --- a/packages/software-factory/src/harness/api.ts +++ b/packages/software-factory/src/harness/api.ts @@ -53,6 +53,12 @@ import { seedRealmPermissions, warnIfSnapshotLooksCold, } from './database'; +import { + checkCommittedSnapshot, + isCanonicalFixtureSet, + restoreTemplateFromDisk, + saveSnapshot, +} from './db-snapshot'; import { startFactorySupportServices } from './support-services'; import { startIsolatedRealmStack, @@ -165,6 +171,58 @@ export async function ensureFactoryRealmTemplate( : 'template database has not been prepared yet' : 'template database is missing'; + // Tier 2: Try restoring from committed pg_dump snapshot. + // Only attempt when the DB is actually missing (not when metadata is missing + // but the DB exists — CREATE DATABASE would fail in that case). + let snapshotFixtures: CombinedRealmFixture[] = [ + { realmDir, realmPath: realmRelativePath(realmURL, realmServerURL) }, + ]; + if (!hasTemplateDatabase) { + let snapshotData = checkCommittedSnapshot(snapshotFixtures); + if (snapshotData) { + harnessLog.info( + 'Restoring template from committed snapshot (fast path)', + ); + let snapshotServerURL = new URL(snapshotData.realmServerURL); + let snapshotRealmURL = new URL( + realmRelativePath(realmURL, realmServerURL), + snapshotServerURL, + ); + try { + await restoreTemplateFromDisk(templateDatabaseName); + writePreparedTemplateMetadata({ + realmDir, + templateDatabaseName, + templateRealmURL: snapshotRealmURL.href, + templateRealmServerURL: snapshotData.realmServerURL, + }); + return { + cacheKey, + templateDatabaseName, + fixtureHash, + cacheHit: false, + cacheMissReason: 'restored from snapshot', + realmURL: snapshotRealmURL, + realmServerURL: snapshotServerURL, + }; + } catch (error) { + harnessLog.warn( + `Snapshot restore failed, falling back to full build: ${error}`, + ); + try { + await dropDatabase(templateDatabaseName); + } catch { + /* best effort */ + } + } + } else { + harnessLog.info( + 'Snapshot not available or stale, proceeding with full build', + ); + } + } + + // Tier 3: Full build from scratch. let ownedSupport: | { context: FactorySupportContext; @@ -194,6 +252,19 @@ export async function ensureFactoryRealmTemplate( templateRealmServerURL: realmServerURL.href, }); + // Save snapshot for future fast restores (only for canonical fixtures). + if (isCanonicalFixtureSet(snapshotFixtures)) { + try { + await saveSnapshot( + templateDatabaseName, + realmServerURL.href, + snapshotFixtures, + ); + } catch (error) { + harnessLog.warn(`Failed to save snapshot: ${error}`); + } + } + return { cacheKey, templateDatabaseName, @@ -289,6 +360,58 @@ export async function ensureCombinedFactoryRealmTemplate( : 'template database has not been prepared yet' : 'template database is missing'; + // Resolve fixtures with absolute paths for snapshot operations. + let resolvedFixtures: CombinedRealmFixture[] = fixtures.map((f) => ({ + realmDir: resolve(f.realmDir), + realmPath: f.realmPath, + })); + + // Tier 2: Try restoring from committed pg_dump snapshot. + // Only attempt when the DB is actually missing (not when metadata is missing + // but the DB exists — CREATE DATABASE would fail in that case). + if (!hasTemplateDatabase) { + let snapshotData = checkCommittedSnapshot(resolvedFixtures); + if (snapshotData) { + harnessLog.info( + 'Restoring template from committed snapshot (fast path)', + ); + try { + await restoreTemplateFromDisk(templateDatabaseName); + writePreparedTemplateMetadata({ + realmDir: resolvedFixtures[0].realmDir, + templateDatabaseName, + templateRealmURL: + snapshotData.realmServerURL + resolvedFixtures[0].realmPath, + templateRealmServerURL: snapshotData.realmServerURL, + coveredRealmDirs: resolvedFixtures.map((f) => f.realmDir), + }); + return { + cacheKey, + templateDatabaseName, + combinedFixtureHash, + cacheHit: false, + cacheMissReason: 'restored from snapshot', + coveredRealmDirs: resolvedFixtures.map((f) => f.realmDir), + realmServerURL: new URL(snapshotData.realmServerURL), + }; + } catch (error) { + harnessLog.warn( + `Snapshot restore failed, falling back to full build: ${error}`, + ); + try { + await dropDatabase(templateDatabaseName); + } catch { + /* best effort */ + } + } + } else { + harnessLog.info( + 'Snapshot not available or stale, proceeding with full build', + ); + } + } + + // Tier 3: Full build from scratch. let ownedSupport: | { context: FactorySupportContext; stop(): Promise } | undefined; @@ -300,10 +423,10 @@ export async function ensureCombinedFactoryRealmTemplate( try { // Resolve realm URLs for each fixture. - let realmFixtures = fixtures.map((f) => { + let realmFixtures = resolvedFixtures.map((f) => { let realmURL = new URL(f.realmPath, realmServerURL); return { - realmDir: resolve(f.realmDir), + realmDir: f.realmDir, realmURL, }; }); @@ -325,6 +448,19 @@ export async function ensureCombinedFactoryRealmTemplate( templateRealmServerURL: realmServerURL.href, }); + // Save snapshot for future fast restores (only for canonical fixtures). + if (isCanonicalFixtureSet(resolvedFixtures)) { + try { + await saveSnapshot( + templateDatabaseName, + realmServerURL.href, + resolvedFixtures, + ); + } catch (error) { + harnessLog.warn(`Failed to save snapshot: ${error}`); + } + } + return { cacheKey, templateDatabaseName, diff --git a/packages/software-factory/src/harness/db-snapshot.ts b/packages/software-factory/src/harness/db-snapshot.ts new file mode 100644 index 00000000000..0efea662bcf --- /dev/null +++ b/packages/software-factory/src/harness/db-snapshot.ts @@ -0,0 +1,400 @@ +import { spawnSync } from 'node:child_process'; +import { + existsSync, + mkdirSync, + readFileSync, + renameSync, + statSync, + writeFileSync, +} from 'node:fs'; +import { dirname, join, resolve } from 'node:path'; + +import { Client as PgClient } from 'pg'; + +import { + baseRealmDir, + CACHE_VERSION, + DEFAULT_PG_HOST, + DEFAULT_PG_PORT, + DEFAULT_PG_USER, + hashCombinedRealmFixtures, + hashRealmFixture, + hashString, + logTimed, + packageRoot, + pgAdminConnectionConfig, + quotePgIdentifier, + sourceRealmDir, + stableStringify, + templateLog, + type CombinedRealmFixture, +} from './shared'; + +// --------------------------------------------------------------------------- +// Constants +// --------------------------------------------------------------------------- + +export const DB_SNAPSHOTS_DIR = resolve(packageRoot, 'db-snapshots'); +export const DUMP_FILE = join(DB_SNAPSHOTS_DIR, 'template.pgdump'); +export const FINGERPRINT_FILE = join(DB_SNAPSHOTS_DIR, 'fingerprint.json'); + +/** Canonical fixture list matching playwright.global-setup.ts. */ +export const DEFAULT_SNAPSHOT_FIXTURES: CombinedRealmFixture[] = [ + { + realmDir: resolve(packageRoot, 'test-fixtures/darkfactory-adopter'), + realmPath: 'test/', + }, + { + realmDir: resolve(packageRoot, 'test-fixtures/bootstrap-target'), + realmPath: 'bootstrap-target/', + }, + { + realmDir: resolve(packageRoot, 'test-fixtures/test-realm-runner'), + realmPath: 'test-realm-runner/', + }, +]; + +/** + * Space-separated glob controlling which source realm files are included + * in the test DB snapshot. Prefix a pattern with ! to exclude. + * Evaluated in order — last matching pattern wins. + * + * Only the core card definitions (darkfactory, test-results) are needed + * for tests. The wiki, document, and other content in realm/ are not + * necessary for the Playwright test suite. + */ +export const SOURCE_REALM_GLOB = '*.gts .realm.json !document.gts !wiki.gts'; + +export function matchesSourceRealmGlob(relativePath: string): boolean { + let filename = relativePath.split('/').pop() ?? relativePath; + let included = false; + for (let pattern of SOURCE_REALM_GLOB.split(/\s+/)) { + let negate = pattern.startsWith('!'); + let glob = negate ? pattern.slice(1) : pattern; + let hit = glob.startsWith('*') + ? filename.endsWith(glob.slice(1)) + : filename === glob; + if (hit) { + included = !negate; + } + } + return included; +} + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +export interface SnapshotFingerprintData { + fingerprint: string; + cacheVersion: number; + /** Metadata only — not part of fingerprint hash. */ + realmServerURL: string; + /** ISO timestamp. */ + generatedAt: string; + /** Output from pg_dump --version. */ + pgDumpVersion: string; +} + +// --------------------------------------------------------------------------- +// Fingerprint computation +// --------------------------------------------------------------------------- + +/** + * Check whether the given fixtures match the canonical snapshot fixture set. + * Only canonical fixtures should read/write the committed snapshot. + */ +export function isCanonicalFixtureSet( + fixtures: CombinedRealmFixture[], +): boolean { + if (fixtures.length !== DEFAULT_SNAPSHOT_FIXTURES.length) { + return false; + } + let canonical = DEFAULT_SNAPSHOT_FIXTURES.map((f) => f.realmDir).sort(); + let actual = fixtures.map((f) => resolve(f.realmDir)).sort(); + return canonical.every((dir, i) => dir === actual[i]); +} + +/** + * Compute a deterministic fingerprint string for the given fixtures. + * Pure computation — no I/O beyond file hashing. + */ +export function computeSnapshotFingerprint( + fixtures: CombinedRealmFixture[], +): string { + let baseRealmHash = hashRealmFixture(baseRealmDir); + let sourceRealmHash = hashRealmFixture(sourceRealmDir, { + fileFilter: matchesSourceRealmGlob, + }); + let combinedFixtureHash = hashCombinedRealmFixtures(fixtures); + return hashString( + stableStringify({ + version: CACHE_VERSION, + baseRealmHash, + sourceRealmHash, + combinedFixtureHash, + }), + ); +} + +// --------------------------------------------------------------------------- +// Fingerprint I/O +// --------------------------------------------------------------------------- + +/** Read and parse the committed fingerprint. Returns undefined if missing or malformed. */ +export function readSnapshotFingerprint(): SnapshotFingerprintData | undefined { + if (!existsSync(FINGERPRINT_FILE)) { + return undefined; + } + try { + return JSON.parse( + readFileSync(FINGERPRINT_FILE, 'utf8'), + ) as SnapshotFingerprintData; + } catch { + return undefined; + } +} + +/** Atomic write of fingerprint data (temp file + rename). */ +export function writeSnapshotFingerprint(data: SnapshotFingerprintData): void { + mkdirSync(DB_SNAPSHOTS_DIR, { recursive: true }); + let tempFile = join( + dirname(FINGERPRINT_FILE), + `.fingerprint.${process.pid}.${Date.now()}.tmp`, + ); + writeFileSync(tempFile, JSON.stringify(data, null, 2) + '\n'); + renameSync(tempFile, FINGERPRINT_FILE); +} + +// --------------------------------------------------------------------------- +// pg_dump / pg_restore helpers +// --------------------------------------------------------------------------- + +/** Parse the pg_dump version string. Returns 'unknown' on failure. */ +export function getPgDumpVersion(): string { + try { + let result = spawnSync('pg_dump', ['--version'], { encoding: 'utf8' }); + if (result.status === 0 && result.stdout) { + return result.stdout.trim(); + } + return 'unknown'; + } catch { + return 'unknown'; + } +} + +function pgEnv(): Record { + let env = { ...process.env } as Record; + if (process.env.PGPASSWORD) { + env.PGPASSWORD = process.env.PGPASSWORD; + } + return env; +} + +/** + * Dump a template database to disk using pg_dump --format=custom. + * + * To minimize dump size, we clone the template to a temporary database, + * strip data that is rebuilt at clone/startup time, then dump that copy: + * - boxel_index_working: rebuilt via rebuildWorkingIndexFromIndex() + * - boxel_index.last_known_good_deps: fallback deps, rebuilt on next index + * - modules: cleared via clearModuleCache() on realm startup + * - jobs / job_reservations: cleared via resetQueueState() + */ +export async function dumpTemplateToDisk(databaseName: string): Promise { + mkdirSync(DB_SNAPSHOTS_DIR, { recursive: true }); + + // Clone to a temporary database so we can strip columns without + // modifying the live template. + let tmpDb = `sf_dump_tmp_${process.pid}`; + let adminClient = new PgClient(pgAdminConnectionConfig()); + try { + await adminClient.connect(); + await adminClient.query( + `DROP DATABASE IF EXISTS ${quotePgIdentifier(tmpDb)}`, + ); + await adminClient.query( + `CREATE DATABASE ${quotePgIdentifier(tmpDb)} TEMPLATE ${quotePgIdentifier(databaseName)}`, + ); + } finally { + await adminClient.end(); + } + + try { + // NULL out large columns that are rebuilt at runtime. + let stripClient = new PgClient(pgAdminConnectionConfig(tmpDb)); + try { + await stripClient.connect(); + await stripClient.query( + `UPDATE boxel_index SET last_known_good_deps = NULL`, + ); + await stripClient.query(`TRUNCATE boxel_index_working`); + await stripClient.query(`TRUNCATE modules`); + await stripClient.query(`TRUNCATE job_reservations, jobs CASCADE`); + await stripClient.query(`VACUUM FULL`); + } finally { + await stripClient.end(); + } + + let result = spawnSync( + 'pg_dump', + [ + '--format=custom', + '--no-owner', + '--no-privileges', + '-h', + DEFAULT_PG_HOST, + '-p', + DEFAULT_PG_PORT, + '-U', + DEFAULT_PG_USER, + '-f', + DUMP_FILE, + tmpDb, + ], + { env: pgEnv(), encoding: 'utf8' }, + ); + if (result.status !== 0) { + throw new Error( + `pg_dump failed with exit code ${result.status}: ${result.stderr || result.error?.message || 'unknown error'}`, + ); + } + } finally { + // Clean up the temporary database. + let cleanupClient = new PgClient(pgAdminConnectionConfig()); + try { + await cleanupClient.connect(); + await cleanupClient.query( + `DROP DATABASE IF EXISTS ${quotePgIdentifier(tmpDb)}`, + ); + } finally { + await cleanupClient.end(); + } + } +} + +/** + * Restore a template database from the committed dump file. + * Steps: CREATE DATABASE, pg_restore, ALTER DATABASE IS_TEMPLATE true. + * On failure at any step, attempts to drop the partially created DB. + */ +export async function restoreTemplateFromDisk( + databaseName: string, +): Promise { + let client = new PgClient(pgAdminConnectionConfig()); + try { + await client.connect(); + await client.query(`CREATE DATABASE ${quotePgIdentifier(databaseName)}`); + } finally { + await client.end(); + } + + try { + let result = spawnSync( + 'pg_restore', + [ + '--no-owner', + '--no-privileges', + '--single-transaction', + '-h', + DEFAULT_PG_HOST, + '-p', + DEFAULT_PG_PORT, + '-U', + DEFAULT_PG_USER, + '-d', + databaseName, + DUMP_FILE, + ], + { env: pgEnv(), encoding: 'utf8' }, + ); + if (result.status !== 0) { + throw new Error( + `pg_restore failed with exit code ${result.status}: ${result.stderr || result.error?.message || 'unknown error'}`, + ); + } + + let templateClient = new PgClient(pgAdminConnectionConfig()); + try { + await templateClient.connect(); + await templateClient.query( + `ALTER DATABASE ${quotePgIdentifier(databaseName)} WITH IS_TEMPLATE true`, + ); + } finally { + await templateClient.end(); + } + } catch (error) { + // Best-effort cleanup: drop the partially created DB. + try { + let cleanupClient = new PgClient(pgAdminConnectionConfig()); + try { + await cleanupClient.connect(); + await cleanupClient.query( + `DROP DATABASE IF EXISTS ${quotePgIdentifier(databaseName)}`, + ); + } finally { + await cleanupClient.end(); + } + } catch { + // Swallow cleanup errors — the original error is more important. + } + throw error; + } +} + +// --------------------------------------------------------------------------- +// High-level snapshot operations +// --------------------------------------------------------------------------- + +/** + * Check whether a committed snapshot exists and matches the current source files. + * Returns fingerprint data if valid, undefined if stale or missing. + */ +export function checkCommittedSnapshot( + fixtures: CombinedRealmFixture[], +): SnapshotFingerprintData | undefined { + if (!existsSync(DUMP_FILE) || !existsSync(FINGERPRINT_FILE)) { + return undefined; + } + + let data = readSnapshotFingerprint(); + if (!data) { + return undefined; + } + + let currentFingerprint = computeSnapshotFingerprint(fixtures); + if (data.fingerprint !== currentFingerprint) { + return undefined; + } + + return data; +} + +/** + * Save a snapshot of the template database to disk with fingerprint metadata. + * Called after a successful full build. + */ +export async function saveSnapshot( + databaseName: string, + realmServerURL: string, + fixtures: CombinedRealmFixture[], +): Promise { + await logTimed(templateLog, 'saveSnapshot', async () => { + let fingerprint = computeSnapshotFingerprint(fixtures); + + await dumpTemplateToDisk(databaseName); + + let dumpSize = statSync(DUMP_FILE).size; + templateLog.info( + `snapshot dump written: ${(dumpSize / 1024 / 1024).toFixed(1)} MB`, + ); + + writeSnapshotFingerprint({ + fingerprint, + cacheVersion: CACHE_VERSION, + realmServerURL, + generatedAt: new Date().toISOString(), + pgDumpVersion: getPgDumpVersion(), + }); + }); +} diff --git a/packages/software-factory/src/harness/isolated-realm-stack.ts b/packages/software-factory/src/harness/isolated-realm-stack.ts index bf490d1dd30..5c5a9540015 100644 --- a/packages/software-factory/src/harness/isolated-realm-stack.ts +++ b/packages/software-factory/src/harness/isolated-realm-stack.ts @@ -7,6 +7,7 @@ import { mkdtempSync, readdirSync, readFileSync, + realpathSync, rmSync, writeFileSync, } from 'node:fs'; @@ -14,6 +15,7 @@ import { join, relative } from 'node:path'; import { tmpdir } from 'node:os'; import fsExtra from 'fs-extra'; import { spawn } from 'node:child_process'; +import { matchesSourceRealmGlob } from './db-snapshot'; import { baseRealmDir, @@ -248,12 +250,25 @@ function copyRealmFixture( realmDir: string, destination: string, sourceRealmURL: URL, + options?: { fileFilter?: (relativePath: string) => boolean }, ): void { - copySync(realmDir, destination, { + // Resolve symlinks so copySync sees the real directory, not the symlink itself. + let resolvedDir = realpathSync(realmDir); + copySync(resolvedDir, destination, { preserveTimestamps: true, filter(src) { - let relativePath = relative(realmDir, src).replace(/\\/g, '/'); - return relativePath === '' || !shouldIgnoreFixturePath(relativePath); + let relativePath = relative(resolvedDir, src).replace(/\\/g, '/'); + if (relativePath !== '' && shouldIgnoreFixturePath(relativePath)) { + return false; + } + if ( + relativePath !== '' && + options?.fileFilter && + !options.fileFilter(relativePath) + ) { + return false; + } + return true; }, }); rewriteFixtureSourceModuleUrls(destination, sourceRealmURL); @@ -299,6 +314,15 @@ export async function startIsolatedRealmStack({ prerenderURL?: string; }): Promise { let rootDir = mkdtempSync(join(tmpdir(), 'software-factory-realms-')); + // Create a filtered copy of the source realm — only card definitions + // (via SOURCE_REALM_GLOB), not instance data like wiki briefs or documents. + let filteredSourceRealmDir = join(rootDir, 'source-realm'); + copyRealmFixture( + sourceRealmDir, + filteredSourceRealmDir, + new URL('https://placeholder/'), + { fileFilter: matchesSourceRealmGlob }, + ); let testRealmDir = join(rootDir, 'test'); let workerManagerMetadataFile = join(rootDir, 'worker-manager.runtime.json'); let realmServerMetadataFile = join(rootDir, 'realm-server.runtime.json'); @@ -486,7 +510,7 @@ export async function startIsolatedRealmStack({ `--fromUrl=${publicBaseRealmURL.href}`, `--toUrl=${actualBaseRealmURL.href}`, '--username=software_factory_realm', - `--path=${sourceRealmDir}`, + `--path=${filteredSourceRealmDir}`, `--fromUrl=${sourceRealmURL.href}`, `--toUrl=${actualSourceRealmURL.href}`, '--username=test_realm', diff --git a/packages/software-factory/src/harness/shared.ts b/packages/software-factory/src/harness/shared.ts index 3be1e1540eb..4e995e8a10e 100644 --- a/packages/software-factory/src/harness/shared.ts +++ b/packages/software-factory/src/harness/shared.ts @@ -146,6 +146,7 @@ export const sourceRealmDir = resolve( process.env.SOFTWARE_FACTORY_SOURCE_REALM_DIR ?? 'realm', ); export const boxelIconsDir = resolve(packageRoot, '..', 'boxel-icons'); +export const dbSnapshotDir = resolve(packageRoot, 'db-snapshots'); export const prepareTestPgScript = resolve( realmServerDir, 'tests', @@ -412,7 +413,10 @@ export function shouldIgnoreFixturePath(relativePath: string): boolean { ); } -export function hashRealmFixture(realmDir: string): string { +export function hashRealmFixture( + realmDir: string, + options?: { fileFilter?: (relativePath: string) => boolean }, +): string { let entries: string[] = []; function visit(currentDir: string) { @@ -429,6 +433,9 @@ export function hashRealmFixture(realmDir: string): string { if (!entry.isFile()) { continue; } + if (options?.fileFilter && !options.fileFilter(relativePath)) { + continue; + } let stats = statSync(absolutePath); let contentsHash = createHash('sha256') .update(readFileSync(absolutePath)) diff --git a/packages/software-factory/test-fixtures/public-software-factory-source b/packages/software-factory/test-fixtures/public-software-factory-source new file mode 120000 index 00000000000..12eaf350ad8 --- /dev/null +++ b/packages/software-factory/test-fixtures/public-software-factory-source @@ -0,0 +1 @@ +../realm \ No newline at end of file diff --git a/packages/software-factory/test-fixtures/public-software-factory-source/.realm.json b/packages/software-factory/test-fixtures/public-software-factory-source/.realm.json deleted file mode 100644 index 4264b01cd90..00000000000 --- a/packages/software-factory/test-fixtures/public-software-factory-source/.realm.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "name": "Software Factory Public Source Test Realm", - "iconURL": null, - "backgroundURL": null -} diff --git a/packages/software-factory/test-fixtures/public-software-factory-source/darkfactory.gts b/packages/software-factory/test-fixtures/public-software-factory-source/darkfactory.gts deleted file mode 120000 index cf4d0995a32..00000000000 --- a/packages/software-factory/test-fixtures/public-software-factory-source/darkfactory.gts +++ /dev/null @@ -1 +0,0 @@ -../../realm/darkfactory.gts \ No newline at end of file diff --git a/packages/software-factory/test-fixtures/public-software-factory-source/home.gts b/packages/software-factory/test-fixtures/public-software-factory-source/home.gts deleted file mode 100644 index 7716c8d2b81..00000000000 --- a/packages/software-factory/test-fixtures/public-software-factory-source/home.gts +++ /dev/null @@ -1,9 +0,0 @@ -import { Component, CardDef } from 'https://cardstack.com/base/card-api'; - -export class Home extends CardDef { - static isolated = class Isolated extends Component { - - }; -} diff --git a/packages/software-factory/test-fixtures/public-software-factory-source/index.json b/packages/software-factory/test-fixtures/public-software-factory-source/index.json deleted file mode 100644 index f20df53720a..00000000000 --- a/packages/software-factory/test-fixtures/public-software-factory-source/index.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "data": { - "type": "card", - "attributes": {}, - "meta": { - "adoptsFrom": { - "module": "./home.gts", - "name": "Home" - } - } - } -} diff --git a/packages/software-factory/tests/darkfactory.spec.ts b/packages/software-factory/tests/darkfactory.spec.ts deleted file mode 100644 index c101d7a3a98..00000000000 --- a/packages/software-factory/tests/darkfactory.spec.ts +++ /dev/null @@ -1,108 +0,0 @@ -import { resolve } from 'node:path'; - -import type { Page } from '@playwright/test'; - -import { expect, test } from './fixtures'; - -const adopterRealmDir = resolve( - process.cwd(), - 'test-fixtures', - 'darkfactory-adopter', -); - -test.use({ realmDir: adopterRealmDir }); -test.use({ realmServerMode: 'shared' }); - -async function gotoCard(page: Page, url: string) { - await page.goto(url, { waitUntil: 'commit' }); -} - -test('renders a project adopted from the public DarkFactory module', async ({ - authedPage, - cardURL, -}) => { - await gotoCard(authedPage, cardURL('project-demo')); - - await expect( - authedPage.getByRole('heading', { name: 'DarkFactory Adoption Harness' }), - ).toBeVisible({ timeout: 120_000 }); - await expect( - authedPage.getByRole('heading', { name: 'Objective' }), - ).toBeVisible(); - await expect( - authedPage.getByRole('heading', { name: 'Success Criteria' }), - ).toBeVisible(); - await expect( - authedPage.getByRole('heading', { name: 'Knowledge Base' }), - ).toBeVisible(); - await expect(authedPage.getByText('Agent Onboarding')).toBeVisible(); -}); - -test('renders an issue adopted from the public DarkFactory module', async ({ - authedPage, - cardURL, -}) => { - await gotoCard(authedPage, cardURL('issue-demo')); - - await expect( - authedPage.getByRole('heading', { - name: 'Verify public DarkFactory adoption', - }), - ).toBeVisible({ timeout: 120_000 }); - await expect( - authedPage.getByRole('heading', { name: 'Project' }), - ).toBeVisible(); - await expect( - authedPage.getByText('DarkFactory Adoption Harness'), - ).toBeVisible(); - await expect( - authedPage.getByRole('heading', { name: 'Acceptance Criteria' }), - ).toBeVisible(); - await expect( - authedPage.getByRole('heading', { name: 'Related Knowledge' }), - ).toBeVisible(); -}); - -test('renders a knowledge article and agent profile adopted from the public DarkFactory module', async ({ - authedPage, - cardURL, -}) => { - await gotoCard(authedPage, cardURL('knowledge-article-demo')); - - await expect( - authedPage.getByRole('heading', { name: 'Agent Onboarding' }).first(), - ).toBeVisible({ timeout: 120_000 }); - await expect( - authedPage.getByText('onboarding', { exact: true }).first(), - ).toBeVisible(); - await expect( - authedPage.getByText( - 'Use the project card for scope, the issue card for execution, and update notes as you go.', - ), - ).toBeVisible(); - - await gotoCard(authedPage, cardURL('agent-demo')); - - await expect( - authedPage.getByRole('heading', { name: 'codex-darkfactory' }), - ).toBeVisible({ timeout: 120_000 }); - await expect(authedPage.getByText('Boxel tracker workflows')).toBeVisible(); - await expect(authedPage.getByText('issue triage')).toBeVisible(); -}); - -test('renders a DarkFactory card with active projects from the adopter realm', async ({ - authedPage, - cardURL, -}) => { - await gotoCard(authedPage, cardURL('factory-demo')); - - await expect( - authedPage.getByRole('heading', { name: 'DarkFactory Test Fixture' }), - ).toBeVisible({ timeout: 120_000 }); - await expect( - authedPage.getByRole('heading', { name: 'Active Projects' }), - ).toBeVisible(); - await expect( - authedPage.getByText('DarkFactory Adoption Harness'), - ).toBeVisible(); -}); diff --git a/packages/software-factory/tests/fixtures.ts b/packages/software-factory/tests/fixtures.ts index 55bb882749e..19648aaf1d0 100644 --- a/packages/software-factory/tests/fixtures.ts +++ b/packages/software-factory/tests/fixtures.ts @@ -82,10 +82,6 @@ const defaultRealmDir = resolve( packageRoot, process.env.SOFTWARE_FACTORY_REALM_DIR ?? 'test-fixtures/darkfactory-adopter', ); -const testSourceRealmDir = resolve( - packageRoot, - 'test-fixtures/public-software-factory-source', -); const sharedRealms = new Map>(); const testWorkerPortBlockSize = 10; const testWorkerPortSearchStride = 200; @@ -293,7 +289,6 @@ async function startRealmProcess( ...process.env, NODE_NO_WARNINGS: '1', SOFTWARE_FACTORY_METADATA_FILE: metadataFile, - SOFTWARE_FACTORY_SOURCE_REALM_DIR: testSourceRealmDir, ...(supportMetadata?.context ? { SOFTWARE_FACTORY_CONTEXT: JSON.stringify(supportMetadata.context),