From 862ea8cf6ba3f16637b6ec3534acf8a2c161de8d Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Tue, 23 Jun 2026 17:28:08 -0500 Subject: [PATCH 1/8] Change publishing endpoint to be async --- .../handlers/handle-publish-realm.ts | 172 +++++++++--------- .../tests/publish-unpublish-realm-test.ts | 20 ++ .../server-endpoints/index-responses-test.ts | 14 ++ packages/runtime-common/realm.ts | 10 + 4 files changed, 131 insertions(+), 85 deletions(-) diff --git a/packages/realm-server/handlers/handle-publish-realm.ts b/packages/realm-server/handlers/handle-publish-realm.ts index 8be89597236..9f876f7feb0 100644 --- a/packages/realm-server/handlers/handle-publish-realm.ts +++ b/packages/realm-server/handlers/handle-publish-realm.ts @@ -1,6 +1,5 @@ import type Koa from 'koa'; import { - createResponse, fetchUserPermissions, isResolvedCodeRef, query, @@ -21,14 +20,10 @@ import { getPublishedRealmDomainOverrides } from '@cardstack/runtime-common/cons import { join } from 'path'; import fsExtra from 'fs-extra'; -const { - copySync, - readJsonSync, - writeJsonSync, - removeSync, - existsSync, - moveSync, -} = fsExtra; +// Async fs ops only: the publish handler runs these inside the request, and a +// synchronous copy/move of a whole realm directory would freeze the Node event +// loop, stalling every other concurrent request until it finished. +const { copy, readJson, writeJson, remove, pathExists, move } = fsExtra; import { fetchRequestFromContext, @@ -154,15 +149,20 @@ async function maybeApplyPublishedRealmOverride( // the index.json's adoptsFrom — a published realm that has customised // its index to a different CardDef is left alone (its isolated render // is presumably the bespoke landing page the publisher wanted). -function ensureRealmIndexBoilerplateOptIn(publishedRealmPath: string): void { +async function ensureRealmIndexBoilerplateOptIn( + publishedRealmPath: string, +): Promise { let indexJsonPath = join(publishedRealmPath, 'index.json'); let realmJsonPath = join(publishedRealmPath, 'realm.json'); - if (!existsSync(indexJsonPath) || !existsSync(realmJsonPath)) { + if ( + !(await pathExists(indexJsonPath)) || + !(await pathExists(realmJsonPath)) + ) { return; } let indexDoc: unknown; try { - indexDoc = readJsonSync(indexJsonPath); + indexDoc = await readJson(indexJsonPath); } catch (e) { log.warn( `could not parse published index.json at ${indexJsonPath}: ${ @@ -186,7 +186,7 @@ function ensureRealmIndexBoilerplateOptIn(publishedRealmPath: string): void { } let realmConfigDoc: Record; try { - realmConfigDoc = readJsonSync(realmJsonPath) as Record; + realmConfigDoc = (await readJson(realmJsonPath)) as Record; } catch (e) { log.warn( `could not parse published realm.json at ${realmJsonPath}: ${ @@ -204,7 +204,7 @@ function ensureRealmIndexBoilerplateOptIn(publishedRealmPath: string): void { data.attributes = attributes; realmConfigDoc.data = data; try { - writeJsonSync(realmJsonPath, realmConfigDoc, { spaces: 2 }); + await writeJson(realmJsonPath, realmConfigDoc, { spaces: 2 }); } catch (e) { log.warn( `could not write includePrerenderedDefaultRealmIndex into ${realmJsonPath}: ${ @@ -482,21 +482,24 @@ export default function handlePublishRealm({ // enqueueReindexRealmJob below to refresh the index. let tempCopyPath = `${publishedRealmPath}.tmp`; let backupPath = `${publishedRealmPath}.backup`; - removeSync(tempCopyPath); - removeSync(backupPath); - copySync(sourceRealmPath, tempCopyPath); + await remove(tempCopyPath); + await remove(backupPath); + await copy(sourceRealmPath, tempCopyPath); try { - if (existsSync(publishedRealmPath)) { - moveSync(publishedRealmPath, backupPath); + if (await pathExists(publishedRealmPath)) { + await move(publishedRealmPath, backupPath); } - moveSync(tempCopyPath, publishedRealmPath); - removeSync(backupPath); + await move(tempCopyPath, publishedRealmPath); + await remove(backupPath); } catch (swapError) { // Restore the old published realm if the swap failed - if (!existsSync(publishedRealmPath) && existsSync(backupPath)) { - moveSync(backupPath, publishedRealmPath); + if ( + !(await pathExists(publishedRealmPath)) && + (await pathExists(backupPath)) + ) { + await move(backupPath, publishedRealmPath); } - removeSync(tempCopyPath); + await remove(tempCopyPath); throw swapError; } @@ -524,7 +527,7 @@ export default function handlePublishRealm({ // The flag is written to the published realm's RealmConfig // card (/realm.json) on disk before the reindex below picks // it up. - ensureRealmIndexBoilerplateOptIn(publishedRealmPath); + await ensureRealmIndexBoilerplateOptIn(publishedRealmPath); // Clear stale modules cache for the published realm (including // error entries from a previous publish) before the reindex's @@ -554,7 +557,7 @@ export default function handlePublishRealm({ // Phase 3 PR 2 rollback simplification: no in-memory // realms[]/virtualNetwork state to unwind. Just remove the // FS swap that we just put in place. - removeSync(publishedRealmPath); + await remove(publishedRealmPath); throw dbError; } @@ -573,12 +576,13 @@ export default function handlePublishRealm({ // an async race against the immediately-enqueued reindex. // Force the invalidation synchronously here. // - // For a new publish, lookupOrMount mounts the realm fresh - // (registry row was just upserted above); the cache is - // empty so clearLocalSourceCaches is a no-op. Either way the - // reindex below sees correct source. + // Use the non-mounting `mounted` map rather than lookupOrMount: + // for a new publish the realm isn't mounted here yet and there's + // nothing cached to clear — and mounting it would await a + // from-scratch index inside the request, which this handler must + // not block on. It lazy-mounts fresh on its first request instead. let mountedRealmForCacheClear = - await reconciler.lookupOrMount(publishedRealmURL); + reconciler.mounted.get(publishedRealmURL); if (mountedRealmForCacheClear) { // Sync local clear + cross-replica NOTIFY in one call. The // local clear is what this replica's reindex fan-out needs; @@ -587,15 +591,16 @@ export default function handlePublishRealm({ await mountedRealmForCacheClear.clearLocalSourceCachesAndBroadcast(); } - // Refresh the index. For a new publish this is redundant - // (lazy-mount's first start() does its own fullIndex on a - // fresh DB), but the from-scratch-index coalesce handler - // (CS-10893) collapses both into a single canonical job. For - // a republish where the realm is already mounted with a - // resolved #startedUp, this is the only mechanism that - // re-indexes against the swapped files. clearLastModified - // forces every row to re-render even if mtimes appear - // unchanged (file copies preserve mtimes). + // Durability enqueue: guarantees the swapped files get indexed + // even if no client ever polls this published realm. The index is + // not awaited here — the handler returns 202 (pending) and the + // client polls _readiness-check. For a realm not mounted on this + // instance, its first request (typically the readiness poll) + // lazy-mounts it and start()'s from-scratch pass coalesces with + // this job. For a republish already mounted here, the post-lock + // fullIndex below tracks completion for readiness. clearLastModified + // forces every row to re-render even though file copies preserve + // mtimes. await enqueueReindexRealmJob( publishedRealmURL, realmUsername, @@ -609,44 +614,38 @@ export default function handlePublishRealm({ }, ); - // Mount + start the published realm on this instance now. The - // reconciler's prepareRealmFromRow constructs a Realm and adds - // it to realms[] / virtualNetwork; ensureMounted then awaits - // realm.start() which awaits the from-scratch-index job we - // enqueued above (the chooseFromScratch coalesce JOINs the - // start()-enqueued job with ours). By the time we return 202, - // indexing is complete on this instance — sibling instances - // pick the published realm up via NOTIFY and lazy-mount on - // first request. This preserves the test-suite's synchronous- - // publish semantics while keeping the handler purely registry- - // driven. - let publishedRealm = await reconciler.lookupOrMount(publishedRealmURL); - if (!publishedRealm) { - throw new Error( - `expected published realm ${publishedRealmURL} to be mounted after publish — registry row missing or mount failed`, - ); - } - // Re-run a full index after start()'s pass so the RealmConfig card - // at /realm.json is queryable by parseRealmInfo before /index is - // re-rendered. start()'s from-scratch pass walks files in order and - // typically renders /index before /realm.json — at which point - // attachRealmInfo → getRealmInfo → parseRealmInfo finds /realm.json - // not yet indexed, falls back to "Unnamed Workspace", and caches - // that. The prerendered head HTML for /index is baked with the - // stale value, surfacing as og:title="Unnamed Workspace" on the - // published page. + // Indexing/prerender is not awaited in the request: the handler returns + // 202 (pending) and the client polls _readiness-check. + // Awaiting a full index + prerender (pool-bound) here would hold the HTTP + // request open for the entire indexing duration. // - // clearLastModified: true forces every row to re-render on this - // pass even though copySync preserves mtimes — without it, the - // indexer's mtime-cache check would skip the already-rendered - // /index and the stale prerendered HTML would persist. - // Realm.fullIndex clears #cachedRealmInfo before this pass so the - // first attachRealmInfo call re-reads parseRealmInfo against the - // now-populated index and bakes the correct realm name into the - // re-rendered prerendered HTML. - await publishedRealm.fullIndex(userInitiatedPriority, { - clearLastModified: true, - }); + // If this realm is already mounted on this instance (a republish), its + // #startedUp is already resolved, so _readiness-check would otherwise + // return 200 immediately — before the swapped files are reindexed. + // Kick a fire-and-forget full index here: publishFullIndex registers + // its in-flight deferred synchronously, so Realm.indexing() (which + // readinessCheck awaits) reflects this reindex until it completes. + // A single pass suffices for the correct og:title — Realm.fullIndex + // invalidates the cached RealmInfo before the pass, and parseRealmInfo + // reads the realm name from the swapped realm.json via its on-disk + // overlay, so /index re-bakes with the right name without a second + // pass. (This job coalesces with the durability enqueue above.) + // + // For a realm not mounted here (new publish, or a peer instance), the + // durability enqueue above plus lazy-mount-on-first-request handle the + // index; #startedUp resolves only after that from-scratch pass. + let mountedPublishedRealm = reconciler.mounted.get(publishedRealmURL); + if (mountedPublishedRealm) { + void mountedPublishedRealm + .fullIndex(userInitiatedPriority, { clearLastModified: true }) + .catch((err: unknown) => { + log.error( + `background publish reindex failed for ${publishedRealmURL}: ${ + err instanceof Error ? err.message : String(err) + }`, + ); + }); + } // The source realm's `RealmInfo.lastPublishedAt` map is built // from `realm_registry` rows joined on `source_url = sourceRealmURL`, @@ -663,8 +662,11 @@ export default function handlePublishRealm({ new URL(publishedRealmURL), ); - let response = createResponse({ - body: JSON.stringify( + // Build the 202 directly rather than via createResponse: the published + // realm may not be mounted on this instance (a new publish lazy-mounts + // on first request), so there is no Realm object to read a url from. + let response = new Response( + JSON.stringify( { data: { type: 'published_realm', @@ -680,17 +682,17 @@ export default function handlePublishRealm({ null, 2, ), - init: { + { status: 202, headers: { 'content-type': SupportedMimeType.JSONAPI, + 'X-Boxel-Realm-Url': publishedRealmURL, + ...(publishedPermissions['*']?.includes('read') && { + 'X-Boxel-Realm-Public-Readable': 'true', + }), }, }, - requestContext: { - realm: publishedRealm, - permissions: publishedPermissions, - }, - }); + ); await setContextResponse(ctxt, response); return; } catch (error: any) { diff --git a/packages/realm-server/tests/publish-unpublish-realm-test.ts b/packages/realm-server/tests/publish-unpublish-realm-test.ts index eb1a509350f..21cde21b713 100644 --- a/packages/realm-server/tests/publish-unpublish-realm-test.ts +++ b/packages/realm-server/tests/publish-unpublish-realm-test.ts @@ -1272,6 +1272,26 @@ module(basename(import.meta.filename), function () { let publishedRealmURL = publishResponse.body.data.attributes.publishedRealmURL; + // Publish returns 202 before indexing finishes: drive a reconcile + // pass to mount the published realm, then wait for the from-scratch + // index to populate boxel_index before asserting on it. + await testRealmServer.testingOnlyReconcile(); + await waitUntil( + async () => { + let rows = await dbAdapter.execute( + `SELECT 1 FROM boxel_index WHERE realm_url = $1 LIMIT 1`, + { bind: [publishedRealmURL] }, + ); + return rows.length > 0 ? rows : undefined; + }, + { + timeout: 30_000, + interval: 100, + timeoutMessage: + 'boxel_index entries for published realm did not appear', + }, + ); + // Verify that boxel_index entries exist before unpublishing let indexResultsBefore = await dbAdapter.execute( `SELECT * FROM boxel_index WHERE realm_url = '${publishedRealmURL}'`, diff --git a/packages/realm-server/tests/server-endpoints/index-responses-test.ts b/packages/realm-server/tests/server-endpoints/index-responses-test.ts index 130373fb037..10bd8ec0e3e 100644 --- a/packages/realm-server/tests/server-endpoints/index-responses-test.ts +++ b/packages/realm-server/tests/server-endpoints/index-responses-test.ts @@ -1517,6 +1517,20 @@ module(`server-endpoints/${basename(import.meta.filename)}`, function () { `Failed to publish realm: ${publishResponse.status} ${publishResponse.text}`, ); } + + // `_publish-realm` returns 202 before indexing finishes. The + // published realm's readiness check lazy-mounts it and awaits + // start() + any in-flight index, so this single request blocks until + // the swapped files are indexed and the assertions below query + // indexed content. + let readinessResponse = await request + .get(`${publishedRealmPath}_readiness-check`) + .set('Host', publishedRealmHost); + if (readinessResponse.status !== 200) { + throw new Error( + `Published realm not ready: ${readinessResponse.status} ${readinessResponse.text}`, + ); + } }, afterEach: async () => { await closeServer(testRealmHttpServer); diff --git a/packages/runtime-common/realm.ts b/packages/runtime-common/realm.ts index ee8584c5ba5..e1e0a8a572d 100644 --- a/packages/runtime-common/realm.ts +++ b/packages/runtime-common/realm.ts @@ -1140,6 +1140,16 @@ export class Realm { requestContext: RequestContext, ) { await this.#startedUp.promise; + // #startedUp is a one-time gate that resolves after the first start()'s + // from-scratch index. On a republish the realm is already mounted with a + // resolved #startedUp, so awaiting it alone would report ready before the + // reindex of the swapped files completes. Also await any in-flight full or + // incremental index so a publish poll only succeeds once the just-published + // content is indexed and viewable. + let inflight = this.indexing(); + if (inflight) { + await inflight; + } return createResponse({ body: null, From f112c6b7ad9927852c83b0040dd05d09f846fbd6 Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Tue, 23 Jun 2026 17:28:38 -0500 Subject: [PATCH 2/8] Update front-end for new contract --- packages/host/app/commands/publish-realm.ts | 11 ++++++----- packages/host/app/services/realm-server.ts | 17 +++++++++++++++++ packages/host/app/services/realm.ts | 8 +++++++- .../host/tests/acceptance/host-submode-test.gts | 3 +++ .../integration/commands/publish-realm-test.gts | 6 ++++++ 5 files changed, 39 insertions(+), 6 deletions(-) diff --git a/packages/host/app/commands/publish-realm.ts b/packages/host/app/commands/publish-realm.ts index f78f3a6fc49..007f6bdceb3 100644 --- a/packages/host/app/commands/publish-realm.ts +++ b/packages/host/app/commands/publish-realm.ts @@ -17,11 +17,12 @@ import type RealmService from '../services/realm'; import type { PublishabilityViolation } from '../services/realm'; // Publishes a realm to one or more destinations (subdirectory Boxel Spaces or -// custom domains). The command resolves once the realm-server accepts each -// publish request and reports per-target status. Indexed-and-viewable -// readiness is not awaited here: realm `index` events aren't delivered to the -// run-command/prerender context, so a caller that needs the published realm -// ready polls its `_readiness-check` over HTTP instead. +// custom domains) and reports per-target status. `_publish-realm` returns 202 +// before the published realm is indexed, so `realm.publish` polls each +// target's `_readiness-check` over HTTP before resolving — the command +// thus completes only once every published realm is indexed and viewable. +// (Readiness is polled over HTTP rather than awaited via realm `index` events, +// which aren't delivered to the run-command/prerender context.) export default class PublishRealmCommand extends HostBaseCommand< typeof BaseCommandModule.PublishRealmInput, typeof BaseCommandModule.PublishRealmResult diff --git a/packages/host/app/services/realm-server.ts b/packages/host/app/services/realm-server.ts index 4ca9d4d599b..af70e88be00 100644 --- a/packages/host/app/services/realm-server.ts +++ b/packages/host/app/services/realm-server.ts @@ -21,6 +21,7 @@ import { ri, testRealmURL, unpublishRealm as unpublishRealmOperation, + waitForReady as waitForReadyOperation, type RealmClient, type RealmIdentifier, type RealmInfo, @@ -944,6 +945,22 @@ export default class RealmServerService extends Service { return unpublishRealmOperation(this.realmClient, { publishedRealmURL }); } + // Polls _readiness-check until the published realm is + // indexed and viewable. `_publish-realm` returns 202 before indexing + // finishes, so callers that need the realm ready wait here — the Publish UI + // keeps its "Publishing…" state until this resolves. + async waitForRealmReady( + publishedRealmURL: string, + opts?: { timeoutMs?: number; pollIntervalMs?: number }, + ) { + await this.login(); + return waitForReadyOperation(this.realmClient, { + publishedRealmURL, + timeoutMs: opts?.timeoutMs, + pollIntervalMs: opts?.pollIntervalMs, + }); + } + async checkDomainAvailability( subdomain: string, ): Promise { diff --git a/packages/host/app/services/realm.ts b/packages/host/app/services/realm.ts index ece37d92e0a..5a791c1b28e 100644 --- a/packages/host/app/services/realm.ts +++ b/packages/host/app/services/realm.ts @@ -627,7 +627,13 @@ class RealmResource { this._publishingRealms.push(url); try { - return await this.realmServer.publishRealm(this.url, url); + let result = await this.realmServer.publishRealm(this.url, url); + // `_publish-realm` returns 202 before the published realm is + // indexed. Keep the "Publishing…" state until the realm passes its + // readiness check so "Open Site" only enables once the page is + // actually viewable. + await this.realmServer.waitForRealmReady(url); + return result; } catch (error) { console.error(`Error publishing to URL ${url}:`, error); throw error; // Re-throw so Promise.allSettled can capture it as rejected diff --git a/packages/host/tests/acceptance/host-submode-test.gts b/packages/host/tests/acceptance/host-submode-test.gts index 45cbeea60d1..e9aa7411190 100644 --- a/packages/host/tests/acceptance/host-submode-test.gts +++ b/packages/host/tests/acceptance/host-submode-test.gts @@ -688,6 +688,9 @@ module('Acceptance | host submode', function (hooks) { getService('realm-server').publishRealm = publishRealm; getService('realm-server').unpublishRealm = unpublishRealm; + // realm.publish polls readiness after the 202; these tests drive + // publish timing via publishDeferred, so report ready instantly. + getService('realm-server').waitForRealmReady = async () => {}; // The publish modal asks the server for the unlisted-link slug on open; // default it so the unlisted card renders a URL (not a stuck "Generating // link…") in tests that don't exercise it. Tests that do use diff --git a/packages/host/tests/integration/commands/publish-realm-test.gts b/packages/host/tests/integration/commands/publish-realm-test.gts index 70b5b3294a1..8d3ee0cbb14 100644 --- a/packages/host/tests/integration/commands/publish-realm-test.gts +++ b/packages/host/tests/integration/commands/publish-realm-test.gts @@ -97,6 +97,12 @@ module('Integration | commands | publish-realm', function (hooks) { ); }, }, + { + // `realm.publish` polls each target's readiness after the 202. + // Report ready immediately so the command resolves. + route: '_readiness-check', + getResponse: async () => new Response(null, { status: 200 }), + }, ]); setupRealmCacheTeardown(hooks); From 331486918fa56f2e3593b032c3b2c632c446940a Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Wed, 24 Jun 2026 16:02:19 -0500 Subject: [PATCH 3/8] Mount published realm on publish so it serves before indexing finishes The async publish handler returned 202 without mounting the published realm on this instance, relying on lazy-mount-on-request to serve it. That left freshly-published realms 404ing until something mounted them, which lazy-mount did not reliably do on the publishing instance. Mount the realm via reconciler.lookupOrMount without awaiting it: ensureMounted publishes the realm into virtualNetwork synchronously, so it is served as soon as the 202 returns, while start()'s from-scratch index runs in the background and clients poll _readiness-check. For a republish (already mounted) the follow-up kicks fullIndex to reindex the swapped files. Tests: index-responses reconciles before the readiness check; the host publish command test stubs waitForRealmReady at the service level (its network mock could not match subdirectory readiness pollers). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../commands/publish-realm-test.gts | 10 ++- .../handlers/handle-publish-realm.ts | 68 ++++++++++--------- .../server-endpoints/index-responses-test.ts | 14 ++-- 3 files changed, 50 insertions(+), 42 deletions(-) diff --git a/packages/host/tests/integration/commands/publish-realm-test.gts b/packages/host/tests/integration/commands/publish-realm-test.gts index 8d3ee0cbb14..f5a963a45ff 100644 --- a/packages/host/tests/integration/commands/publish-realm-test.gts +++ b/packages/host/tests/integration/commands/publish-realm-test.gts @@ -97,18 +97,16 @@ module('Integration | commands | publish-realm', function (hooks) { ); }, }, - { - // `realm.publish` polls each target's readiness after the 202. - // Report ready immediately so the command resolves. - route: '_readiness-check', - getResponse: async () => new Response(null, { status: 200 }), - }, ]); setupRealmCacheTeardown(hooks); hooks.beforeEach(async function (this: RenderingTestContext) { getOwner(this)!.register('service:realm', StubRealmService); + // realm.publish polls each target's readiness after the 202; these tests + // assert publish resolution, not readiness, and publish to URLs with no + // backing realm — so report ready instantly rather than poll the network. + getService('realm-server').waitForRealmReady = async () => {}; loader = getService('loader-service').loader; PublishTarget = ( await loader.import(`${baseRealm.url}command`) diff --git a/packages/realm-server/handlers/handle-publish-realm.ts b/packages/realm-server/handlers/handle-publish-realm.ts index 9f876f7feb0..deadb2a863b 100644 --- a/packages/realm-server/handlers/handle-publish-realm.ts +++ b/packages/realm-server/handlers/handle-publish-realm.ts @@ -614,38 +614,44 @@ export default function handlePublishRealm({ }, ); - // Indexing/prerender is not awaited in the request: the handler returns - // 202 (pending) and the client polls _readiness-check. - // Awaiting a full index + prerender (pool-bound) here would hold the HTTP - // request open for the entire indexing duration. + // Mount the published realm on this instance so it is served as soon as + // the 202 returns, but do NOT await its index/prerender — that runs in + // the background and clients poll _readiness-check. + // ensureMounted publishes the realm into virtualNetwork synchronously, so + // a request arriving right after this 202 (the readiness poll, or a + // visitor) resolves to the realm rather than 404ing; awaiting the full + // index + prerender (pool-bound) instead would hold the HTTP request open + // for the entire indexing duration. Sibling instances pick the realm up + // via the realm_registry NOTIFY and lazy-mount on their first request. // - // If this realm is already mounted on this instance (a republish), its - // #startedUp is already resolved, so _readiness-check would otherwise - // return 200 immediately — before the swapped files are reindexed. - // Kick a fire-and-forget full index here: publishFullIndex registers - // its in-flight deferred synchronously, so Realm.indexing() (which - // readinessCheck awaits) reflects this reindex until it completes. - // A single pass suffices for the correct og:title — Realm.fullIndex - // invalidates the cached RealmInfo before the pass, and parseRealmInfo - // reads the realm name from the swapped realm.json via its on-disk - // overlay, so /index re-bakes with the right name without a second - // pass. (This job coalesces with the durability enqueue above.) - // - // For a realm not mounted here (new publish, or a peer instance), the - // durability enqueue above plus lazy-mount-on-first-request handle the - // index; #startedUp resolves only after that from-scratch pass. - let mountedPublishedRealm = reconciler.mounted.get(publishedRealmURL); - if (mountedPublishedRealm) { - void mountedPublishedRealm - .fullIndex(userInitiatedPriority, { clearLastModified: true }) - .catch((err: unknown) => { - log.error( - `background publish reindex failed for ${publishedRealmURL}: ${ - err instanceof Error ? err.message : String(err) - }`, - ); - }); - } + // For a new publish, mount's start() runs a from-scratch index and + // #startedUp resolves only after it completes — readinessCheck awaits + // that. For a republish the realm is already mounted with a resolved + // #startedUp, so start() won't re-run; kick an explicit reindex of the + // swapped files. fullIndex invalidates the cached RealmInfo before the + // pass, so the og:title re-bakes from the swapped realm.json (read via + // parseRealmInfo's disk overlay) in a single pass; publishFullIndex + // registers its in-flight deferred synchronously, so Realm.indexing() + // (which readinessCheck also awaits) reflects the reindex until it + // completes. (Both index paths coalesce with the durability enqueue.) + let wasMounted = reconciler.mounted.has(publishedRealmURL); + reconciler + .lookupOrMount(publishedRealmURL) + .then((publishedRealm) => { + if (wasMounted && publishedRealm) { + return publishedRealm.fullIndex(userInitiatedPriority, { + clearLastModified: true, + }); + } + return undefined; + }) + .catch((err: unknown) => { + log.error( + `background mount/reindex failed for ${publishedRealmURL}: ${ + err instanceof Error ? err.message : String(err) + }`, + ); + }); // The source realm's `RealmInfo.lastPublishedAt` map is built // from `realm_registry` rows joined on `source_url = sourceRealmURL`, diff --git a/packages/realm-server/tests/server-endpoints/index-responses-test.ts b/packages/realm-server/tests/server-endpoints/index-responses-test.ts index 10bd8ec0e3e..606298be3ae 100644 --- a/packages/realm-server/tests/server-endpoints/index-responses-test.ts +++ b/packages/realm-server/tests/server-endpoints/index-responses-test.ts @@ -1347,6 +1347,9 @@ module(`server-endpoints/${basename(import.meta.filename)}`, function () { 'Published realm: theme icon links after _publish-realm', function (hooks) { let testRealmHttpServer: Server; + let testRealmServer: Awaited< + ReturnType + >['testRealmServer']; let request: SuperTest; let dbAdapter: PgAdapter; let dir: DirResult; @@ -1365,7 +1368,7 @@ module(`server-endpoints/${basename(import.meta.filename)}`, function () { let virtualNetwork = createVirtualNetwork(); let testRealmDir = join(dir.name, 'realm_server_theme', 'test'); ensureDirSync(testRealmDir); - ({ testRealmHttpServer } = await runTestRealmServer({ + ({ testRealmHttpServer, testRealmServer } = await runTestRealmServer({ virtualNetwork, testRealmDir, fileSystem: {}, @@ -1518,11 +1521,12 @@ module(`server-endpoints/${basename(import.meta.filename)}`, function () { ); } - // `_publish-realm` returns 202 before indexing finishes. The - // published realm's readiness check lazy-mounts it and awaits - // start() + any in-flight index, so this single request blocks until - // the swapped files are indexed and the assertions below query + // `_publish-realm` returns 202 before indexing finishes. Drive a + // reconcile pass to mount the published realm, then hit its readiness + // check, which awaits start() + any in-flight index — so this blocks + // until the swapped files are indexed and the assertions below query // indexed content. + await testRealmServer.testingOnlyReconcile(); let readinessResponse = await request .get(`${publishedRealmPath}_readiness-check`) .set('Host', publishedRealmHost); From d73a3371a6b99e406feee18ec8207f5d98079d3b Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Thu, 25 Jun 2026 07:36:28 -0500 Subject: [PATCH 4/8] Send Accept header on the published-realm readiness check in test The realm route table matches `_readiness-check` only for the RealmInfo mime type, so a readiness GET without an Accept header falls through to the card handler and 404s. Send `application/vnd.api+json`, matching the real readiness-poll client. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../tests/server-endpoints/index-responses-test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/realm-server/tests/server-endpoints/index-responses-test.ts b/packages/realm-server/tests/server-endpoints/index-responses-test.ts index 606298be3ae..61ec91bd5f9 100644 --- a/packages/realm-server/tests/server-endpoints/index-responses-test.ts +++ b/packages/realm-server/tests/server-endpoints/index-responses-test.ts @@ -1529,7 +1529,8 @@ module(`server-endpoints/${basename(import.meta.filename)}`, function () { await testRealmServer.testingOnlyReconcile(); let readinessResponse = await request .get(`${publishedRealmPath}_readiness-check`) - .set('Host', publishedRealmHost); + .set('Host', publishedRealmHost) + .set('Accept', 'application/vnd.api+json'); if (readinessResponse.status !== 200) { throw new Error( `Published realm not ready: ${readinessResponse.status} ${readinessResponse.text}`, From 29c6bf4fdac29f17de34a98b6cf7c608c9c32d1b Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Thu, 25 Jun 2026 07:44:21 -0500 Subject: [PATCH 5/8] Mock readiness check in publish-realm command tests realm.publish polls each target's _readiness-check after the 202. The command integration tests publish to URLs with no backing realm, so mock the readiness endpoint (root path for custom/pre-resolved/force targets and the subdirectory path for the derived-subdomain target) to report ready immediately. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../integration/commands/publish-realm-test.gts | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/packages/host/tests/integration/commands/publish-realm-test.gts b/packages/host/tests/integration/commands/publish-realm-test.gts index f5a963a45ff..e139164d9b2 100644 --- a/packages/host/tests/integration/commands/publish-realm-test.gts +++ b/packages/host/tests/integration/commands/publish-realm-test.gts @@ -97,16 +97,24 @@ module('Integration | commands | publish-realm', function (hooks) { ); }, }, + // `realm.publish` polls each target's `_readiness-check` after the 202. + // The endpoint matcher keys on pathname, so cover the root-domain targets + // (`/_readiness-check`) and the subdirectory target (`/my-space/...`). + // Report ready immediately so the command resolves. + { + route: '_readiness-check', + getResponse: async () => new Response(null, { status: 200 }), + }, + { + route: 'my-space/_readiness-check', + getResponse: async () => new Response(null, { status: 200 }), + }, ]); setupRealmCacheTeardown(hooks); hooks.beforeEach(async function (this: RenderingTestContext) { getOwner(this)!.register('service:realm', StubRealmService); - // realm.publish polls each target's readiness after the 202; these tests - // assert publish resolution, not readiness, and publish to URLs with no - // backing realm — so report ready instantly rather than poll the network. - getService('realm-server').waitForRealmReady = async () => {}; loader = getService('loader-service').loader; PublishTarget = ( await loader.import(`${baseRealm.url}command`) From 902399148d6d681d88a9c95dae4e04993f1321a0 Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Thu, 25 Jun 2026 07:47:19 -0500 Subject: [PATCH 6/8] Register republish reindex synchronously so readiness can't report early MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On a republish the realm is already mounted with a resolved #startedUp, so the readiness check relies on indexing() to know the swapped files are still being reindexed. Triggering that reindex behind a deferred lookupOrMount().then() left a window where a readiness poll arriving right after the 202 saw an empty indexing() and reported ready before the reindex started — so the published page served stale content. Call fullIndex directly on the already-mounted realm: publishFullIndex registers its in-flight deferred synchronously, before the handler returns, so readiness waits for the reindex. New publishes still mount via lookupOrMount, where #startedUp gates readiness. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../handlers/handle-publish-realm.ts | 54 ++++++++++++------- 1 file changed, 36 insertions(+), 18 deletions(-) diff --git a/packages/realm-server/handlers/handle-publish-realm.ts b/packages/realm-server/handlers/handle-publish-realm.ts index deadb2a863b..fe8be61bdb0 100644 --- a/packages/realm-server/handlers/handle-publish-realm.ts +++ b/packages/realm-server/handlers/handle-publish-realm.ts @@ -634,24 +634,42 @@ export default function handlePublishRealm({ // registers its in-flight deferred synchronously, so Realm.indexing() // (which readinessCheck also awaits) reflects the reindex until it // completes. (Both index paths coalesce with the durability enqueue.) - let wasMounted = reconciler.mounted.has(publishedRealmURL); - reconciler - .lookupOrMount(publishedRealmURL) - .then((publishedRealm) => { - if (wasMounted && publishedRealm) { - return publishedRealm.fullIndex(userInitiatedPriority, { - clearLastModified: true, - }); - } - return undefined; - }) - .catch((err: unknown) => { - log.error( - `background mount/reindex failed for ${publishedRealmURL}: ${ - err instanceof Error ? err.message : String(err) - }`, - ); - }); + let mountedPublishedRealm = reconciler.mounted.get(publishedRealmURL); + if (mountedPublishedRealm) { + // Republish: the realm is already mounted with a resolved #startedUp, + // so readinessCheck relies on indexing() to know the swapped files are + // still being reindexed. Call fullIndex directly (not behind a deferred + // `.then`) so publishFullIndex registers its in-flight deferred + // SYNCHRONOUSLY, before this handler returns 202 — otherwise a readiness + // poll arriving first would see an empty indexing() and report ready + // before the reindex even starts. fullIndex invalidates the cached + // RealmInfo first, so og:title re-bakes from the swapped realm.json + // (parseRealmInfo's disk overlay) in a single pass. + void mountedPublishedRealm + .fullIndex(userInitiatedPriority, { clearLastModified: true }) + .catch((err: unknown) => { + log.error( + `background publish reindex failed for ${publishedRealmURL}: ${ + err instanceof Error ? err.message : String(err) + }`, + ); + }); + } else { + // New publish (or not mounted on this instance): mount so the realm is + // served as soon as the 202 returns. ensureMounted publishes it into + // virtualNetwork synchronously; start()'s from-scratch index runs in + // the background and #startedUp resolves only after it completes, which + // readinessCheck awaits. Sibling instances lazy-mount on first request. + void reconciler + .lookupOrMount(publishedRealmURL) + .catch((err: unknown) => { + log.error( + `background mount failed for ${publishedRealmURL}: ${ + err instanceof Error ? err.message : String(err) + }`, + ); + }); + } // The source realm's `RealmInfo.lastPublishedAt` map is built // from `realm_registry` rows joined on `source_url = sourceRealmURL`, From 44996940690233525a43b992e693f87387d9e31f Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Thu, 25 Jun 2026 08:18:41 -0500 Subject: [PATCH 7/8] Reload the published tab until the republish reindex lands (CS-11043 spec) The publish handler returns 202 before indexing finishes, so the previous publish's HTML keeps being served until the reindex lands. The test opened the published URL right after the 202 and then waited on a tab that never re-fetches, so it saw the stale sentinel until timing out. Reload the tab until it serves the updated sentinel, and give the two-publish flow a larger per-attempt timeout. The load-bearing CS-11043 assertions (updated sentinel present, initial sentinel absent) are unchanged. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/matrix/tests/publish-realm.spec.ts | 30 ++++++++++++++------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/packages/matrix/tests/publish-realm.spec.ts b/packages/matrix/tests/publish-realm.spec.ts index 2f29251f666..e2a3f46dbb1 100644 --- a/packages/matrix/tests/publish-realm.spec.ts +++ b/packages/matrix/tests/publish-realm.spec.ts @@ -280,6 +280,9 @@ test.describe('Publish realm', () => { page, request, }) => { + // Two full publish+index cycles plus a readiness wait — give this E2E + // flow more than the default per-attempt budget. + test.setTimeout(120_000); // CS-11043 regression net. The bug was: a republish reported success // server-side but the published URL kept serving the previous publish's // rendered HTML, sometimes for tens of hours. Every existing @@ -463,15 +466,24 @@ test.describe('Publish realm', () => { .click(); let secondTab = await secondTabPromise; await secondTab.waitForLoadState(); - // Generous retry budget: if waitForResponse above was downgraded - // to null, the publish may not yet be done by the time we land on - // the published URL. The assertion retries until the sentinel - // appears or this budget expires, which gives slow republishes - // room to land without flapping the test. - await expect(secondTab.locator('[data-test-sentinel-output]')).toHaveText( - updatedSentinel, - { timeout: 120_000 }, - ); + // The publish handler returns 202 before the reindex finishes, and a tab + // that loaded before it landed won't re-fetch on its own — so reload until + // the published URL serves the updated sentinel. The retry budget gives the + // background reindex room to settle. + await expect + .poll( + async () => { + await secondTab.reload({ waitUntil: 'domcontentloaded' }); + return ( + (await secondTab + .locator('[data-test-sentinel-output]') + .textContent() + .catch(() => null)) ?? '' + ); + }, + { timeout: 60_000, intervals: [2_000] }, + ) + .toBe(updatedSentinel); await expect(secondTab.locator('body')).not.toContainText(initialSentinel); await secondTab.close(); await page.bringToFront(); From 8431077246742fab5438e1996ad81752546d9972 Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Thu, 25 Jun 2026 08:50:48 -0500 Subject: [PATCH 8/8] Wait for published realm_versions rows in delete-realm test Publish returns 202 before indexing finishes, so the published realm's realm_versions rows (written when its index completes) aren't present right after the reconcile mounts it. Wait for them before asserting they exist prior to deletion. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../server-endpoints/delete-realm-test.ts | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/packages/realm-server/tests/server-endpoints/delete-realm-test.ts b/packages/realm-server/tests/server-endpoints/delete-realm-test.ts index 3e4e9791f6c..44fecf1a252 100644 --- a/packages/realm-server/tests/server-endpoints/delete-realm-test.ts +++ b/packages/realm-server/tests/server-endpoints/delete-realm-test.ts @@ -14,7 +14,12 @@ import { query, } from '@cardstack/runtime-common'; -import { insertJob, insertUser, realmSecretSeed } from '../helpers/index.ts'; +import { + insertJob, + insertUser, + realmSecretSeed, + waitUntil, +} from '../helpers/index.ts'; import { createJWT as createRealmServerJWT } from '../../utils/jwt.ts'; import { setupServerEndpointsTest } from './helpers.ts'; @@ -144,6 +149,25 @@ module(`server-endpoints/${basename(import.meta.filename)}`, function (hooks) { // observe its post-DELETE unmount. await context.testRealmServer.testingOnlyReconcile(); + // Publish returns 202 before indexing finishes, and the mount kicks the + // published realm's index in the background — its realm_versions rows are + // written when that index completes. Wait for them before asserting they + // exist below. + await waitUntil( + async () => { + let rows = await context.dbAdapter.execute( + `SELECT 1 FROM realm_versions WHERE realm_url = '${publishedRealmURL}' LIMIT 1`, + ); + return rows.length > 0 ? rows : undefined; + }, + { + timeout: 30_000, + interval: 100, + timeoutMessage: + 'published realm_versions rows did not appear after publish', + }, + ); + let sourceIndexURL = `${realmURL}cleanup-${uuidv4()}.json`; let publishedIndexURL = `${publishedRealmURL}cleanup-${uuidv4()}.json`; let unrelatedIndexURL = `${unrelatedRealmURL}cleanup-${uuidv4()}.json`;