Lazy migration on host boot — seed app.boxel.realm-servers from legacy realms

packages/host/app/services/matrix-service.ts

@@ -182,6 +182,12 @@ export default class MatrixService extends Service {
   // gains content at runtime. Login-related side effects (`loginToRealms`,
   // `loadMoreAuthRooms`) still run regardless.
   private trustedRealmServersAuthoritative = false;
+  // Sticky for the lifetime of this instance once a boot assembles from the
+  // legacy `app.boxel.realms` list. Keeps later start() calls on the legacy
+  // path even after the lazy migration writes `app.boxel.realm-servers`, so
+  // the migration only takes effect on the next fresh session. Reset by
+  // resetState() so a logout/login re-evaluates against the persisted key.
+  private bootedFromLegacyRealmsList = false;
   @tracked private _currentRoomId: string | undefined;
   @tracked private timelineLoadingState: Map<string, boolean> =
     new TrackedMap();
@@ -421,6 +427,18 @@ export default class MatrixService extends Service {
               break;
             }
             case APP_BOXEL_REALM_SERVERS_EVENT_TYPE: {
+              // A session that booted from the legacy `app.boxel.realms` list
+              // stays on the legacy path for the lifetime of this instance
+              // (see `bootedFromLegacyRealmsList` in start()). The boot-time
+              // lazy migration writes `app.boxel.realm-servers`, and that write
+              // echoes back here — both synchronously and again when
+              // startClient()'s initial sync re-emits account data. Ignoring
+              // these keeps the migrated key from re-running trusted-servers
+              // assembly mid-boot and overwriting the legacy-assembled realm
+              // list; the new key only takes effect on the next fresh session.
+              if (this.bootedFromLegacyRealmsList) {
+                break;
+              }
               let realmServers = e.event.content.realmServers as string[];
               this.trustedRealmServersAuthoritative = realmServers.length > 0;
               if (this.trustedRealmServersAuthoritative) {
@@ -477,6 +495,17 @@ export default class MatrixService extends Service {
     };
   }
 
+  // Test-only diagnostic exposing which boot path the current session is on.
+  // A legacy-booted session must stay non-authoritative even after the lazy
+  // migration writes `app.boxel.realm-servers` and that write echoes back
+  // through the AccountData listener. No production caller.
+  get bootAssemblyDebug() {
+    return {
+      trustedRealmServersAuthoritative: this.trustedRealmServersAuthoritative,
+      bootedFromLegacyRealmsList: this.bootedFromLegacyRealmsList,
+    };
+  }
+
   private get client() {
     if (!this._client) {
       throw new Error(`cannot use matrix client before matrix SDK has loaded`);
@@ -903,26 +932,76 @@ export default class MatrixService extends Service {
         // the lazy migration that populates `app.boxel.realm-servers` has
         // run on all active accounts.
         let trustedServers = realmServersData?.realmServers ?? [];
+        // A session that first assembled from the legacy `app.boxel.realms`
+        // list stays on the legacy path for the lifetime of this
+        // MatrixService instance. The lazy migration below persists
+        // `app.boxel.realm-servers` for the next fresh session; switching
+        // this same instance to the trusted path on a later start() (e.g. a
+        // test that re-boots to pick up a newly-added realm) would re-derive
+        // the realm list from `_realm-auth` for no benefit and drop realms
+        // that the trusted servers don't advertise.
+        let useTrustedServers =
+          trustedServers.length > 0 && !this.bootedFromLegacyRealmsList;
         // The legacy `app.boxel.realms` AccountData event is re-emitted by
         // the matrix sync that runs inside `startClient()` below. Setting
         // this flag here makes that re-emission a no-op for the available-
         // realms list — the realm-servers path is the authoritative source.
-        this.trustedRealmServersAuthoritative = trustedServers.length > 0;
+        this.trustedRealmServersAuthoritative = useTrustedServers;
         let userRealmURLs: string[];
-        if (trustedServers.length > 0) {
+        if (useTrustedServers) {
           if (isTesting())
             console.warn('[start-phase] fetchUserRealmsFromTrustedServers');
           userRealmURLs =
             await this.realmServer.fetchUserRealmsFromTrustedServers(
               trustedServers,
             );
         } else {
+          this.bootedFromLegacyRealmsList = true;
           if (isTesting())
             console.warn('[start-phase] getAccountData(realms-legacy)');
           let legacyRealmsData = (await this.client.getAccountDataFromServer(
             APP_BOXEL_REALMS_EVENT_TYPE,
           )) as { realms: string[] } | null;
           userRealmURLs = legacyRealmsData?.realms ?? [];
+
+          // Lazy migration: this account has no `app.boxel.realm-servers`
+          // entry (the key was absent or empty, so boot fell back to the
+          // legacy realm list above). Seed the new key with the realm-server
+          // backing the user's existing realms so subsequent boots take the
+          // authoritative trusted-servers assembly path. We use
+          // `getRealmServersForRealms`, which derives the server from each
+          // realm's JWT `realmServerURL` claim and falls back to this host's
+          // own realm server — never the bare realm-URL origin. That matters
+          // because a realm URL's origin can differ from its realm server
+          // (e.g. the shared base realm at cardstack.com); persisting such a
+          // foreign origin would make the next boot's `assertOwnRealmServer`
+          // reject the list and log the user out. The legacy
+          // `app.boxel.realms` key is intentionally retained for rollback
+          // safety. Gated on `trustedServers` being genuinely empty so a
+          // re-boot of this same legacy session (where the key we just wrote
+          // is now present) doesn't re-write it. A no-op for an account with
+          // no realms. Best-effort: a failure must not break boot.
+          if (trustedServers.length === 0 && userRealmURLs.length > 0) {
+            try {
+              let derivedRealmServers =
+                this.realmServer.getRealmServersForRealms(userRealmURLs);
+              if (derivedRealmServers.length > 0) {
+                if (isTesting())
+                  console.warn('[start-phase] migrateRealmServersAccountData');
+                // `bootedFromLegacyRealmsList` is already set above, so the
+                // AccountData listener ignores both this self-write and the
+                // echo from startClient()'s initial sync — no extra guard
+                // needed. This session is already assembled from the legacy
+                // list; the new key takes effect on the next boot.
+                await this.setRealmServersInAccountData(derivedRealmServers);
+              }
+            } catch (err) {
+              console.error(
+                'Failed to migrate legacy realms to app.boxel.realm-servers account data',
+                err,
+              );
+            }
+          }
         }
 
         let noRealmsLoggedIn = Array.from(this.realm.realms.entries()).every(
@@ -1784,6 +1863,7 @@ export default class MatrixService extends Service {
       );
     }
     this.postLoginCompleted = false;
+    this.bootedFromLegacyRealmsList = false;
     this._isLoadingMoreAIRooms = false;
     this.messagesToSend.clear();
     this.cardsToSend.clear();

packages/host/app/services/realm-server.ts

@@ -332,15 +332,7 @@ export default class RealmServerService extends Service {
     let testRealmOrigin = isTesting()
       ? new URL(testRealmURL).origin
       : undefined;
-    let sessionTokens: Record<string, string> = {};
-    let sessionStr =
-      window.localStorage.getItem(SessionLocalStorageKey) ?? '{}';
-
-    try {
-      sessionTokens = JSON.parse(sessionStr) as Record<string, string>;
-    } catch {
-      sessionTokens = {};
-    }
+    let sessionTokens = this.readSessionTokens();
 
     let realmServerURLs = new Set<string>();
 
@@ -372,6 +364,16 @@ export default class RealmServerService extends Service {
     return [...realmServerURLs];
   }
 
+  private readSessionTokens(): Record<string, string> {
+    let sessionStr =
+      window.localStorage.getItem(SessionLocalStorageKey) ?? '{}';
+    try {
+      return JSON.parse(sessionStr) as Record<string, string>;
+    } catch {
+      return {};
+    }
+  }
+
   private normalizeRealmServerURL(url: string): string {
     let normalizedURL = ensureTrailingSlash(url);
     if (isTesting()) {
@@ -1223,16 +1225,7 @@ export default class RealmServerService extends Service {
   }
 
   private getRealmTokenForRealms(realms: string[]): string | undefined {
-    let sessionTokens: Record<string, string> = {};
-    let sessionStr = window.localStorage.getItem(SessionLocalStorageKey);
-    if (!sessionStr) {
-      return undefined;
-    }
-    try {
-      sessionTokens = JSON.parse(sessionStr) as Record<string, string>;
-    } catch {
-      return undefined;
-    }
+    let sessionTokens = this.readSessionTokens();
     for (let realmURL of realms) {
       let normalizedRealmURL = ensureTrailingSlash(realmURL);
       let token = sessionTokens[normalizedRealmURL] ?? sessionTokens[realmURL];

packages/host/tests/helpers/mock-matrix/_client.ts

@@ -145,6 +145,11 @@ export class MockClient implements ExtendedClient {
       }),
     );
 
+    // The real matrix initial sync re-emits every account-data key, not just
+    // the legacy realms list. Re-emit both so boot-path handlers see the same
+    // echoes they would in production — notably the `app.boxel.realm-servers`
+    // echo a lazy-migration boot triggers by writing that key just before
+    // startClient().
     this.emitEvent(
       new MatrixEvent({
         type: APP_BOXEL_REALMS_EVENT_TYPE,
@@ -153,6 +158,14 @@ export class MockClient implements ExtendedClient {
         },
       }),
     );
+    this.emitEvent(
+      new MatrixEvent({
+        type: APP_BOXEL_REALM_SERVERS_EVENT_TYPE,
+        content: {
+          realmServers: this.sdkOpts.activeRealmServers ?? [],
+        },
+      }),
+    );
   }
 
   async createRealmSession(realmURL: URL): Promise<string> {

packages/host/tests/helpers/mock-matrix/_utils.ts

@@ -55,6 +55,14 @@ export class MockUtils {
     return this.testState.opts?.systemCardAccountData;
   };
 
+  getActiveRealms = () => {
+    return this.testState.opts?.activeRealms ?? [];
+  };
+
+  getActiveRealmServers = () => {
+    return this.testState.opts?.activeRealmServers ?? [];
+  };
+
   getRealmEventMessagesSince = (roomId: string, since: number) => {
     return this.testState
       .sdk!.serverState.getRoomEvents(roomId)

packages/host/tests/integration/matrix-service-boot-assembly-test.ts

@@ -22,6 +22,12 @@ import { setupMockMatrix } from '../helpers/mock-matrix';
 import { setupRenderingTest } from '../helpers/setup';
 
 const testRealmServerURL = ensureTrailingSlash(ENV.realmServerURL);
+// The realm-server service normalizes the test-realm origin onto the base
+// realm origin (see `normalizeRealmServerURL`), so a realm rooted at the
+// test-realm origin resolves to this canonical server identity.
+const normalizedTrustedServerURL = ensureTrailingSlash(
+  new URL(ENV.resolvedBaseRealmURL).origin,
+);
 
 // Boot assembles the available-realms list from the user's trusted
 // realm-servers (`app.boxel.realm-servers`) by asking each via
@@ -182,3 +188,149 @@ module(
     });
   },
 );
+
+// Lazy migration on host boot. An account with no `app.boxel.realm-servers`
+// entry (only `app.boxel.realms` set) has the new key seeded on next boot
+// with the realm-server backing its existing realms (derived via JWT
+// `realmServerURL` claim / own-server fallback, never the bare realm-URL
+// origin). The legacy key is retained for rollback safety.
+module(
+  'Integration | matrix-service | lazy migration seeds realm-servers',
+  function (hooks) {
+    setupRenderingTest(hooks);
+    setupBaseRealm(hooks);
+    setupLocalIndexing(hooks);
+
+    // Only the legacy `app.boxel.realms` key is set (no activeRealmServers),
+    // matching an unmigrated account.
+    let mockMatrixUtils = setupMockMatrix(hooks, {
+      loggedInAs: '@testuser:localhost',
+      activeRealms: [testRealmURL],
+    });
+
+    hooks.beforeEach(async function (this: RenderingTestContext) {
+      await setupIntegrationTestRealm({
+        mockMatrixUtils,
+        contents: {},
+        startMatrix: false,
+      });
+      let realmServer = getService('realm-server') as RealmServerService;
+      await realmServer.setAvailableRealmIdentifiers([]);
+      let matrixService = getService('matrix-service') as MatrixService;
+      await matrixService.ready;
+      await matrixService.start();
+    });
+
+    test('boot writes `app.boxel.realm-servers` with the backing realm-server', async function (assert) {
+      let matrixService = getService('matrix-service') as MatrixService;
+      let realmServers = await matrixService.getRealmServersFromAccountData();
+      assert.deepEqual(
+        realmServers,
+        [normalizedTrustedServerURL],
+        'the realm-server backing testRealmURL is persisted (own server, not the base-realm origin)',
+      );
+    });
+
+    test('boot retains the legacy `app.boxel.realms` key', async function (assert) {
+      assert.deepEqual(
+        mockMatrixUtils.getActiveRealms(),
+        [testRealmURL],
+        'app.boxel.realms is left intact for rollback safety',
+      );
+    });
+
+    test('boot still assembles the available realms', async function (assert) {
+      let realmServer = getService('realm-server') as RealmServerService;
+      assert.ok(
+        realmServer.availableRealmIdentifiers.includes(ri(testRealmURL)),
+        'testRealmURL is present in availableRealmIdentifiers',
+      );
+    });
+
+    test('the migration self-write echo does not flip the boot to the trusted path', async function (assert) {
+      // The migration writes `app.boxel.realm-servers` during start(), and that
+      // write echoes back through the AccountData listener twice: synchronously
+      // from setAccountData, and again when startClient()'s initial sync
+      // re-emits every account-data key. Neither echo may switch this
+      // legacy-booted session to the authoritative trusted-servers path —
+      // doing so would re-derive the realm list from `_realm-auth` and could
+      // drop realms the trusted servers don't advertise.
+      let matrixService = getService('matrix-service') as MatrixService;
+      let realmServer = getService('realm-server') as RealmServerService;
+
+      assert.deepEqual(
+        matrixService.bootAssemblyDebug,
+        {
+          trustedRealmServersAuthoritative: false,
+          bootedFromLegacyRealmsList: true,
+        },
+        'the session stays on the legacy path despite the realm-servers echo',
+      );
+      assert.ok(
+        realmServer.availableRealmIdentifiers.includes(ri(testRealmURL)),
+        'the legacy-assembled realm survives the echo',
+      );
+    });
+
+    test('a re-boot of the same session after migration stays on the legacy path', async function (assert) {
+      let matrixService = getService('matrix-service') as MatrixService;
+      let realmServer = getService('realm-server') as RealmServerService;
+
+      // The first boot (beforeEach) migrated and wrote `app.boxel.realm-servers`.
+      // Re-booting the same MatrixService instance must keep assembling from the
+      // legacy realm list rather than switching to the trusted-servers path,
+      // which would re-derive the list from `_realm-auth`. The migration only
+      // takes effect on the next fresh session.
+      await matrixService.start();
+
+      assert.deepEqual(
+        await matrixService.getRealmServersFromAccountData(),
+        [normalizedTrustedServerURL],
+        'realm-servers stays as migrated — not re-derived or duplicated',
+      );
+      assert.ok(
+        realmServer.availableRealmIdentifiers.includes(ri(testRealmURL)),
+        'testRealmURL from the legacy list survives the re-boot',
+      );
+    });
+  },
+);
+
+module(
+  'Integration | matrix-service | already-migrated account is untouched',
+  function (hooks) {
+    setupRenderingTest(hooks);
+    setupBaseRealm(hooks);
+    setupLocalIndexing(hooks);
+
+    // `app.boxel.realm-servers` is already populated, so boot takes the
+    // trusted-servers path and the migration must not run.
+    let mockMatrixUtils = setupMockMatrix(hooks, {
+      loggedInAs: '@testuser:localhost',
+      activeRealms: [testRealmURL],
+      activeRealmServers: [testRealmServerURL],
+    });
+
+    hooks.beforeEach(async function (this: RenderingTestContext) {
+      await setupIntegrationTestRealm({
+        mockMatrixUtils,
+        contents: {},
+        startMatrix: false,
+      });
+      let realmServer = getService('realm-server') as RealmServerService;
+      await realmServer.setAvailableRealmIdentifiers([]);
+      let matrixService = getService('matrix-service') as MatrixService;
+      await matrixService.ready;
+      await matrixService.start();
+    });
+
+    test('boot leaves `app.boxel.realm-servers` unchanged', async function (assert) {
+      let matrixService = getService('matrix-service') as MatrixService;
+      assert.deepEqual(
+        await matrixService.getRealmServersFromAccountData(),
+        [testRealmServerURL],
+        'the existing realm-servers list is neither rewritten nor duplicated',
+      );
+    });
+  },
+);