Turn on settings automigration for everyone (#295)

* Turn on settings automigration for everyone

* Cleanup

* Fix type

* Prefer postgres over stately?

* Typing

* Really prefer postgres

* Don't worry about missing tokens

* More test

* Fix import

* More robust token handling

Author: bhollis

api/db/settings-queries.ts

@@ -9,7 +9,7 @@ export async function getSettings(
   bungieMembershipId: number,
 ): Promise<{ settings: Partial<Settings>; deleted: boolean; lastModifiedAt: number } | undefined> {
   const results = await client.query<{
-    settings: Settings;
+    settings: Partial<Settings>;
     deleted_at: Date | null;
     last_updated_at: Date;
   }>({

api/routes/import.ts

@@ -1,6 +1,8 @@
 import { uniqBy } from 'es-toolkit';
 import { isEmpty } from 'es-toolkit/compat';
 import asyncHandler from 'express-async-handler';
+import { transaction } from '../db/index.js';
+import { replaceSettings } from '../db/settings-queries.js';
 import { ExportResponse } from '../shapes/export.js';
 import { DestinyVersion } from '../shapes/general.js';
 import { ImportResponse } from '../shapes/import.js';
@@ -15,7 +17,6 @@ import { importTags } from '../stately/item-annotations-queries.js';
 import { importHashTags } from '../stately/item-hash-tags-queries.js';
 import { importLoadouts } from '../stately/loadouts-queries.js';
 import { importSearches } from '../stately/searches-queries.js';
-import { convertToStatelyItem } from '../stately/settings-queries.js';
 import { batches } from '../stately/stately-utils.js';
 import { importTriumphs } from '../stately/triumphs-queries.js';
 import { badRequest, delay, subtractObject } from '../utils.js';
@@ -119,11 +120,6 @@ export async function statelyImport(
   let numTriumphs = 0;
   await deleteAllDataForUser(bungieMembershipId, platformMembershipIds);
 
-  const settingsItem = convertToStatelyItem(
-    { ...defaultSettings, ...settings },
-    bungieMembershipId,
-  );
-
   // The export will have duplicates because import saved to each profile
   // instead of the one that was exported.
   itemHashTags = uniqBy(itemHashTags, (a) => a.hash);
@@ -152,10 +148,9 @@ export async function statelyImport(
     items.push(...importSearches(platformMembershipId, searches));
   }
 
-  // Put the settings in first since it's in a different group
-  await client.put({
-    item: settingsItem,
-  });
+  // Settings live in Postgres now
+  await transaction(async (client) => replaceSettings(client, bungieMembershipId, settings));
+
   // OK now put them in as fast as we can
   for (const batch of batches(items)) {
     // We shouldn't have any existing items...

api/routes/profile.ts

@@ -147,7 +147,10 @@ function extractSyncToken(syncTokenParam: string | undefined) {
       const tokenMap = JSON.parse(syncTokenParam) as { [component: string]: string | number };
       return Object.entries(tokenMap).reduce<{ [component: string]: Buffer | number }>(
         (acc, [component, token]) => {
-          acc[component] = typeof token === 'string' ? Buffer.from(token, 'base64') : token;
+          acc[component] =
+            typeof token === 'string' && !/^\d+$/.exec(token)
+              ? Buffer.from(token, 'base64')
+              : Number(token);
           return acc;
         },
         {},
@@ -172,7 +175,7 @@ async function statelyProfile(
   };
   const timerPrefix = response.sync ? 'profileSync' : 'profileStately';
   const counterPrefix = response.sync ? 'sync' : 'stately';
-  const syncTokens: { [component: string]: string } = {};
+  const syncTokens: { [component: string]: string | number } = {};
   const addSyncToken = (
     name: string,
     token: ListToken | { canSync: boolean; tokenData: number },
@@ -181,14 +184,14 @@ async function statelyProfile(
       syncTokens[name] =
         token.tokenData instanceof Uint8Array
           ? Buffer.from(token.tokenData).toString('base64')
-          : token.tokenData.toString();
+          : token.tokenData;
     }
   };
   const getSyncToken = <T extends number | Buffer>(name: string) => {
     const tokenData = incomingSyncTokens?.[name];
-    if (incomingSyncTokens && !tokenData) {
-      throw new Error(`Missing sync token: ${name}`);
-    }
+    // if (incomingSyncTokens && !tokenData) {
+    //   throw new Error(`Missing sync token: ${name}`);
+    // }
     return tokenData as T | undefined;
   };
 
@@ -202,29 +205,22 @@ async function statelyProfile(
         // Load settings from Stately. If they're there, you're done. Otherwise load from Postgres.
         const start = new Date();
 
-        const statelySettings = await querySettings(bungieMembershipId);
-        if (!statelySettings.settings) {
-          const now = Date.now();
-          const pgSettings = await readTransaction(async (pgClient) =>
-            getSettings(pgClient, bungieMembershipId),
-          );
-          if (pgSettings) {
-            const tokenData = getSyncToken<number>('s');
-            if (tokenData === undefined || pgSettings.lastModifiedAt > tokenData) {
-              response.settings = { ...defaultSettings, ...pgSettings.settings };
-            }
-          } else {
-            response.settings = defaultSettings;
+        const now = Date.now();
+        // TODO: Should add the token to the query to avoid fetching if unchanged
+        const pgSettings = await readTransaction(async (pgClient) =>
+          getSettings(pgClient, bungieMembershipId),
+        );
+        if (pgSettings) {
+          const tokenData = getSyncToken<number>('s');
+          if (tokenData === undefined || pgSettings.lastModifiedAt > tokenData) {
+            response.settings = { ...defaultSettings, ...pgSettings.settings };
           }
           addSyncToken('s', { canSync: true, tokenData: pgSettings?.lastModifiedAt ?? now });
         } else {
           const tokenData = getSyncToken<Buffer>('settings');
           const { settings: storedSettings, token: settingsToken } = tokenData
             ? await syncSettings(tokenData)
-            : {
-                settings: statelySettings.settings ?? defaultSettings,
-                token: statelySettings.token,
-              };
+            : await querySettings(bungieMembershipId);
           response.settings = storedSettings;
           addSyncToken('settings', settingsToken);
         }
@@ -335,6 +331,6 @@ async function statelyProfile(
   return response;
 }
 
-function serializeSyncToken(syncTokens: { [component: string]: string }) {
+function serializeSyncToken(syncTokens: { [component: string]: string | number }) {
   return JSON.stringify(syncTokens);
 }

api/routes/update.ts

@@ -2,9 +2,13 @@ import { captureMessage } from '@sentry/node';
 import { chunk, groupBy, partition, sortBy } from 'es-toolkit';
 import express from 'express';
 import asyncHandler from 'express-async-handler';
-import { transaction } from '../db/index.js';
+import { readTransaction, transaction } from '../db/index.js';
 import { backfillMigrationState } from '../db/migration-state-queries.js';
-import { replaceSettings, setSetting as setSettingInPostgres } from '../db/settings-queries.js';
+import {
+  getSettings,
+  replaceSettings,
+  setSetting as setSettingInPostgres,
+} from '../db/settings-queries.js';
 import { metrics } from '../metrics/index.js';
 import { ApiApp } from '../shapes/app.js';
 import { DestinyVersion } from '../shapes/general.js';
@@ -43,11 +47,7 @@ import {
   UpdateSearch,
   updateSearches,
 } from '../stately/searches-queries.js';
-import {
-  deleteSettings as deleteSettingsInStately,
-  getSettingsForUpdate,
-  setSetting as setSettingInStately,
-} from '../stately/settings-queries.js';
+import { getSettingsForUpdate, keyFor } from '../stately/settings-queries.js';
 import { trackUntrackTriumphs } from '../stately/triumphs-queries.js';
 import {
   badRequest,
@@ -313,9 +313,16 @@ async function statelyUpdate(
               mergedSettings = { ...mergedSettings, ...update.payload };
             }
 
-            if (bungieMembershipId === 7094) {
+            // TODO: Remove the check for settings in Postgres once we're fully migrated off Stately
+            const pgSettings = await readTransaction((client) =>
+              getSettings(client, bungieMembershipId),
+            );
+            if (pgSettings) {
+              await transaction(async (pgClient) => {
+                await setSettingInPostgres(pgClient, bungieMembershipId, mergedSettings);
+              });
+            } else {
               const statelySettings = await getSettingsForUpdate(txn, bungieMembershipId);
-
               if (statelySettings) {
                 mergedSettings = { ...statelySettings, ...mergedSettings };
                 await transaction(async (pgClient) => {
@@ -325,14 +332,12 @@ async function statelyUpdate(
                     subtractObject(mergedSettings, defaultSettings),
                   );
                 });
-                await deleteSettingsInStately(bungieMembershipId);
+                await txn.del(keyFor(bungieMembershipId));
               } else {
                 await transaction(async (pgClient) => {
                   await setSettingInPostgres(pgClient, bungieMembershipId, mergedSettings);
                 });
               }
-            } else {
-              await setSettingInStately(txn, bungieMembershipId, mergedSettings);
             }
             break;
           }

api/server.test.ts

@@ -210,11 +210,40 @@ describe('profile', () => {
 
     expect(profileSyncResponse.syncToken).toBeDefined();
     expect(profileSyncResponse.syncToken).not.toBe(profileResponse.syncToken);
+    expect(profileSyncResponse.syncToken).toContain('"s":');
     expect(profileSyncResponse.sync).toBe(true);
     expect(profileSyncResponse.settings?.showNewItems).toBe(true);
     expect(profileSyncResponse.tags?.length).toBe(1);
     expect(profileSyncResponse.tags?.[0].id).toBe('1234');
     expect(profileSyncResponse.deletedLoadoutIds?.length).toBe(1);
+
+    const request2: ProfileUpdateRequest = {
+      platformMembershipId,
+      destinyVersion: 2,
+      updates: [
+        {
+          action: 'setting',
+          payload: {
+            compareBaseStats: true,
+          },
+        },
+      ],
+    };
+    await postRequestAuthed('/profile', request2).expect(200);
+
+    const profileSyncResponse2 = (await getRequestAuthed(
+      `/profile?components=settings,loadouts,tags,triumphs,searches,hashtags&platformMembershipId=${platformMembershipId}&sync=${encodeURIComponent(profileSyncResponse.syncToken!)}`,
+    )
+      .expect(200)
+      .json()) as ProfileResponse;
+
+    expect(profileSyncResponse2.syncToken).toBeDefined();
+    expect(profileSyncResponse2.syncToken).not.toBe(profileSyncResponse.syncToken);
+    expect(profileSyncResponse2.syncToken).toContain('"s":');
+    expect(profileSyncResponse2.sync).toBe(true);
+    expect(profileSyncResponse2.settings).toBeDefined();
+    expect(profileSyncResponse.settings?.compareBaseStats).toBe(true);
+    expect(profileSyncResponse2.settings?.showNewItems).toBe(true);
   });
 
   it('can retrieve only settings, without needing a platform membership ID', async () => {

api/stately/bulk-queries.ts

@@ -6,7 +6,7 @@ import { DeleteAllResponse } from '../shapes/delete-all.js';
 import { ExportResponse } from '../shapes/export.js';
 import { DestinyVersion } from '../shapes/general.js';
 import { ProfileResponse } from '../shapes/profile.js';
-import { defaultSettings } from '../shapes/settings.js';
+import { defaultSettings, Settings } from '../shapes/settings.js';
 import { delay, subtractObject } from '../utils.js';
 import { client } from './client.js';
 import { AnyItem } from './generated/index.js';
@@ -115,12 +115,14 @@ export async function exportDataForUser(
   bungieMembershipId: number,
   platformMembershipIds: string[],
 ): Promise<ExportResponse> {
-  let settings = await getSettings(bungieMembershipId);
-  if (!settings) {
-    const pgSettings = await readTransaction((client) =>
-      getSettingsFromPostgres(client, bungieMembershipId),
-    );
-    settings = { ...defaultSettings, ...pgSettings?.settings };
+  let settings: Settings;
+  const pgSettings = await readTransaction((client) =>
+    getSettingsFromPostgres(client, bungieMembershipId),
+  );
+  if (pgSettings) {
+    settings = { ...defaultSettings, ...pgSettings.settings };
+  } else {
+    settings = (await getSettings(bungieMembershipId)) ?? defaultSettings;
   }
 
   const responses = await Promise.all(platformMembershipIds.map((p) => exportDataForProfile(p)));

api/utils.ts

@@ -1,4 +1,4 @@
-import { camelCase, mapKeys } from 'es-toolkit';
+import { camelCase, isEqual, mapKeys } from 'es-toolkit';
 import { Response } from 'express';
 
 /**
@@ -141,7 +141,7 @@ export function subtractObject<T extends object>(obj: Partial<T>, defaults: T):
   const result: Partial<T> = {};
   if (obj) {
     for (const key in defaults) {
-      if (obj[key] !== undefined && obj[key] !== defaults[key]) {
+      if (obj[key] !== undefined && !isEqual(obj[key], defaults[key])) {
         result[key] = obj[key];
       }
     }