Implement the postgres post-migrated paths

.vscode/settings.json

@@ -104,5 +104,6 @@
   ],
   "yaml.schemas": {
     "https://json.schemastore.org/github-workflow.json": "file:///Users/brh/Documents/oss/dim-api/.github/workflows/deploy.yml"
-  }
+  },
+  "js/ts.tsdk.path": "node_modules/typescript/lib"
 }

api/db/global-settings-queries.ts

@@ -13,8 +13,8 @@ export async function setGlobalSettings(flavor: string, settings: Partial<Global
   return pool.query({
     name: 'set_global_settings',
     text: `
-      INSERT INTO global_settings (flavor, settings, updated_at)
-      VALUES ($1, $2, NOW())
+      INSERT INTO global_settings (flavor, settings)
+      VALUES ($1, $2)
       ON CONFLICT (flavor)
       DO UPDATE SET settings = (global_settings.settings || $2)
     `,

api/db/item-annotations-queries.test.ts

@@ -4,6 +4,7 @@ import {
   deleteItemAnnotation,
   deleteItemAnnotationList,
   getItemAnnotationsForProfile,
+  softDeleteAllItemAnnotations,
   updateItemAnnotation,
 } from './item-annotations-queries.js';
 
@@ -132,3 +133,41 @@ it('can clear tags', async () => {
     expect(annotations).toEqual([]);
   });
 });
+
+it('can soft delete all annotations and recreate them', async () => {
+  await transaction(async (client) => {
+    await updateItemAnnotation(client, bungieMembershipId, platformMembershipId, 2, {
+      id: '123456',
+      tag: 'favorite',
+      notes: 'the best',
+    });
+
+    // Verify it exists
+    let annotations = await getItemAnnotationsForProfile(client, platformMembershipId, 2);
+    expect(annotations).toEqual([
+      {
+        id: '123456',
+        tag: 'favorite',
+        notes: 'the best',
+      },
+    ]);
+
+    await softDeleteAllItemAnnotations(client, platformMembershipId, 2);
+
+    annotations = await getItemAnnotationsForProfile(client, platformMembershipId, 2);
+    expect(annotations).toEqual([]);
+
+    await updateItemAnnotation(client, bungieMembershipId, platformMembershipId, 2, {
+      id: '123456',
+      tag: 'junk',
+    });
+
+    annotations = await getItemAnnotationsForProfile(client, platformMembershipId, 2);
+    expect(annotations).toEqual([
+      {
+        id: '123456',
+        tag: 'junk',
+      },
+    ]);
+  });
+});

api/db/item-annotations-queries.ts

@@ -1,3 +1,4 @@
+import { partition } from 'es-toolkit';
 import { ClientBase, QueryResult } from 'pg';
 import { metrics } from '../metrics/index.js';
 import { DestinyVersion } from '../shapes/general.js';
@@ -7,6 +8,7 @@ interface ItemAnnotationRow {
   inventory_item_id: string;
   tag: TagValue | null;
   notes: string | null;
+  deleted_at: Date | null;
   crafted_date: Date | null;
 }
 
@@ -37,31 +39,26 @@ export async function getItemAnnotationsForProfile(
 }
 
 /**
- * Get ALL of the item annotations for a particular user across all platforms.
+ * Get all of the item annotations for a particular platform_membership_id and destiny_version that have changed since the token timestamp, including all tombstones.
  */
-export async function getAllItemAnnotationsForUser(
+export async function syncItemAnnotationsForProfile(
   client: ClientBase,
-  bungieMembershipId: number,
-): Promise<
-  {
-    platformMembershipId: string;
-    destinyVersion: DestinyVersion;
-    annotation: ItemAnnotation;
-  }[]
-> {
-  // TODO: this isn't indexed!
-  const results = await client.query<
-    ItemAnnotationRow & { platform_membership_id: string; destiny_version: DestinyVersion }
-  >({
-    name: 'get_all_item_annotations',
-    text: 'SELECT platform_membership_id, destiny_version, inventory_item_id, tag, notes, crafted_date FROM item_annotations WHERE inventory_item_id != 0 and platform_membership_id = $1 and deleted_at IS NULL',
-    values: [bungieMembershipId],
+  platformMembershipId: string,
+  destinyVersion: DestinyVersion,
+  syncTimestamp: number,
+): Promise<{ updated: ItemAnnotation[]; deletedItemIds: string[] }> {
+  const results = await client.query<ItemAnnotationRow>({
+    name: 'sync_item_annotations',
+    text: 'SELECT inventory_item_id, tag, notes, crafted_date, deleted_at FROM item_annotations WHERE platform_membership_id = $1 and destiny_version = $2 and last_updated_at > $3',
+    values: [platformMembershipId, destinyVersion, new Date(syncTimestamp)],
   });
-  return results.rows.map((row) => ({
-    platformMembershipId: row.platform_membership_id,
-    destinyVersion: row.destiny_version,
-    annotation: convertItemAnnotation(row),
-  }));
+
+  const [updatedRows, deletedRows] = partition(results.rows, (row) => row.deleted_at === null);
+
+  return {
+    updated: updatedRows.map(convertItemAnnotation),
+    deletedItemIds: deletedRows.map((row) => row.inventory_item_id),
+  };
 }
 
 function convertItemAnnotation(row: ItemAnnotationRow): ItemAnnotation {
@@ -98,10 +95,45 @@ export async function updateItemAnnotation(
   }
   const response = await client.query({
     name: 'upsert_item_annotation',
-    text: `insert INTO item_annotations (membership_id, platform_membership_id, destiny_version, inventory_item_id, tag, notes, crafted_date)
-values ($1, $2, $3, $4, (CASE WHEN $5 = 0 THEN NULL ELSE $5 END), (CASE WHEN $6 = 'clear' THEN NULL ELSE $6 END), $7)
-on conflict (platform_membership_id, inventory_item_id)
-do update set (tag, notes, crafted_date, deleted_at) = ((CASE WHEN $5 = 0 THEN NULL WHEN $5 IS NULL THEN item_annotations.tag ELSE $5 END), (CASE WHEN $6 = 'clear' THEN NULL WHEN $6 IS NULL THEN item_annotations.notes ELSE $6 END), $7, null)`,
+    text: `
+      INSERT INTO item_annotations (
+        membership_id,
+        platform_membership_id,
+        destiny_version,
+        inventory_item_id,
+        tag,
+        notes,
+        crafted_date
+      )
+      VALUES (
+        $1,
+        $2,
+        $3,
+        $4,
+        (CASE WHEN $5 = 0 THEN NULL ELSE $5 END),
+        (CASE WHEN $6 = 'clear' THEN NULL ELSE $6 END),
+        $7
+      )
+      ON CONFLICT (platform_membership_id, inventory_item_id)
+      DO UPDATE SET
+        tag = (CASE
+          WHEN $5 = 0 THEN NULL
+          WHEN $5 IS NULL THEN (CASE WHEN item_annotations.deleted_at IS NULL THEN item_annotations.tag ELSE NULL END)
+          ELSE $5
+        END),
+        notes = (CASE
+          WHEN $6 = 'clear' THEN NULL
+          WHEN $6 IS NULL THEN (CASE WHEN item_annotations.deleted_at IS NULL THEN item_annotations.notes ELSE NULL END)
+          ELSE $6
+        END),
+        crafted_date = $7,
+        deleted_at = (CASE
+          WHEN (CASE WHEN $5 = 0 THEN NULL WHEN $5 IS NULL THEN (CASE WHEN item_annotations.deleted_at IS NULL THEN item_annotations.tag ELSE NULL END) ELSE $5 END) IS NULL
+            AND (CASE WHEN $6 = 'clear' THEN NULL WHEN $6 IS NULL THEN (CASE WHEN item_annotations.deleted_at IS NULL THEN item_annotations.notes ELSE NULL END) ELSE $6 END) IS NULL
+          THEN now()
+          ELSE NULL
+        END)
+    `,
     values: [
       bungieMembershipId, // $1
       platformMembershipId, // $2
@@ -147,7 +179,7 @@ export async function deleteItemAnnotation(
 ): Promise<QueryResult> {
   return client.query({
     name: 'delete_item_annotation',
-    text: `update item_annotations set (tag, notes, deleted_at) = (null, null, now()) where platform_membership_id = $1 and inventory_item_id = $2`,
+    text: `update item_annotations set deleted_at = now() where platform_membership_id = $1 and inventory_item_id = $2 and deleted_at is null`,
     values: [platformMembershipId, inventoryItemId],
   });
 }
@@ -162,7 +194,7 @@ export async function deleteItemAnnotationList(
 ): Promise<QueryResult> {
   return client.query({
     name: 'delete_item_annotation_list',
-    text: `update item_annotations set (tag, notes, deleted_at) = (null, null, now()) where platform_membership_id = $1 and inventory_item_id::bigint = ANY($2::bigint[])`,
+    text: `update item_annotations set deleted_at = now() where platform_membership_id = $1 and inventory_item_id::bigint = ANY($2::bigint[]) and deleted_at is null`,
     values: [platformMembershipId, inventoryItemIds],
   });
 }
@@ -181,3 +213,18 @@ export async function deleteAllItemAnnotations(
     values: [bungieMembershipId],
   });
 }
+
+/**
+ * Soft-delete all item annotations for a platform (sets deleted_at timestamp for sync support).
+ */
+export async function softDeleteAllItemAnnotations(
+  client: ClientBase,
+  platformMembershipId: string,
+  destinyVersion: DestinyVersion,
+): Promise<QueryResult> {
+  return client.query({
+    name: 'soft_delete_all_item_annotations',
+    text: `update item_annotations set deleted_at = now() where platform_membership_id = $1 and destiny_version = $2 and deleted_at is null`,
+    values: [platformMembershipId, destinyVersion],
+  });
+}

api/db/item-hash-tags-queries.test.ts

@@ -3,6 +3,7 @@ import {
   deleteAllItemHashTags,
   deleteItemHashTag,
   getItemHashTagsForProfile,
+  softDeleteAllItemHashTags,
   updateItemHashTag,
 } from './item-hash-tags-queries.js';
 
@@ -111,3 +112,39 @@ it('can delete item hash tags by setting both values to null/empty', async () =>
     expect(annotations).toEqual([]);
   });
 });
+
+it('handles soft delete properly', async () => {
+  await transaction(async (client) => {
+    // Create a hash tag
+    await updateItemHashTag(client, bungieMembershipId, platformMembershipId, {
+      hash: 2926662838,
+      tag: 'favorite',
+      notes: 'the best',
+    });
+
+    let annotations = await getItemHashTagsForProfile(client, platformMembershipId);
+    expect(annotations).toHaveLength(1);
+    expect(annotations[0]).toEqual({
+      hash: 2926662838,
+      tag: 'favorite',
+      notes: 'the best',
+    });
+
+    await softDeleteAllItemHashTags(client, platformMembershipId);
+
+    annotations = await getItemHashTagsForProfile(client, platformMembershipId);
+    expect(annotations).toEqual([]);
+
+    await updateItemHashTag(client, bungieMembershipId, platformMembershipId, {
+      hash: 2926662838,
+      tag: 'keep',
+    });
+
+    annotations = await getItemHashTagsForProfile(client, platformMembershipId);
+    expect(annotations).toHaveLength(1);
+    expect(annotations[0]).toEqual({
+      hash: 2926662838,
+      tag: 'keep',
+    });
+  });
+});

api/db/item-hash-tags-queries.ts

@@ -1,3 +1,4 @@
+import { partition } from 'es-toolkit';
 import { ClientBase, QueryResult } from 'pg';
 import { metrics } from '../metrics/index.js';
 import { ItemHashTag, TagValue } from '../shapes/item-annotations.js';
@@ -7,23 +8,46 @@ interface ItemHashTagRow {
   item_hash: string;
   tag: TagValue | null;
   notes: string | null;
+  deleted_at: Date | null;
 }
 
 /**
- * Get all of the hash tags for a particular platform_membership_id and destiny_version.
+ * Get all of the hash tags for a particular platform_membership_id.
  */
 export async function getItemHashTagsForProfile(
   client: ClientBase,
   platformMembershipId: string,
 ): Promise<ItemHashTag[]> {
-  const results = await client.query({
+  const results = await client.query<ItemHashTagRow>({
     name: 'get_item_hash_tags',
     text: 'SELECT item_hash, tag, notes FROM item_hash_tags WHERE platform_membership_id = $1 and deleted_at IS NULL',
     values: [platformMembershipId],
   });
   return results.rows.map(convertItemHashTag);
 }
 
+/**
+ * Get all of the hash tags for a particular platform_membership_id that have changed since syncTimestamp, including tombstones.
+ */
+export async function syncItemHashTagsForProfile(
+  client: ClientBase,
+  platformMembershipId: string,
+  syncTimestamp: number,
+): Promise<{ updated: ItemHashTag[]; deletedItemHashes: number[] }> {
+  const results = await client.query<ItemHashTagRow>({
+    name: 'sync_item_hash_tags',
+    text: 'SELECT item_hash, tag, notes, deleted_at FROM item_hash_tags WHERE platform_membership_id = $1 and last_updated_at > $2',
+    values: [platformMembershipId, new Date(syncTimestamp)],
+  });
+
+  const [updatedRows, deletedRows] = partition(results.rows, (row) => row.deleted_at === null);
+
+  return {
+    updated: updatedRows.map(convertItemHashTag),
+    deletedItemHashes: deletedRows.map((row) => parseInt(row.item_hash, 10)),
+  };
+}
+
 function convertItemHashTag(row: ItemHashTagRow): ItemHashTag {
   const result: ItemHashTag = {
     hash: parseInt(row.item_hash, 10),
@@ -55,10 +79,40 @@ export async function updateItemHashTag(
 
   const response = await client.query({
     name: 'upsert_hash_tag',
-    text: `insert INTO item_hash_tags (membership_id, platform_membership_id, item_hash, tag, notes)
-values ($1, $2, $3, (CASE WHEN $4 = 0 THEN NULL ELSE $4 END), (CASE WHEN $5 = 'clear' THEN NULL ELSE $5 END))
-on conflict (platform_membership_id, item_hash)
-do update set (tag, notes, deleted_at) = ((CASE WHEN $4 = 0 THEN NULL WHEN $4 IS NULL THEN item_hash_tags.tag ELSE $4 END), (CASE WHEN $5 = 'clear' THEN NULL WHEN $5 IS NULL THEN item_hash_tags.notes ELSE $5 END), null)`,
+    text: `
+      INSERT INTO item_hash_tags (
+        membership_id,
+        platform_membership_id,
+        item_hash,
+        tag,
+        notes
+      )
+      VALUES (
+        $1,
+        $2,
+        $3,
+        (CASE WHEN $4 = 0 THEN NULL ELSE $4 END),
+        (CASE WHEN $5 = 'clear' THEN NULL ELSE $5 END)
+      )
+      ON CONFLICT (platform_membership_id, item_hash)
+      DO UPDATE SET
+        tag = (CASE
+          WHEN $4 = 0 THEN NULL
+          WHEN $4 IS NULL THEN (CASE WHEN item_hash_tags.deleted_at IS NULL THEN item_hash_tags.tag ELSE NULL END)
+          ELSE $4
+        END),
+        notes = (CASE
+          WHEN $5 = 'clear' THEN NULL
+          WHEN $5 IS NULL THEN (CASE WHEN item_hash_tags.deleted_at IS NULL THEN item_hash_tags.notes ELSE NULL END)
+          ELSE $5
+        END),
+        deleted_at = (CASE
+          WHEN (CASE WHEN $4 = 0 THEN NULL WHEN $4 IS NULL THEN (CASE WHEN item_hash_tags.deleted_at IS NULL THEN item_hash_tags.tag ELSE NULL END) ELSE $4 END) IS NULL
+            AND (CASE WHEN $5 = 'clear' THEN NULL WHEN $5 IS NULL THEN (CASE WHEN item_hash_tags.deleted_at IS NULL THEN item_hash_tags.notes ELSE NULL END) ELSE $5 END) IS NULL
+          THEN now()
+          ELSE NULL
+        END)
+    `,
     values: [
       bungieMembershipId,
       platformMembershipId,
@@ -102,7 +156,7 @@ export async function deleteItemHashTag(
 ): Promise<QueryResult> {
   return client.query({
     name: 'delete_item_hash_tag',
-    text: `update item_hash_tags set (tag, notes, deleted_at) = (null, null, now()) where platform_membership_id = $1 and item_hash = $2`,
+    text: `update item_hash_tags set deleted_at = now() where platform_membership_id = $1 and item_hash = $2 and deleted_at is null`,
     values: [platformMembershipId, itemHash],
   });
 }
@@ -120,3 +174,17 @@ export async function deleteAllItemHashTags(
     values: [platformMembershipId],
   });
 }
+
+/**
+ * Soft-delete all item hash tags for a platform (sets deleted_at timestamp for sync support).
+ */
+export async function softDeleteAllItemHashTags(
+  client: ClientBase,
+  platformMembershipId: string,
+): Promise<QueryResult> {
+  return client.query({
+    name: 'soft_delete_all_item_hash_tags',
+    text: `update item_hash_tags set deleted_at = now() where platform_membership_id = $1 and deleted_at is null`,
+    values: [platformMembershipId],
+  });
+}