Implement conversation logging feature and privacy opt-out enhancements (#79)

Author: ajay-bhargava

.github/workflows/staging.yml

@@ -35,5 +35,5 @@ jobs:
       run: bun install
 
     - name: Run tests
-      run: bun run test
+      run: bunx turbo run test --filter='!python-tools'
 

.github/workflows/update-dependencies.yml

@@ -0,0 +1,57 @@
+name: Update Dependencies
+
+on:
+  schedule:
+    # Runs every other Monday at 9am UTC (biweekly check)
+    - cron: '0 9 * * 1'
+  workflow_dispatch:
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  update-deps:
+    runs-on: ubuntu-latest
+    if: github.event_name == 'workflow_dispatch' || (github.event_name == 'schedule' && (github.run_number % 2 == 0))
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v5
+        with:
+          ref: staging
+
+      - name: Setup Bun
+        uses: oven-sh/setup-bun@v2
+        with:
+          bun-version: latest
+
+      - name: Update dependencies with taze
+        run: bunx taze -r -w
+
+      - name: Install updated dependencies
+        run: bun install
+
+      - name: Check for changes
+        id: changes
+        run: |
+          if git diff --quiet; then
+            echo "has_changes=false" >> $GITHUB_OUTPUT
+          else
+            echo "has_changes=true" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Create Pull Request
+        if: steps.changes.outputs.has_changes == 'true'
+        uses: peter-evans/create-pull-request@v7
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          branch: chore/update-dependencies
+          base: staging
+          commit-message: 'chore: update dependencies'
+          title: 'chore: update dependencies'
+          body: |
+            Automated dependency update via `bunx taze -r -w`.
+
+            Please review the changes and ensure tests pass before merging.
+          labels: dependencies
+          delete-branch: true

AGENTS.md

@@ -400,3 +400,56 @@ Environment Variables & Port Configuration
 - Add ngrok domains to `ALLOWED_ORIGINS` for external dev access
 
 Done. This guide covers the minimal, repeatable steps for adding new tools and flows with token‑efficient prompts and predictable UX.
+
+Conversation Logging (ML Training Data)
+
+The system captures all user transcripts and LLM responses to Convex for ML training and improvement.
+
+**Architecture:**
+- **Schema**: `packages/convex/schema.ts` → `conversationLogs` table
+- **Convex mutations**: `packages/convex/conversationLogs.ts` → `logConversation`, `updateResponse`
+- **Application utility**: `apps/application/src/core/conversationLogger.ts` → fire-and-forget logging functions
+
+**Data Flow:**
+1. `handleTranscription` receives utterance → routes via BAML `b.Route()`
+2. Immediately after routing: `logConversation(userId, sessionId, transcript, route)` creates initial record
+3. Handler processes request → displays response on glasses
+4. Handler calls `updateConversationResponse(userId, sessionId, transcript, response)` to capture formatted output
+
+**LogContext Pattern:**
+Handlers that capture responses receive an optional `logContext` parameter:
+```ts
+logContext?: { convexUserId: Id<"users">; sessionId: string; transcript: string }
+```
+
+After displaying lines, call:
+```ts
+if (logContext) {
+  const responseText = lines.map((l) => `W: ${l}`).join("\n");
+  updateConversationResponse(
+    logContext.convexUserId,
+    logContext.sessionId,
+    logContext.transcript,
+    responseText,
+  );
+}
+```
+
+**Coverage by Route:**
+| Route | Initial Log | Response Captured | Notes |
+|-------|-------------|-------------------|-------|
+| WEATHER | ✓ | ✓ | Weather summary lines |
+| MAPS | ✓ | ✓ | Place recommendations |
+| WEB_SEARCH | ✓ | ✓ | Search answer lines |
+| KNOWLEDGE | ✓ | ✓ | General knowledge answer |
+| MEMORY_RECALL | ✓ | ✓ | Synthesized memory lines |
+| MEMORY_CAPTURE | ✓ | ✗ | Silent operation (stores to Honcho) |
+| PASSTHROUGH | ✓ | ✓ (when hint shown) | Null case for ambient speech |
+| NOTE_THIS | ✓ | ✗ | Meta-action, not content response |
+| FOLLOW_UP | ✓ | ✗ | Meta-action, not content response |
+
+**Key Points:**
+- Use fire-and-forget pattern: `void convexClient.mutation(...)` with `.catch()` for error logging
+- Response field captures the formatted text shown on glasses (e.g., `"W: 72°F and sunny"`)
+- PASSTHROUGH with no hint = null response (valuable for training router to identify ambient speech)
+- Initial log happens in `transcriptionFlow.ts`; response update happens in individual handlers

apps/api/package.json

@@ -19,9 +19,9 @@
 		"test": "echo 'No tests configured for @clairvoyant/api'"
 	},
 	"dependencies": {
-		"@elysiajs/cors": "^1.4.0",
+		"@elysiajs/cors": "^1.4.1",
 		"@t3-oss/env-core": "^0.13.10",
-		"convex": "^1.31.2",
+		"convex": "^1.32.0",
 		"jose": "^6.1.3",
 		"jsonwebtoken": "^9.0.3",
 		"zod": "^3.25.76"

apps/application/package.json

@@ -15,10 +15,10 @@
 		"@boundaryml/baml": "0.215.0",
 		"@clairvoyant/baml-client": "workspace:*",
 		"@honcho-ai/sdk": "^1.6.0",
-		"@mentra/sdk": "^2.1.28",
+		"@mentra/sdk": "^2.1.29",
 		"@t3-oss/env-core": "^0.13.10",
 		"@tavily/core": "^0.5.14",
-		"convex": "^1.31.2",
+		"convex": "^1.32.0",
 		"exa-js": "^1.10.2",
 		"openai": "^5.23.2",
 		"zod": "^3.25.76"

apps/application/src/core/conversationLogger.ts

@@ -0,0 +1,53 @@
+import { api } from "@convex/_generated/api";
+import type { Id } from "@convex/_generated/dataModel";
+import { convexClient } from "./convex";
+
+/**
+ * Logs a conversation to Convex for ML training data.
+ * Uses fire-and-forget pattern - errors are logged but don't block.
+ */
+export function logConversation(
+	userId: Id<"users">,
+	sessionId: string,
+	transcript: string,
+	route: string,
+	response?: string,
+): void {
+	void convexClient
+		.mutation(api.conversationLogs.logConversation, {
+			userId,
+			sessionId,
+			transcript,
+			route,
+			response,
+		})
+		.catch((error) => {
+			console.error("[ConversationLogger] Failed to log conversation:", error);
+		});
+}
+
+/**
+ * Updates the response field for a previously logged conversation.
+ * Used by handlers to capture the final formatted response shown on glasses.
+ * Uses fire-and-forget pattern - errors are logged but don't block.
+ */
+export function updateConversationResponse(
+	userId: Id<"users">,
+	sessionId: string,
+	transcript: string,
+	response: string,
+): void {
+	void convexClient
+		.mutation(api.conversationLogs.updateResponse, {
+			userId,
+			sessionId,
+			transcript,
+			response,
+		})
+		.catch((error) => {
+			console.error(
+				"[ConversationLogger] Failed to update conversation response:",
+				error,
+			);
+		});
+}

apps/application/src/handlers/hints.ts

@@ -1,7 +1,9 @@
 import { b, HintCategory } from "@clairvoyant/baml-client";
 import { api } from "@convex/_generated/api";
+import type { Id } from "@convex/_generated/dataModel";
 import type { Peer, Session } from "@honcho-ai/sdk";
 import type { AppSession } from "@mentra/sdk";
+import { updateConversationResponse } from "../core/conversationLogger";
 import { checkUserIsPro, convexClient } from "../core/convex";
 import type { DisplayQueueManager } from "../core/displayQueue";
 
@@ -14,6 +16,7 @@ export async function tryPassthroughHint(
 	peers: Peer[],
 	mentraUserId: string,
 	displayQueue: DisplayQueueManager,
+	logContext?: { convexUserId: Id<"users">; sessionId: string; transcript: string },
 ): Promise<void> {
 	const runId = Date.now();
 	hintRunIds.set(session, runId);
@@ -163,6 +166,16 @@ export async function tryPassthroughHint(
 			durationMs: 4000,
 			priority: 3,
 		});
+
+		// Log the hint response for ML training
+		if (logContext) {
+			updateConversationResponse(
+				logContext.convexUserId,
+				logContext.sessionId,
+				logContext.transcript,
+				`H: ${hintResult.hint}`,
+			);
+		}
 	} catch (error) {
 		session.logger.error(`[tryPassthroughHint] Error: ${String(error)}`);
 	}

apps/application/src/handlers/knowledge.ts

@@ -1,7 +1,9 @@
 import { b } from "@clairvoyant/baml-client";
 import { api } from "@convex/_generated/api";
+import type { Id } from "@convex/_generated/dataModel";
 import type { Peer, Session } from "@honcho-ai/sdk";
 import type { AppSession } from "@mentra/sdk";
+import { updateConversationResponse } from "../core/conversationLogger";
 import { checkUserIsPro, convexClient } from "../core/convex";
 import type { DisplayQueueManager } from "../core/displayQueue";
 import { showTextDuringOperation } from "../core/textWall";
@@ -17,6 +19,7 @@ export async function startKnowledgeFlow(
 	peers: Peer[],
 	mentraUserId: string,
 	displayQueue: DisplayQueueManager,
+	logContext?: { convexUserId: Id<"users">; sessionId: string; transcript: string },
 ) {
 	const runId = Date.now();
 	knowledgeRunIds.set(session, runId);
@@ -228,6 +231,21 @@ export async function startKnowledgeFlow(
 					priority: 2,
 				});
 			}
+
+			if (logContext && answerLines.length > 0) {
+				const responseText = answerLines
+					.map((line, i) => {
+						const label = answerLines.length > 1 ? `A${i + 1}` : "A";
+						return `Q: ${questionLine}\n${label}: ${line}`;
+					})
+					.join("\n");
+				updateConversationResponse(
+					logContext.convexUserId,
+					logContext.sessionId,
+					logContext.transcript,
+					responseText,
+				);
+			}
 		} else {
 			displayQueue.enqueue({
 				text: "// Clairvoyant\nK: No question detected.",

apps/application/src/handlers/maps.ts

@@ -1,7 +1,9 @@
 import { b } from "@clairvoyant/baml-client";
 import { api } from "@convex/_generated/api";
+import type { Id } from "@convex/_generated/dataModel";
 import type { Peer, Session } from "@honcho-ai/sdk";
 import type { AppSession } from "@mentra/sdk";
+import { updateConversationResponse } from "../core/conversationLogger";
 import {
 	checkUserIsPro,
 	convexClient,
@@ -30,6 +32,7 @@ async function processPlacesData(
 	places: PlaceSuggestion[],
 	runId: number,
 	displayQueue: DisplayQueueManager,
+	logContext?: { convexUserId: Id<"users">; sessionId: string; transcript: string },
 ) {
 	if (!places?.length) {
 		displayQueue.enqueue({
@@ -188,6 +191,16 @@ async function processPlacesData(
 			priority: 2,
 		});
 	}
+
+	if (logContext && lines.length > 0) {
+		const responseText = lines.map((l) => `M: ${l}`).join("\n");
+		updateConversationResponse(
+			logContext.convexUserId,
+			logContext.sessionId,
+			logContext.transcript,
+			responseText,
+		);
+	}
 }
 
 export async function startMapsFlow(
@@ -197,6 +210,7 @@ export async function startMapsFlow(
 	peers: Peer[],
 	mentraUserId: string,
 	displayQueue: DisplayQueueManager,
+	logContext?: { convexUserId: Id<"users">; sessionId: string; transcript: string },
 ) {
 	const runId = Date.now();
 	mapsRunIds.set(session, runId);
@@ -287,6 +301,7 @@ export async function startMapsFlow(
 				places,
 				runId,
 				displayQueue,
+				logContext,
 			);
 		} catch (error) {
 			session.logger.error(`[startMapsFlow] Maps flow error: ${String(error)}`);
@@ -376,6 +391,7 @@ export async function startMapsFlow(
 					places,
 					runId,
 					displayQueue,
+					logContext,
 				);
 			} catch (error) {
 				session.logger.error(

apps/application/src/handlers/memory.ts

@@ -1,7 +1,9 @@
 import { b } from "@clairvoyant/baml-client";
 import { api } from "@convex/_generated/api";
+import type { Id } from "@convex/_generated/dataModel";
 import type { Peer, Session } from "@honcho-ai/sdk";
 import type { AppSession } from "@mentra/sdk";
+import { updateConversationResponse } from "../core/conversationLogger";
 import { checkUserIsPro, convexClient } from "../core/convex";
 import type { DisplayQueueManager } from "../core/displayQueue";
 
@@ -78,6 +80,7 @@ export async function MemoryRecall(
 	peers: Peer[],
 	mentraUserId: string,
 	displayQueue: DisplayQueueManager,
+	logContext?: { convexUserId: Id<"users">; sessionId: string; transcript: string },
 ) {
 	const runId = Date.now();
 	memoryRunCallIds.set(session, runId);
@@ -251,6 +254,17 @@ export async function MemoryRecall(
 						priority: 2,
 					});
 				}
+
+				// Log the response for ML training
+				if (logContext) {
+					const responseText = lines.map((l) => `R: ${l}`).join("\n");
+					updateConversationResponse(
+						logContext.convexUserId,
+						logContext.sessionId,
+						logContext.transcript,
+						responseText,
+					);
+				}
 			} else {
 				session.logger.error(
 					`[startMemoryRecallFlow] No lines in synthesis results`,