feat(memory): Lakebase-backed persistent memory for CODA sessions

Makefile

@@ -15,14 +15,23 @@ endif
 ifdef app_name
 APP_NAME := $(app_name)
 endif
-PROFILE       ?= DEFAULT
-APP_NAME      ?= coding-agents
+ifdef pat
+PAT := $(pat)
+endif
+PROFILE          ?= DEFAULT
+APP_NAME         ?= coding-agents
+SECRET_SCOPE     ?= $(APP_NAME)-secrets
+SECRET_KEY       ?= databricks-token
+# Lakebase memory: set LAKEBASE_ENDPOINT to wire database resource (grants SP access + injects host)
+# Format: projects/{project_id}/branches/{branch_id}/endpoints/{endpoint_id}
+LAKEBASE_ENDPOINT ?=
+LAKEBASE_DB       ?= databricks_postgres
 
 # Resolve user email and workspace path from the profile
 USER_EMAIL    = $(shell databricks current-user me --profile $(PROFILE) --output json 2>/dev/null | python3 -c "import sys,json; print(json.load(sys.stdin).get('userName',''))")
 WORKSPACE_PATH = /Workspace/Users/$(USER_EMAIL)/apps/$(APP_NAME)
 
-.PHONY: help test deploy redeploy create-app create-pat sync deploy-app status open clean
+.PHONY: help test deploy-e2e deploy redeploy create-app create-pat setup-secret link-resources sync deploy-app status open clean clean-secret
 
 # ── Help ─────────────────────────────────────────────
 
@@ -66,13 +75,72 @@ create-app: ## Create the Databricks App (idempotent)
 		databricks apps create $(APP_NAME) --profile $(PROFILE); \
 	fi
 
-create-pat: ## Generate a 1-day PAT and copy it to your clipboard
-	@echo "==> Generating a 1-day PAT..."
-	@token=$$(databricks tokens create --lifetime-seconds $$((1 * 24 * 60 * 60)) --comment "coding-agents (1-day)" --profile $(PROFILE) --output json \
-		| python3 -c "import sys,json; print(json.load(sys.stdin)['token_value'])") && \
-	echo "$$token" | pbcopy && \
-	echo "    PAT copied to clipboard! (expires in 24 hours)"
+create-pat: ## Generate a 90-day PAT and store it as the app secret
+	@echo "==> Ensuring secret scope '$(SECRET_SCOPE)' exists..."
+	@if databricks secrets list-scopes --profile $(PROFILE) --output json 2>/dev/null | python3 -c "import sys,json; d=json.load(sys.stdin); scopes=[s['name'] for s in (d if isinstance(d,list) else d.get('scopes',[]))]; exit(0 if '$(SECRET_SCOPE)' in scopes else 1)" 2>/dev/null; then \
+		echo "    Secret scope '$(SECRET_SCOPE)' already exists."; \
+	else \
+		echo "    Creating secret scope '$(SECRET_SCOPE)'..."; \
+		databricks secrets create-scope $(SECRET_SCOPE) --profile $(PROFILE); \
+	fi
+	@echo "==> Generating a 90-day PAT..."
+	@databricks tokens create --lifetime-seconds $$((90 * 24 * 60 * 60)) --comment "coding-agents (auto-generated)" --profile $(PROFILE) --output json \
+		| python3 -c "import sys,json; print(json.load(sys.stdin)['token_value'])" \
+		| databricks secrets put-secret $(SECRET_SCOPE) $(SECRET_KEY) --profile $(PROFILE)
+	@echo "    PAT created and stored in $(SECRET_SCOPE)/$(SECRET_KEY)"
+
+setup-secret: ## Create secret scope and store PAT (interactive)
+	@echo "==> Setting up DATABRICKS_TOKEN secret..."
+	@# Create scope if it doesn't exist
+	@if databricks secrets list-scopes --profile $(PROFILE) --output json 2>/dev/null | python3 -c "import sys,json; d=json.load(sys.stdin); scopes=[s['name'] for s in (d if isinstance(d,list) else d.get('scopes',[]))]; exit(0 if '$(SECRET_SCOPE)' in scopes else 1)" 2>/dev/null; then \
+		echo "    Secret scope '$(SECRET_SCOPE)' already exists."; \
+	else \
+		echo "    Creating secret scope '$(SECRET_SCOPE)'..."; \
+		databricks secrets create-scope $(SECRET_SCOPE) --profile $(PROFILE); \
+	fi
+	@# Store the PAT - prompt if not provided
+	@if [ -z "$(PAT)" ]; then \
+		echo "    Enter your Databricks PAT (will not echo):"; \
+		read -s pat_value && \
+		echo "$$pat_value" | databricks secrets put-secret $(SECRET_SCOPE) $(SECRET_KEY) --profile $(PROFILE); \
+	else \
+		echo "$(PAT)" | databricks secrets put-secret $(SECRET_SCOPE) $(SECRET_KEY) --profile $(PROFILE); \
+	fi
+	@echo "    Secret stored in $(SECRET_SCOPE)/$(SECRET_KEY)"
 
+link-resources: ## Add Lakebase postgres resource to app (preserves existing resources)
+	@if [ -z "$(LAKEBASE_ENDPOINT)" ]; then \
+		echo "Error: LAKEBASE_ENDPOINT required."; \
+		echo "Usage: make link-resources LAKEBASE_ENDPOINT=projects/.../branches/.../endpoints/... PROFILE=daveok"; \
+		exit 1; \
+	fi
+	@echo "==> Ensuring secret scope and ENDPOINT_NAME secret..."
+	@if ! databricks secrets list-scopes --profile $(PROFILE) --output json 2>/dev/null | python3 -c "import sys,json; d=json.load(sys.stdin); scopes=[s['name'] for s in (d if isinstance(d,list) else d.get('scopes',[]))]; exit(0 if '$(SECRET_SCOPE)' in scopes else 1)" 2>/dev/null; then \
+		databricks secrets create-scope $(SECRET_SCOPE) --profile $(PROFILE); \
+	fi
+	@printf '%s' '$(LAKEBASE_ENDPOINT)' | databricks secrets put-secret $(SECRET_SCOPE) ENDPOINT_NAME --profile $(PROFILE)
+	@echo "==> Merging Lakebase resource into existing app resources..."
+	@DATABRICKS_CONFIG_PROFILE=$(PROFILE) uv run python3 -c "\
+import json, os, sys; \
+from databricks.sdk import WorkspaceClient; \
+w = WorkspaceClient(profile='$(PROFILE)'); \
+ep = '$(LAKEBASE_ENDPOINT)'; \
+branch = ep.rsplit('/endpoints/', 1)[0]; \
+dbs = list(w.postgres.list_databases(parent=branch)); \
+db = next((d.name for d in dbs if d.database_name == '$(LAKEBASE_DB)'), dbs[0].name if dbs else None); \
+assert db, 'no database found under ' + branch; \
+app = w.apps.get(name='$(APP_NAME)'); \
+existing = [r.as_dict() for r in (app.resources or [])]; \
+merged = {r['name']: r for r in existing}; \
+merged['postgres'] = {'name':'postgres','description':'Lakebase Autoscaling memory database','postgres':{'branch':branch,'database':db,'permission':'CAN_CONNECT_AND_CREATE'}}; \
+merged['ENDPOINT_NAME'] = {'name':'ENDPOINT_NAME','description':'Lakebase endpoint resource path','secret':{'scope':'$(SECRET_SCOPE)','key':'ENDPOINT_NAME','permission':'READ'}}; \
+print(json.dumps({'resources': list(merged.values())})) \
+" | curl -s -X PATCH \
+		"$$(databricks auth env --profile $(PROFILE) 2>/dev/null | python3 -c "import sys,json; print(json.load(sys.stdin)['env']['DATABRICKS_HOST'])")/api/2.0/apps/$(APP_NAME)" \
+		-H "Authorization: Bearer $$(databricks auth token --profile $(PROFILE) 2>/dev/null | python3 -c "import sys,json; print(json.load(sys.stdin)['access_token'])")" \
+		-H "Content-Type: application/json" \
+		-d @- >/dev/null
+	@echo "    Linked postgres resource ($(LAKEBASE_ENDPOINT)) + ENDPOINT_NAME secret; existing resources preserved."
 
 sync: ## Sync local files to Databricks workspace
 	@echo "==> Syncing to $(WORKSPACE_PATH)..."

agents/memory-recall.md

@@ -0,0 +1,48 @@
+---
+name: memory-recall
+description: Search past CODA sessions stored in Lakebase for relevant context, prior decisions, and learned preferences. Invoke when the user references past work, asks "do you remember...", mentions prior conversations, or when a [coda-memory] signal appears in context.
+tools: Bash
+---
+
+# Role
+
+You are a focused memory retrieval agent. You search Lakebase for relevant memories from past coding sessions and return a concise synthesis. You run in an isolated fork — do NOT use tools other than Bash.
+
+# Process
+
+## Step 1: Formulate a targeted query
+
+Based on the user's question or the current task, identify 1-3 specific search terms. Be concrete: prefer "FastAPI OAuth Databricks" over "auth stuff".
+
+## Step 2: Search
+
+```bash
+cd /app/python/source_code && uv run python -m memory.searcher "YOUR QUERY HERE"
+```
+
+Run up to 2 searches with different queries if the first returns nothing useful. You may pass `--project <name>` if the project context is known.
+
+## Step 3: Evaluate and synthesise
+
+- Filter out memories that are clearly not relevant to the current question
+- Do NOT return raw database output — synthesise into 3-5 bullets
+- Include the memory type (preference, project decision, reference) so the caller knows how to weight it
+- If nothing relevant was found, say so in one sentence and stop
+
+## Output format
+
+Return your synthesis directly. Example:
+
+**From past sessions:**
+- [Preference] Always use `uv`, never pip — user is strict about this
+- [Project: my-app] Chose FastAPI over Flask for async support; Databricks SDK wired via `WorkspaceClient`
+- [Reference] Databricks BGE embedding endpoint: `databricks-bge-large-en`
+
+If no relevant memories: "No relevant memories found for this query."
+
+## Rules
+
+- Never make up memories — only report what the search returns
+- Never include raw SQL output, error tracebacks, or timestamps unless directly relevant
+- Keep synthesis under 200 words
+- Stop after synthesis — do not continue with implementation or ask follow-up questions

app.py

@@ -114,6 +114,7 @@ def handle_sigterm(signum, frame):
         {"id": "hermes",     "label": "Configuring Hermes Agent",     "status": "pending", "started_at": None, "completed_at": None, "error": None},
         {"id": "databricks", "label": "Setting up Databricks CLI",    "status": "pending", "started_at": None, "completed_at": None, "error": None},
         {"id": "mlflow",     "label": "Enabling MLflow tracing",       "status": "pending", "started_at": None, "completed_at": None, "error": None},
+        {"id": "memory",     "label": "Wiring Lakebase memory",       "status": "pending", "started_at": None, "completed_at": None, "error": None},
     ]
 }
 
@@ -386,11 +387,14 @@ def run_setup():
         ]
         wait(futures)
 
-    # --- MLflow setup runs AFTER claude setup to avoid settings.json race ---
+    # --- Sequential post-parallel steps (these modify settings.json and must not race) ---
     # setup_mlflow.py merges env vars into ~/.claude/settings.json which
     # setup_claude.py also writes; running sequentially prevents clobbering.
     _run_step("mlflow", ["uv", "run", "python", "setup_mlflow.py"])
 
+    # setup_memory writes a Stop hook alongside setup_mlflow's hook — must run after
+    _run_step("memory", ["uv", "run", "python", "setup_memory.py"])
+
     # Sync latest token into all CLI configs — covers the race where PAT
     # rotation happened while a setup script was still installing (the
     # rotation's update_cli_tokens() call silently skips missing config files).

app.yaml

@@ -14,3 +14,16 @@ env:
     value: 0
   - name: MAX_CONCURRENT_SESSIONS
     value: "5"
+  # Lakebase memory — Databricks auto-injects PGHOST/PGPORT/PGDATABASE/PGUSER/PGSSLMODE
+  # when a "postgres" database resource is configured on the app. You still must set
+  # ENDPOINT_NAME manually — generate_database_credential() needs the endpoint path.
+  # Wire via: make link-resources LAKEBASE_ENDPOINT=projects/.../branches/.../endpoints/... PROFILE=<profile>
+  # Memory feature is disabled when ENDPOINT_NAME is absent.
+  - name: ENDPOINT_NAME
+    valueFrom: ENDPOINT_NAME
+  # Model used for memory extraction (cheap/fast Haiku by default).
+  # Must be a Databricks Model Serving endpoint name — ANTHROPIC_BASE_URL points
+  # at the Databricks Claude proxy, which rejects Anthropic-style model IDs
+  # (e.g. "claude-haiku-4-5-20251001") with ENDPOINT_NOT_FOUND 404.
+  - name: MEMORY_EXTRACTION_MODEL
+    value: databricks-claude-haiku-4-5