feat: ship coda-essentials as a bundled plugin marketplace

claude_brain_sync.py

@@ -0,0 +1,177 @@
+#!/usr/bin/env python
+"""Sync Claude Code's auto-memory ("brain") to/from Databricks Workspace.
+
+The "brain" is the set of memory files Claude Code maintains at
+`~/.claude/projects/{slug}/memory/`, one slug per working directory.
+They accumulate user/project/feedback/reference memories that make
+future sessions smarter.
+
+Ephemeral Databricks App compute means these files vanish when the
+app restarts unless we persist them. This script syncs them to the
+user's workspace so they survive redeploys and restarts.
+
+Usage:
+    python claude_brain_sync.py push                      # local -> workspace ([DEFAULT])
+    python claude_brain_sync.py pull                      # workspace -> local ([DEFAULT])
+    python claude_brain_sync.py push --profile daveok     # use a named profile
+    python claude_brain_sync.py                           # push (default)
+"""
+from __future__ import annotations
+
+import argparse
+import configparser
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+try:
+    from databricks.sdk import WorkspaceClient
+except ImportError:
+    print("databricks-sdk not available, skipping brain sync", file=sys.stderr)
+    sys.exit(0)
+
+
+CLAUDE_PROJECTS = Path.home() / ".claude" / "projects"
+WORKSPACE_SUBPATH = ".coda/claude-brain/projects"
+
+
+def _read_databrickscfg(profile: str = "DEFAULT") -> tuple[str | None, str | None]:
+    cfg = Path.home() / ".databrickscfg"
+    if not cfg.exists():
+        return None, None
+    p = configparser.ConfigParser()
+    p.read(cfg)
+    if profile not in p and profile != "DEFAULT":
+        return None, None
+    return (
+        p.get(profile, "host", fallback=None),
+        p.get(profile, "token", fallback=None),
+    )
+
+
+def _workspace_client(profile: str | None) -> WorkspaceClient:
+    """Build a WorkspaceClient. Named profiles delegate auth to the SDK
+    so OAuth (`auth_type = databricks-cli`) works for local testing;
+    the default path reads [DEFAULT] explicitly for the production PAT flow."""
+    if profile:
+        return WorkspaceClient(profile=profile)
+    host, token = _read_databrickscfg("DEFAULT")
+    if not host or not token:
+        raise RuntimeError("~/.databrickscfg [DEFAULT] missing host or token")
+    return WorkspaceClient(host=host, token=token, auth_type="pat")
+
+
+def _user_email(profile: str | None) -> str:
+    return _workspace_client(profile).current_user.me().user_name
+
+
+def _sync_env() -> dict[str, str]:
+    """Env for databricks CLI. Strip OAuth M2M vars so CLI falls through to
+    the profile config. Profile selection is passed via --profile CLI flag."""
+    env = os.environ.copy()
+    for var in ("DATABRICKS_CLIENT_ID", "DATABRICKS_CLIENT_SECRET",
+                "DATABRICKS_HOST", "DATABRICKS_TOKEN"):
+        env.pop(var, None)
+    return env
+
+
+def _profile_args(profile: str | None) -> list[str]:
+    return ["--profile", profile] if profile else []
+
+
+def _memory_dirs() -> list[Path]:
+    """Return memory dirs that actually contain files worth syncing."""
+    if not CLAUDE_PROJECTS.exists():
+        return []
+    dirs = []
+    for project_dir in CLAUDE_PROJECTS.iterdir():
+        if not project_dir.is_dir():
+            continue
+        memory = project_dir / "memory"
+        if memory.exists() and any(memory.iterdir()):
+            dirs.append(memory)
+    return dirs
+
+
+def push(profile: str | None = None) -> int:
+    """Push each project's memory dir to workspace."""
+    dirs = _memory_dirs()
+    if not dirs:
+        print("brain-sync: no memory dirs to push")
+        return 0
+
+    try:
+        email = _user_email(profile)
+    except Exception as e:
+        print(f"brain-sync: could not resolve user email: {e}", file=sys.stderr)
+        return 1
+
+    env = _sync_env()
+    profile_flags = _profile_args(profile)
+    failures = 0
+    for memory_dir in dirs:
+        project_slug = memory_dir.parent.name
+        remote = f"/Workspace/Users/{email}/{WORKSPACE_SUBPATH}/{project_slug}/memory"
+        result = subprocess.run(
+            ["databricks", "sync", str(memory_dir), remote, "--watch=false"] + profile_flags,
+            capture_output=True, text=True, env=env,
+        )
+        if result.returncode == 0:
+            print(f"brain-sync push: {project_slug}")
+        else:
+            print(f"brain-sync push FAILED for {project_slug}: {result.stderr.strip()}",
+                  file=sys.stderr)
+            failures += 1
+    return 0 if failures == 0 else 1
+
+
+def pull(profile: str | None = None) -> int:
+    """Pull brain from workspace into ~/.claude/projects/.
+
+    Uses databricks workspace export-dir because `databricks sync` is
+    local->remote only.
+    """
+    try:
+        email = _user_email(profile)
+    except Exception as e:
+        print(f"brain-sync: could not resolve user email: {e}", file=sys.stderr)
+        return 1
+
+    env = _sync_env()
+    profile_flags = _profile_args(profile)
+    remote_root = f"/Workspace/Users/{email}/{WORKSPACE_SUBPATH}"
+
+    check = subprocess.run(
+        ["databricks", "workspace", "list", remote_root] + profile_flags,
+        capture_output=True, text=True, env=env,
+    )
+    if check.returncode != 0:
+        print(f"brain-sync pull: no remote brain yet at {remote_root}")
+        return 0
+
+    CLAUDE_PROJECTS.mkdir(parents=True, exist_ok=True)
+    result = subprocess.run(
+        ["databricks", "workspace", "export-dir",
+         remote_root, str(CLAUDE_PROJECTS), "--overwrite"] + profile_flags,
+        capture_output=True, text=True, env=env,
+    )
+    if result.returncode == 0:
+        print(f"brain-sync pull: restored from {remote_root}")
+        return 0
+    print(f"brain-sync pull FAILED: {result.stderr.strip()}", file=sys.stderr)
+    return 1
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    parser.add_argument("direction", nargs="?", default="push", choices=["push", "pull"])
+    parser.add_argument("--profile", help="databricks CLI profile name (default: [DEFAULT])")
+    args = parser.parse_args()
+    if args.direction == "push":
+        return push(args.profile)
+    return pull(args.profile)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

coda-marketplace/.claude-plugin/marketplace.json

@@ -0,0 +1,50 @@
+{
+  "name": "coda",
+  "owner": {
+    "name": "Databricks Field Engineering",
+    "email": "field-eng@databricks.com"
+  },
+  "metadata": {
+    "description": "CODA-bundled Claude Code plugins — ship with every CODA deployment",
+    "version": "0.1.0"
+  },
+  "plugins": [
+    {
+      "name": "coda-essentials",
+      "source": "./plugins/coda-essentials",
+      "description": "Subagents, hooks, slash commands, and session lifecycle tooling bundled with every CODA instance. Includes the TDD subagent workflow (prd-writer, test-generator, implementer, build-feature), session-start git context loader, memory staleness checker, crystallization nudge, and the /til slash command.",
+      "version": "0.1.0",
+      "author": {
+        "name": "Databricks Field Engineering"
+      },
+      "category": "productivity",
+      "keywords": [
+        "coda",
+        "databricks",
+        "workshop",
+        "tdd",
+        "memory",
+        "hooks"
+      ]
+    },
+    {
+      "name": "coda-databricks-skills",
+      "source": "./plugins/coda-databricks-skills",
+      "description": "Databricks platform skills synced from databricks-solutions/ai-dev-kit: Agent Bricks, AI/BI Dashboards, AI Functions, Databricks App (Python), BDD Testing, Bundles, Config, DBSQL, Docs, Execution Compute, Genie, Iceberg, Jobs, Lakebase (Autoscale + Provisioned), Metric Views, MLflow Evaluation, Model Serving, Python SDK, Spark SDP, Structured Streaming, Synthetic Data Gen, Unity Catalog, Unstructured PDF Generation, Vector Search, Zerobus Ingest, and Spark Python Data Source.",
+      "version": "0.1.0",
+      "author": {
+        "name": "Databricks Field Engineering",
+        "url": "https://github.com/databricks-solutions/ai-dev-kit"
+      },
+      "category": "platform",
+      "keywords": [
+        "databricks",
+        "ai-dev-kit",
+        "spark",
+        "unity-catalog",
+        "mlflow",
+        "lakebase"
+      ]
+    }
+  ]
+}

coda-marketplace/plugins/coda-essentials/.claude-plugin/plugin.json

@@ -0,0 +1,18 @@
+{
+  "name": "coda-essentials",
+  "description": "Subagents, hooks, slash commands, and session lifecycle tooling bundled with every CODA instance.",
+  "version": "0.1.0",
+  "author": {
+    "name": "Databricks Field Engineering"
+  },
+  "keywords": [
+    "coda",
+    "databricks",
+    "workshop",
+    "tdd",
+    "memory",
+    "hooks"
+  ],
+  "agents": "./agents/",
+  "commands": "./commands/"
+}

coda-marketplace/plugins/coda-essentials/agents/build-feature.md

@@ -0,0 +1,66 @@
+---
+name: build-feature
+description: End-to-end feature builder. Chains prd-writer → test-generator → implementer → web-devloop-tester in TDD flow. Use when asked to "build", "create", or "implement" a feature from scratch. Orchestrates the full cycle including bug fix loops and visual UI testing.
+tools: Read, Write, Edit, Glob, Grep, Bash, Agent, AskUserQuestion, WebSearch, WebFetch
+---
+
+# Role
+You are a tech lead orchestrating a TDD feature build. You coordinate four phases and handle failures.
+
+# Phase 1: PRD
+1. Invoke yourself as a prd-writer: interview the user, write `docs/prd/<slug>.md`
+2. Do NOT proceed until the user approves the PRD
+3. PRD must have status `READY_FOR_IMPLEMENTATION` before moving on
+
+# Phase 2: Tests (TDD)
+1. Read the approved PRD
+2. Extract all Acceptance Criteria (AC-*)
+3. Scan the codebase for test framework and conventions
+4. Write failing tests that define the contract — one or more tests per AC
+5. Run the tests to confirm they fail for the right reasons (missing implementation, not broken tests)
+6. Update PRD status to `TESTS_WRITTEN`
+
+# Phase 3: Implementation
+1. Read the PRD and all test files
+2. Run the test suite to see current failures
+3. Create an implementation plan, present it to the user for approval
+4. Implement code to make tests pass, working through one group at a time
+5. After each group, run tests to verify progress
+
+# Bug Fix Loop
+If tests fail after implementation:
+
+1. Read the failure output carefully
+2. Identify whether the bug is in the **test** or the **implementation**
+3. If test is wrong (doesn't match PRD): fix the test
+4. If implementation is wrong: fix the code
+5. Re-run tests
+6. **Max 3 fix loops** — if still failing after 3 rounds, stop and report to the user with:
+   - Which tests are failing
+   - The error messages
+   - Your hypothesis on the root cause
+   - Ask the user how to proceed
+
+# Phase 4: Visual Testing (Web Apps Only)
+If the feature has a UI component (React, Vue, Streamlit, Dash, etc.):
+
+1. Spawn a `web-devloop-tester` agent (subagent_type: `fe-specialized-agents:web-devloop-tester`)
+2. Tell it to: start the dev server, navigate to the relevant page, take screenshots, check console for errors, and test key interactions from the AC-* list
+3. Review the tester's report:
+   - **All clear** → proceed to Completion
+   - **Issues found** → create fix tasks for the implementer, then re-test
+4. **Max 3 visual fix loops** — if issues persist after 3 rounds, stop and report to the user with screenshots and logs
+
+Skip this phase for:
+- CLI tools, libraries, backend-only APIs
+- Projects with no dev server or browser UI
+
+# Completion
+When all tests pass and visual testing is complete (or skipped):
+1. Run the full test suite one final time
+2. Update PRD status to `COMPLETE`
+3. Summarize what was built:
+   - Files created/modified
+   - Test coverage (AC-* mapping)
+   - Visual test results (screenshots, if applicable)
+   - Any open items or manual testing needed

coda-marketplace/plugins/coda-essentials/agents/implementer.md

@@ -0,0 +1,59 @@
+---
+name: implementer
+description: Reads a PRD and makes all tests pass. Implements code to satisfy the test suite written by test-generator. Use after test-generator has written failing tests. Runs tests iteratively until green.
+tools: Read, Write, Edit, Glob, Grep, Bash, Agent
+---
+
+# Role
+You are a senior software engineer who makes failing tests pass. You implement exactly what's needed to satisfy the test suite and PRD requirements — nothing more.
+
+# Startup
+1. Read the PRD file specified (or scan `docs/prd/` for files with status `TESTS_WRITTEN`)
+2. Read ALL test files listed in the PRD status section
+3. Run the test suite to see the current failures
+4. Read any files referenced in the PRD's Technical Notes or Dependencies sections
+5. Scan the codebase with Glob/Grep to understand existing patterns and architecture
+
+# Planning Phase
+Before writing any code, create a numbered implementation plan:
+
+1. List every failing test and what it expects
+2. Group tests by module/component
+3. Identify files to create or modify
+4. Note the order of operations (what depends on what)
+5. Flag any Open Questions from the PRD that block implementation
+
+Present the plan and wait for approval before proceeding.
+
+# Implementation Phase — Red-Green Loop
+For each group of related tests:
+
+1. **Read the tests** — understand exactly what they expect
+2. **Write minimal code** to make those tests pass
+3. **Run tests** — check if they pass
+4. **If tests fail** — read the error, fix the code, run again
+5. **Repeat** until that group is green
+6. **Commit** — use `git commit -m "message"` directly
+7. Move to the next group
+
+Rules:
+- **Read before writing** — always read existing files before modifying
+- **Follow existing patterns** — match the codebase's style and conventions
+- **Keep it simple** — don't over-engineer; make the tests pass
+- **Max 3 fix attempts per test** — if a test won't pass after 3 tries, flag it and move on
+
+# Final Validation
+After all implementation:
+
+1. Run the FULL test suite
+2. If any tests still fail, attempt fixes (max 2 more rounds)
+3. If tests still fail after retries, document the failures
+
+# Handoff
+When complete, update the PRD status:
+
+> **Status: IMPLEMENTED**
+> Commits: <list of commit hashes>
+> Test results: <X passing, Y failing>
+> If all green: **Status: COMPLETE**
+> If failures remain: **Status: NEEDS_REVIEW** with failure details