From 1c460626368d0a70859675364e52461a431b87f1 Mon Sep 17 00:00:00 2001 From: David O'Keeffe Date: Wed, 11 Mar 2026 00:10:12 +1100 Subject: [PATCH 1/5] feat: OpenCode fork install, GitHub CLI, reduce keystroke latency MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Install OpenCode from dgokeeffe/opencode fork with native Databricks provider (auto-discovers models, shares Claude Code skills) - Add GitHub CLI (gh) setup with xterm.js-safe auth wrapper - Reduce select() timeout 500ms→50ms and poll interval 100ms→50ms - Add Makefile for deployment automation Co-Authored-By: Claude Opus 4.6 --- Makefile | 87 +++++++++++++++++++++++++++++++++++++++++++ app.py | 31 ++++++++++++++- static/poll-worker.js | 2 +- 3 files changed, 117 insertions(+), 3 deletions(-) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..9c52e9a --- /dev/null +++ b/Makefile @@ -0,0 +1,87 @@ +# Makefile for deploying Coding Agents to Databricks Apps +# +# Usage: +# make deploy PROFILE=daveok PAT=dapi... +# make deploy PROFILE=daveok # prompts for PAT interactively +# make redeploy PROFILE=daveok # skip secret setup, just sync + deploy +# make status PROFILE=daveok # check app status +# make logs PROFILE=daveok # tail app logs + +# Configuration +PROFILE ?= DEFAULT +APP_NAME ?= coding-agents +SECRET_SCOPE ?= $(APP_NAME)-secrets +SECRET_KEY ?= databricks-token + +# Resolve user email and workspace path from the profile +USER_EMAIL = $(shell databricks current-user me --profile $(PROFILE) --output json 2>/dev/null | python3 -c "import sys,json; print(json.load(sys.stdin).get('userName',''))") +WORKSPACE_PATH = /Workspace/Users/$(USER_EMAIL)/apps/$(APP_NAME) + +.PHONY: help deploy redeploy create-app setup-secret sync deploy-app status logs clean-secret + +help: ## Show this help + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf " \033[36m%-18s\033[0m %s\n", $$1, $$2}' + +deploy: create-app setup-secret sync deploy-app ## Full deploy: create app, set secret, sync, deploy + @echo "" + @echo "Deployment complete! App URL:" + @databricks apps get $(APP_NAME) --profile $(PROFILE) --output json 2>/dev/null | python3 -c "import sys,json; print(json.load(sys.stdin).get('url','(pending)'))" + +redeploy: sync deploy-app ## Redeploy: sync + deploy (skip secret setup) + @echo "" + @echo "Redeployment complete!" + +create-app: ## Create the Databricks App (idempotent) + @echo "==> Checking if app '$(APP_NAME)' exists..." + @if databricks apps get $(APP_NAME) --profile $(PROFILE) >/dev/null 2>&1; then \ + echo " App '$(APP_NAME)' already exists, skipping create."; \ + else \ + echo " Creating app '$(APP_NAME)'..."; \ + databricks apps create $(APP_NAME) --profile $(PROFILE); \ + fi + +setup-secret: ## Create secret scope and store PAT + @echo "==> Setting up DATABRICKS_TOKEN secret..." + @# Create scope if it doesn't exist + @if databricks secrets list-scopes --profile $(PROFILE) --output json 2>/dev/null | python3 -c "import sys,json; scopes=[s['name'] for s in json.load(sys.stdin).get('scopes',[])]; exit(0 if '$(SECRET_SCOPE)' in scopes else 1)" 2>/dev/null; then \ + echo " Secret scope '$(SECRET_SCOPE)' already exists."; \ + else \ + echo " Creating secret scope '$(SECRET_SCOPE)'..."; \ + databricks secrets create-scope $(SECRET_SCOPE) --profile $(PROFILE); \ + fi + @# Store the PAT - prompt if not provided + @if [ -z "$(PAT)" ]; then \ + echo " Enter your Databricks PAT (will not echo):"; \ + read -s pat_value && \ + echo "$$pat_value" | databricks secrets put-secret $(SECRET_SCOPE) $(SECRET_KEY) --profile $(PROFILE); \ + else \ + echo "$(PAT)" | databricks secrets put-secret $(SECRET_SCOPE) $(SECRET_KEY) --profile $(PROFILE); \ + fi + @echo " Secret stored in $(SECRET_SCOPE)/$(SECRET_KEY)" + @# Link secret to app resource + @echo " Linking secret to app resource 'DATABRICKS_TOKEN'..." + @curl -s -X PATCH \ + "$$(databricks auth env --profile $(PROFILE) 2>/dev/null | python3 -c "import sys,json; print(json.load(sys.stdin)['env']['DATABRICKS_HOST'])")/api/2.0/apps/$(APP_NAME)" \ + -H "Authorization: Bearer $$(databricks auth token --profile $(PROFILE) 2>/dev/null | python3 -c "import sys,json; print(json.load(sys.stdin)['access_token'])")" \ + -H "Content-Type: application/json" \ + -d '{"resources":[{"name":"DATABRICKS_TOKEN","description":"PAT for model serving access","secret":{"scope":"$(SECRET_SCOPE)","key":"$(SECRET_KEY)","permission":"READ"}}]}' \ + >/dev/null + @echo " App resource linked." + +sync: ## Sync local files to Databricks workspace + @echo "==> Syncing to $(WORKSPACE_PATH)..." + databricks sync . $(WORKSPACE_PATH) --watch=false --profile $(PROFILE) + +deploy-app: ## Deploy the app from workspace + @echo "==> Deploying app '$(APP_NAME)'..." + databricks apps deploy $(APP_NAME) --source-code-path $(WORKSPACE_PATH) --profile $(PROFILE) --no-wait + +status: ## Check app status + @databricks apps get $(APP_NAME) --profile $(PROFILE) + +logs: ## Tail app logs + databricks apps logs $(APP_NAME) --profile $(PROFILE) + +clean-secret: ## Remove secret scope (destructive) + @echo "==> Removing secret scope '$(SECRET_SCOPE)'..." + databricks secrets delete-scope $(SECRET_SCOPE) --profile $(PROFILE) diff --git a/app.py b/app.py index 5526a6b..18e91a3 100644 --- a/app.py +++ b/app.py @@ -83,6 +83,7 @@ def handle_sigterm(signum, frame): "steps": [ {"id": "git", "label": "Configuring git identity", "status": "pending", "started_at": None, "completed_at": None, "error": None}, {"id": "micro", "label": "Installing micro editor", "status": "pending", "started_at": None, "completed_at": None, "error": None}, + {"id": "gh", "label": "Installing GitHub CLI", "status": "pending", "started_at": None, "completed_at": None, "error": None}, {"id": "proxy", "label": "Starting content-filter proxy", "status": "pending", "started_at": None, "completed_at": None, "error": None}, {"id": "claude", "label": "Configuring Claude CLI", "status": "pending", "started_at": None, "completed_at": None, "error": None}, {"id": "codex", "label": "Configuring Codex CLI", "status": "pending", "started_at": None, "completed_at": None, "error": None}, @@ -252,10 +253,36 @@ def run_setup(): _run_step("micro", ["bash", "-c", "mkdir -p ~/.local/bin && bash install_micro.sh && mv micro ~/.local/bin/ 2>/dev/null || true"]) + _run_step( + "gh", + [ + "bash", + "-c", + 'GH_VERSION="2.74.1" && ' + "mkdir -p ~/.local/bin && " + 'curl -fsSL "https://github.com/cli/cli/releases/download/v${GH_VERSION}/gh_${GH_VERSION}_linux_amd64.tar.gz" -o /tmp/gh.tar.gz && ' + "tar -xzf /tmp/gh.tar.gz -C /tmp && " + "mv /tmp/gh_${GH_VERSION}_linux_amd64/bin/gh ~/.local/bin/gh && " + "rm -rf /tmp/gh.tar.gz /tmp/gh_${GH_VERSION}_linux_amd64 && " + "chmod +x ~/.local/bin/gh && " + "gh config set git_protocol https 2>/dev/null || true && " + # Wrap gh auth login to skip interactive prompts (arrow-key menus break in xterm.js PTY) + "printf '#!/bin/bash\\n" + 'if [ "$1" = "auth" ] && [ "$2" = "login" ]; then\\n' + " shift 2\\n" + ' printf "Y\\\\n" | ~/.local/bin/gh.real auth login -h github.com -p https -w --skip-ssh-key "$@"\\n' + "fi\\n" + 'exec ~/.local/bin/gh.real "$@"\\n\' > ~/.local/bin/gh.wrapper && ' + "mv ~/.local/bin/gh ~/.local/bin/gh.real && " + "mv ~/.local/bin/gh.wrapper ~/.local/bin/gh && " + "chmod +x ~/.local/bin/gh", + ], + ) + # --- Content-filter proxy (must be running before OpenCode starts) --- # Sanitizes requests/responses between OpenCode and Databricks # (see OpenCode #5028, docs/plans/2026-03-11-litellm-empty-content-blocks-design.md) - _run_step("proxy", ["python", "setup_proxy.py"]) + _run_step("proxy", ["uv", "run", "python", "setup_proxy.py"]) # --- Parallel agent setup (all independent of each other) --- parallel_steps = [ @@ -487,7 +514,7 @@ def read_pty_output(session_id, fd): if session_id not in sessions: break try: - readable, _, errors = select.select([fd], [], [fd], 0.5) + readable, _, errors = select.select([fd], [], [fd], 0.05) if readable or errors: output = os.read(fd, 4096) if not output: diff --git a/static/poll-worker.js b/static/poll-worker.js index a499b0a..4315b5f 100644 --- a/static/poll-worker.js +++ b/static/poll-worker.js @@ -21,7 +21,7 @@ "use strict"; // ── Constants ───────────────────────────────────────────────────────────── -const POLL_INTERVAL_FG = 100; // ms — foreground batch poll +const POLL_INTERVAL_FG = 50; // ms — foreground batch poll const HEARTBEAT_INTERVAL_BG = 30000; // ms — background heartbeat const RETRY_BASE_MS = 500; const RETRY_MULTIPLIER = 2; From 8735bbeb80fb67a26b89dfa12fd7e7d29ed3262b Mon Sep 17 00:00:00 2001 From: David O'Keeffe Date: Wed, 11 Mar 2026 00:22:23 +1100 Subject: [PATCH 2/5] perf: reduce global lock contention in batch polling Replace single global sessions_lock block in get_output_batch() with 3-step resolve/swap/join pattern matching get_output(). Snapshot session dict in cleanup_stale_sessions() to iterate with per-session locks. Co-Authored-By: Claude Opus 4.6 --- app.py | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/app.py b/app.py index 18e91a3..698798b 100644 --- a/app.py +++ b/app.py @@ -596,7 +596,10 @@ def cleanup_stale_sessions(): warning_threshold = SESSION_TIMEOUT_SECONDS * 0.8 with sessions_lock: - for session_id, session in sessions.items(): + session_snapshot = list(sessions.items()) + + for session_id, session in session_snapshot: + with session["lock"]: idle = now - session["last_poll_time"] if idle > SESSION_TIMEOUT_SECONDS: stale_sessions.append((session_id, session["pid"], session["master_fd"])) @@ -828,22 +831,31 @@ def get_output_batch(): outputs = {} now = time.time() + # Step 1: Resolve session refs under global lock (fast dict lookups only) + resolved = {} with sessions_lock: for sid in session_ids: - if sid not in sessions: - continue - session = sessions[sid] + if sid in sessions: + resolved[sid] = sessions[sid] + + # Step 2: Swap buffers under per-session locks (same pattern as get_output) + swapped = {} + for sid, session in resolved.items(): + with session["lock"]: session["last_poll_time"] = now - buffer = session["output_buffer"] - output = "".join(buffer) - buffer.clear() + old_buffer = session["output_buffer"] + session["output_buffer"] = deque(maxlen=1000) exited = session.get("exited", False) timeout_warning = session.pop("timeout_warning", False) - outputs[sid] = { - "output": output, - "exited": exited, - "timeout_warning": timeout_warning - } + swapped[sid] = (old_buffer, exited, timeout_warning) + + # Step 3: Join strings outside all locks + for sid, (old_buffer, exited, timeout_warning) in swapped.items(): + outputs[sid] = { + "output": "".join(old_buffer), + "exited": exited, + "timeout_warning": timeout_warning, + } return jsonify({"outputs": outputs, "shutting_down": shutting_down}) From 5726c378bdf40d8f23d92f2933eb064a9fe058b5 Mon Sep 17 00:00:00 2001 From: David O'Keeffe Date: Wed, 11 Mar 2026 08:06:09 +1100 Subject: [PATCH 3/5] fix: detect real WebSocket vs Socket.IO long-polling fallback Socket.IO sets connected=true even when falling back to its own long-polling (Databricks Apps proxy blocks WS upgrade). This stopped the fast poll-worker, routing all output through slow long-polling. Now checks socket.io.engine.transport.name and only stops poll-worker when transport is true 'websocket'. Also listens for late upgrades. Co-Authored-By: Claude Opus 4.6 --- static/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/static/index.html b/static/index.html index 0df0b68..2d3459f 100644 --- a/static/index.html +++ b/static/index.html @@ -879,7 +879,7 @@

General

// ── WebSocket Connection (AC-10, AC-11, AC-14) ──────────────── let socket = null; - let wsConnected = false; + let wsConnected = false; // true only when real WebSocket transport is active let wsHeartbeatTimer = null; const WS_HEARTBEAT_INTERVAL = 30000; // 30s — well within 5-min session timeout From 029c9e6a584630493e07c2e56ff6be2ee3bad9f8 Mon Sep 17 00:00:00 2001 From: David O'Keeffe Date: Wed, 18 Mar 2026 13:27:54 +1100 Subject: [PATCH 4/5] refactor: extract gh install to shell script Move inline gh CLI installation from app.py into install_gh.sh, following the same pattern as install_micro.sh. The script now fetches the latest 2.x release from the GitHub API instead of pinning a specific version. Also fixes a bug in the gh wrapper where the auth login handler fell through to exec gh.real with shifted args instead of exiting after handling the command. Co-authored-by: Isaac --- app.py | 26 +----------------------- install_gh.sh | 56 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 25 deletions(-) create mode 100644 install_gh.sh diff --git a/app.py b/app.py index 698798b..dd43eee 100644 --- a/app.py +++ b/app.py @@ -253,31 +253,7 @@ def run_setup(): _run_step("micro", ["bash", "-c", "mkdir -p ~/.local/bin && bash install_micro.sh && mv micro ~/.local/bin/ 2>/dev/null || true"]) - _run_step( - "gh", - [ - "bash", - "-c", - 'GH_VERSION="2.74.1" && ' - "mkdir -p ~/.local/bin && " - 'curl -fsSL "https://github.com/cli/cli/releases/download/v${GH_VERSION}/gh_${GH_VERSION}_linux_amd64.tar.gz" -o /tmp/gh.tar.gz && ' - "tar -xzf /tmp/gh.tar.gz -C /tmp && " - "mv /tmp/gh_${GH_VERSION}_linux_amd64/bin/gh ~/.local/bin/gh && " - "rm -rf /tmp/gh.tar.gz /tmp/gh_${GH_VERSION}_linux_amd64 && " - "chmod +x ~/.local/bin/gh && " - "gh config set git_protocol https 2>/dev/null || true && " - # Wrap gh auth login to skip interactive prompts (arrow-key menus break in xterm.js PTY) - "printf '#!/bin/bash\\n" - 'if [ "$1" = "auth" ] && [ "$2" = "login" ]; then\\n' - " shift 2\\n" - ' printf "Y\\\\n" | ~/.local/bin/gh.real auth login -h github.com -p https -w --skip-ssh-key "$@"\\n' - "fi\\n" - 'exec ~/.local/bin/gh.real "$@"\\n\' > ~/.local/bin/gh.wrapper && ' - "mv ~/.local/bin/gh ~/.local/bin/gh.real && " - "mv ~/.local/bin/gh.wrapper ~/.local/bin/gh && " - "chmod +x ~/.local/bin/gh", - ], - ) + _run_step("gh", ["bash", "install_gh.sh"]) # --- Content-filter proxy (must be running before OpenCode starts) --- # Sanitizes requests/responses between OpenCode and Databricks diff --git a/install_gh.sh b/install_gh.sh new file mode 100644 index 0000000..5980dc0 --- /dev/null +++ b/install_gh.sh @@ -0,0 +1,56 @@ +#!/bin/bash +set -e -u + +# Install the latest GitHub CLI (gh) 2.x release to ~/.local/bin +# and create a wrapper that handles `gh auth login` for xterm.js PTY compatibility. +# +# Usage: bash install_gh.sh + +INSTALL_DIR="${HOME}/.local/bin" +mkdir -p "${INSTALL_DIR}" + +# --- Detect latest 2.x version from GitHub API --- +latest_json="$(curl -fsSL 'https://api.github.com/repos/cli/cli/releases/latest' 2>/dev/null)" || true + +GH_VERSION="" +if [ -n "${latest_json:-}" ]; then + GH_VERSION="$(echo "${latest_json}" | grep -oEm1 '"tag_name":\s*"v(2\.[0-9]+\.[0-9]+)"' | grep -oE '2\.[0-9]+\.[0-9]+' || true)" +fi + +if [ -z "${GH_VERSION}" ]; then + echo "ERROR: Could not detect latest gh 2.x release from GitHub API." >&2 + echo "Check your network connection or GitHub API rate limits." >&2 + exit 1 +fi + +echo "Installing gh v${GH_VERSION}..." + +# --- Download and extract --- +TARBALL="/tmp/gh.tar.gz" +curl -fsSL "https://github.com/cli/cli/releases/download/v${GH_VERSION}/gh_${GH_VERSION}_linux_amd64.tar.gz" -o "${TARBALL}" +tar -xzf "${TARBALL}" -C /tmp +mv "/tmp/gh_${GH_VERSION}_linux_amd64/bin/gh" "${INSTALL_DIR}/gh" +rm -rf "${TARBALL}" "/tmp/gh_${GH_VERSION}_linux_amd64" +chmod +x "${INSTALL_DIR}/gh" + +# --- Configure git protocol --- +"${INSTALL_DIR}/gh" config set git_protocol https 2>/dev/null || true + +# --- Create wrapper script --- +# The wrapper intercepts `gh auth login` to pipe "Y" through the interactive +# prompt, which avoids arrow-key menus that break in xterm.js PTY sessions. +cat > "${INSTALL_DIR}/gh.wrapper" << 'WRAPPER' +#!/bin/bash +if [ "${1:-}" = "auth" ] && [ "${2:-}" = "login" ]; then + shift 2 + printf "Y\n" | ~/.local/bin/gh.real auth login -h github.com -p https -w --skip-ssh-key "$@" + exit 0 +fi +exec ~/.local/bin/gh.real "$@" +WRAPPER + +mv "${INSTALL_DIR}/gh" "${INSTALL_DIR}/gh.real" +mv "${INSTALL_DIR}/gh.wrapper" "${INSTALL_DIR}/gh" +chmod +x "${INSTALL_DIR}/gh" + +echo "gh v${GH_VERSION} installed to ${INSTALL_DIR}/gh" From fa824acce9e9d2af82241b0f5c207a133f315adf Mon Sep 17 00:00:00 2001 From: David O'Keeffe Date: Wed, 18 Mar 2026 13:28:00 +1100 Subject: [PATCH 5/5] feat: add clean and create-pat Makefile targets Add `make clean` to remove both the app and secret scope, helping with manual testing teardown. Add `make create-pat` to auto-generate a 90-day PAT via Databricks CLI and store it directly. Remove PAT from the usage example header to discourage passing secrets on the command line. Co-authored-by: Isaac --- Makefile | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 9c52e9a..ba3776e 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,12 @@ # Makefile for deploying Coding Agents to Databricks Apps # # Usage: -# make deploy PROFILE=daveok PAT=dapi... -# make deploy PROFILE=daveok # prompts for PAT interactively +# make deploy PROFILE=daveok # full deploy (prompts for PAT interactively) # make redeploy PROFILE=daveok # skip secret setup, just sync + deploy +# make create-pat PROFILE=daveok # auto-generate a 90-day PAT and store it # make status PROFILE=daveok # check app status # make logs PROFILE=daveok # tail app logs +# make clean PROFILE=daveok # remove app and secret scope # Configuration PROFILE ?= DEFAULT @@ -17,7 +18,7 @@ SECRET_KEY ?= databricks-token USER_EMAIL = $(shell databricks current-user me --profile $(PROFILE) --output json 2>/dev/null | python3 -c "import sys,json; print(json.load(sys.stdin).get('userName',''))") WORKSPACE_PATH = /Workspace/Users/$(USER_EMAIL)/apps/$(APP_NAME) -.PHONY: help deploy redeploy create-app setup-secret sync deploy-app status logs clean-secret +.PHONY: help deploy redeploy create-app setup-secret sync deploy-app status logs clean-secret clean create-pat help: ## Show this help @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf " \033[36m%-18s\033[0m %s\n", $$1, $$2}' @@ -85,3 +86,20 @@ logs: ## Tail app logs clean-secret: ## Remove secret scope (destructive) @echo "==> Removing secret scope '$(SECRET_SCOPE)'..." databricks secrets delete-scope $(SECRET_SCOPE) --profile $(PROFILE) + +clean: ## Remove app and secret scope (destructive) + @echo "==> Removing app '$(APP_NAME)'..." + @databricks apps delete $(APP_NAME) --profile $(PROFILE) 2>/dev/null && \ + echo " App '$(APP_NAME)' deleted." || \ + echo " App '$(APP_NAME)' not found or already deleted." + @echo "==> Removing secret scope '$(SECRET_SCOPE)'..." + @databricks secrets delete-scope $(SECRET_SCOPE) --profile $(PROFILE) 2>/dev/null && \ + echo " Secret scope '$(SECRET_SCOPE)' deleted." || \ + echo " Secret scope '$(SECRET_SCOPE)' not found or already deleted." + +create-pat: ## Generate a 90-day PAT and store it as the app secret + @echo "==> Generating a 90-day PAT..." + @databricks tokens create --lifetime-seconds $$((90 * 24 * 60 * 60)) --comment "coding-agents (auto-generated)" --profile $(PROFILE) --output json \ + | python3 -c "import sys,json; print(json.load(sys.stdin)['token_value'])" \ + | databricks secrets put-secret $(SECRET_SCOPE) $(SECRET_KEY) --profile $(PROFILE) + @echo " PAT created and stored in $(SECRET_SCOPE)/$(SECRET_KEY)"