diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..ba3776e --- /dev/null +++ b/Makefile @@ -0,0 +1,105 @@ +# Makefile for deploying Coding Agents to Databricks Apps +# +# Usage: +# make deploy PROFILE=daveok # full deploy (prompts for PAT interactively) +# make redeploy PROFILE=daveok # skip secret setup, just sync + deploy +# make create-pat PROFILE=daveok # auto-generate a 90-day PAT and store it +# make status PROFILE=daveok # check app status +# make logs PROFILE=daveok # tail app logs +# make clean PROFILE=daveok # remove app and secret scope + +# Configuration +PROFILE ?= DEFAULT +APP_NAME ?= coding-agents +SECRET_SCOPE ?= $(APP_NAME)-secrets +SECRET_KEY ?= databricks-token + +# Resolve user email and workspace path from the profile +USER_EMAIL = $(shell databricks current-user me --profile $(PROFILE) --output json 2>/dev/null | python3 -c "import sys,json; print(json.load(sys.stdin).get('userName',''))") +WORKSPACE_PATH = /Workspace/Users/$(USER_EMAIL)/apps/$(APP_NAME) + +.PHONY: help deploy redeploy create-app setup-secret sync deploy-app status logs clean-secret clean create-pat + +help: ## Show this help + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf " \033[36m%-18s\033[0m %s\n", $$1, $$2}' + +deploy: create-app setup-secret sync deploy-app ## Full deploy: create app, set secret, sync, deploy + @echo "" + @echo "Deployment complete! App URL:" + @databricks apps get $(APP_NAME) --profile $(PROFILE) --output json 2>/dev/null | python3 -c "import sys,json; print(json.load(sys.stdin).get('url','(pending)'))" + +redeploy: sync deploy-app ## Redeploy: sync + deploy (skip secret setup) + @echo "" + @echo "Redeployment complete!" + +create-app: ## Create the Databricks App (idempotent) + @echo "==> Checking if app '$(APP_NAME)' exists..." + @if databricks apps get $(APP_NAME) --profile $(PROFILE) >/dev/null 2>&1; then \ + echo " App '$(APP_NAME)' already exists, skipping create."; \ + else \ + echo " Creating app '$(APP_NAME)'..."; \ + databricks apps create $(APP_NAME) --profile $(PROFILE); \ + fi + +setup-secret: ## Create secret scope and store PAT + @echo "==> Setting up DATABRICKS_TOKEN secret..." + @# Create scope if it doesn't exist + @if databricks secrets list-scopes --profile $(PROFILE) --output json 2>/dev/null | python3 -c "import sys,json; scopes=[s['name'] for s in json.load(sys.stdin).get('scopes',[])]; exit(0 if '$(SECRET_SCOPE)' in scopes else 1)" 2>/dev/null; then \ + echo " Secret scope '$(SECRET_SCOPE)' already exists."; \ + else \ + echo " Creating secret scope '$(SECRET_SCOPE)'..."; \ + databricks secrets create-scope $(SECRET_SCOPE) --profile $(PROFILE); \ + fi + @# Store the PAT - prompt if not provided + @if [ -z "$(PAT)" ]; then \ + echo " Enter your Databricks PAT (will not echo):"; \ + read -s pat_value && \ + echo "$$pat_value" | databricks secrets put-secret $(SECRET_SCOPE) $(SECRET_KEY) --profile $(PROFILE); \ + else \ + echo "$(PAT)" | databricks secrets put-secret $(SECRET_SCOPE) $(SECRET_KEY) --profile $(PROFILE); \ + fi + @echo " Secret stored in $(SECRET_SCOPE)/$(SECRET_KEY)" + @# Link secret to app resource + @echo " Linking secret to app resource 'DATABRICKS_TOKEN'..." + @curl -s -X PATCH \ + "$$(databricks auth env --profile $(PROFILE) 2>/dev/null | python3 -c "import sys,json; print(json.load(sys.stdin)['env']['DATABRICKS_HOST'])")/api/2.0/apps/$(APP_NAME)" \ + -H "Authorization: Bearer $$(databricks auth token --profile $(PROFILE) 2>/dev/null | python3 -c "import sys,json; print(json.load(sys.stdin)['access_token'])")" \ + -H "Content-Type: application/json" \ + -d '{"resources":[{"name":"DATABRICKS_TOKEN","description":"PAT for model serving access","secret":{"scope":"$(SECRET_SCOPE)","key":"$(SECRET_KEY)","permission":"READ"}}]}' \ + >/dev/null + @echo " App resource linked." + +sync: ## Sync local files to Databricks workspace + @echo "==> Syncing to $(WORKSPACE_PATH)..." + databricks sync . $(WORKSPACE_PATH) --watch=false --profile $(PROFILE) + +deploy-app: ## Deploy the app from workspace + @echo "==> Deploying app '$(APP_NAME)'..." + databricks apps deploy $(APP_NAME) --source-code-path $(WORKSPACE_PATH) --profile $(PROFILE) --no-wait + +status: ## Check app status + @databricks apps get $(APP_NAME) --profile $(PROFILE) + +logs: ## Tail app logs + databricks apps logs $(APP_NAME) --profile $(PROFILE) + +clean-secret: ## Remove secret scope (destructive) + @echo "==> Removing secret scope '$(SECRET_SCOPE)'..." + databricks secrets delete-scope $(SECRET_SCOPE) --profile $(PROFILE) + +clean: ## Remove app and secret scope (destructive) + @echo "==> Removing app '$(APP_NAME)'..." + @databricks apps delete $(APP_NAME) --profile $(PROFILE) 2>/dev/null && \ + echo " App '$(APP_NAME)' deleted." || \ + echo " App '$(APP_NAME)' not found or already deleted." + @echo "==> Removing secret scope '$(SECRET_SCOPE)'..." + @databricks secrets delete-scope $(SECRET_SCOPE) --profile $(PROFILE) 2>/dev/null && \ + echo " Secret scope '$(SECRET_SCOPE)' deleted." || \ + echo " Secret scope '$(SECRET_SCOPE)' not found or already deleted." + +create-pat: ## Generate a 90-day PAT and store it as the app secret + @echo "==> Generating a 90-day PAT..." + @databricks tokens create --lifetime-seconds $$((90 * 24 * 60 * 60)) --comment "coding-agents (auto-generated)" --profile $(PROFILE) --output json \ + | python3 -c "import sys,json; print(json.load(sys.stdin)['token_value'])" \ + | databricks secrets put-secret $(SECRET_SCOPE) $(SECRET_KEY) --profile $(PROFILE) + @echo " PAT created and stored in $(SECRET_SCOPE)/$(SECRET_KEY)" diff --git a/app.py b/app.py index 5526a6b..dd43eee 100644 --- a/app.py +++ b/app.py @@ -83,6 +83,7 @@ def handle_sigterm(signum, frame): "steps": [ {"id": "git", "label": "Configuring git identity", "status": "pending", "started_at": None, "completed_at": None, "error": None}, {"id": "micro", "label": "Installing micro editor", "status": "pending", "started_at": None, "completed_at": None, "error": None}, + {"id": "gh", "label": "Installing GitHub CLI", "status": "pending", "started_at": None, "completed_at": None, "error": None}, {"id": "proxy", "label": "Starting content-filter proxy", "status": "pending", "started_at": None, "completed_at": None, "error": None}, {"id": "claude", "label": "Configuring Claude CLI", "status": "pending", "started_at": None, "completed_at": None, "error": None}, {"id": "codex", "label": "Configuring Codex CLI", "status": "pending", "started_at": None, "completed_at": None, "error": None}, @@ -252,10 +253,12 @@ def run_setup(): _run_step("micro", ["bash", "-c", "mkdir -p ~/.local/bin && bash install_micro.sh && mv micro ~/.local/bin/ 2>/dev/null || true"]) + _run_step("gh", ["bash", "install_gh.sh"]) + # --- Content-filter proxy (must be running before OpenCode starts) --- # Sanitizes requests/responses between OpenCode and Databricks # (see OpenCode #5028, docs/plans/2026-03-11-litellm-empty-content-blocks-design.md) - _run_step("proxy", ["python", "setup_proxy.py"]) + _run_step("proxy", ["uv", "run", "python", "setup_proxy.py"]) # --- Parallel agent setup (all independent of each other) --- parallel_steps = [ @@ -487,7 +490,7 @@ def read_pty_output(session_id, fd): if session_id not in sessions: break try: - readable, _, errors = select.select([fd], [], [fd], 0.5) + readable, _, errors = select.select([fd], [], [fd], 0.05) if readable or errors: output = os.read(fd, 4096) if not output: @@ -569,7 +572,10 @@ def cleanup_stale_sessions(): warning_threshold = SESSION_TIMEOUT_SECONDS * 0.8 with sessions_lock: - for session_id, session in sessions.items(): + session_snapshot = list(sessions.items()) + + for session_id, session in session_snapshot: + with session["lock"]: idle = now - session["last_poll_time"] if idle > SESSION_TIMEOUT_SECONDS: stale_sessions.append((session_id, session["pid"], session["master_fd"])) @@ -801,22 +807,31 @@ def get_output_batch(): outputs = {} now = time.time() + # Step 1: Resolve session refs under global lock (fast dict lookups only) + resolved = {} with sessions_lock: for sid in session_ids: - if sid not in sessions: - continue - session = sessions[sid] + if sid in sessions: + resolved[sid] = sessions[sid] + + # Step 2: Swap buffers under per-session locks (same pattern as get_output) + swapped = {} + for sid, session in resolved.items(): + with session["lock"]: session["last_poll_time"] = now - buffer = session["output_buffer"] - output = "".join(buffer) - buffer.clear() + old_buffer = session["output_buffer"] + session["output_buffer"] = deque(maxlen=1000) exited = session.get("exited", False) timeout_warning = session.pop("timeout_warning", False) - outputs[sid] = { - "output": output, - "exited": exited, - "timeout_warning": timeout_warning - } + swapped[sid] = (old_buffer, exited, timeout_warning) + + # Step 3: Join strings outside all locks + for sid, (old_buffer, exited, timeout_warning) in swapped.items(): + outputs[sid] = { + "output": "".join(old_buffer), + "exited": exited, + "timeout_warning": timeout_warning, + } return jsonify({"outputs": outputs, "shutting_down": shutting_down}) diff --git a/install_gh.sh b/install_gh.sh new file mode 100644 index 0000000..5980dc0 --- /dev/null +++ b/install_gh.sh @@ -0,0 +1,56 @@ +#!/bin/bash +set -e -u + +# Install the latest GitHub CLI (gh) 2.x release to ~/.local/bin +# and create a wrapper that handles `gh auth login` for xterm.js PTY compatibility. +# +# Usage: bash install_gh.sh + +INSTALL_DIR="${HOME}/.local/bin" +mkdir -p "${INSTALL_DIR}" + +# --- Detect latest 2.x version from GitHub API --- +latest_json="$(curl -fsSL 'https://api.github.com/repos/cli/cli/releases/latest' 2>/dev/null)" || true + +GH_VERSION="" +if [ -n "${latest_json:-}" ]; then + GH_VERSION="$(echo "${latest_json}" | grep -oEm1 '"tag_name":\s*"v(2\.[0-9]+\.[0-9]+)"' | grep -oE '2\.[0-9]+\.[0-9]+' || true)" +fi + +if [ -z "${GH_VERSION}" ]; then + echo "ERROR: Could not detect latest gh 2.x release from GitHub API." >&2 + echo "Check your network connection or GitHub API rate limits." >&2 + exit 1 +fi + +echo "Installing gh v${GH_VERSION}..." + +# --- Download and extract --- +TARBALL="/tmp/gh.tar.gz" +curl -fsSL "https://github.com/cli/cli/releases/download/v${GH_VERSION}/gh_${GH_VERSION}_linux_amd64.tar.gz" -o "${TARBALL}" +tar -xzf "${TARBALL}" -C /tmp +mv "/tmp/gh_${GH_VERSION}_linux_amd64/bin/gh" "${INSTALL_DIR}/gh" +rm -rf "${TARBALL}" "/tmp/gh_${GH_VERSION}_linux_amd64" +chmod +x "${INSTALL_DIR}/gh" + +# --- Configure git protocol --- +"${INSTALL_DIR}/gh" config set git_protocol https 2>/dev/null || true + +# --- Create wrapper script --- +# The wrapper intercepts `gh auth login` to pipe "Y" through the interactive +# prompt, which avoids arrow-key menus that break in xterm.js PTY sessions. +cat > "${INSTALL_DIR}/gh.wrapper" << 'WRAPPER' +#!/bin/bash +if [ "${1:-}" = "auth" ] && [ "${2:-}" = "login" ]; then + shift 2 + printf "Y\n" | ~/.local/bin/gh.real auth login -h github.com -p https -w --skip-ssh-key "$@" + exit 0 +fi +exec ~/.local/bin/gh.real "$@" +WRAPPER + +mv "${INSTALL_DIR}/gh" "${INSTALL_DIR}/gh.real" +mv "${INSTALL_DIR}/gh.wrapper" "${INSTALL_DIR}/gh" +chmod +x "${INSTALL_DIR}/gh" + +echo "gh v${GH_VERSION} installed to ${INSTALL_DIR}/gh" diff --git a/static/index.html b/static/index.html index 0df0b68..2d3459f 100644 --- a/static/index.html +++ b/static/index.html @@ -879,7 +879,7 @@

General

// ── WebSocket Connection (AC-10, AC-11, AC-14) ──────────────── let socket = null; - let wsConnected = false; + let wsConnected = false; // true only when real WebSocket transport is active let wsHeartbeatTimer = null; const WS_HEARTBEAT_INTERVAL = 30000; // 30s — well within 5-min session timeout diff --git a/static/poll-worker.js b/static/poll-worker.js index a499b0a..4315b5f 100644 --- a/static/poll-worker.js +++ b/static/poll-worker.js @@ -21,7 +21,7 @@ "use strict"; // ── Constants ───────────────────────────────────────────────────────────── -const POLL_INTERVAL_FG = 100; // ms — foreground batch poll +const POLL_INTERVAL_FG = 50; // ms — foreground batch poll const HEARTBEAT_INTERVAL_BG = 30000; // ms — background heartbeat const RETRY_BASE_MS = 500; const RETRY_MULTIPLIER = 2;