diff --git a/drizzle/0007_bent_nocturne.sql b/drizzle/0007_bent_nocturne.sql new file mode 100644 index 00000000..ef700f97 --- /dev/null +++ b/drizzle/0007_bent_nocturne.sql @@ -0,0 +1,14 @@ +-- Preserve the intent represented by pre-reconciliation rows before removing +-- the legacy transitional status column. +UPDATE `projects` +SET `desired_status` = CASE `status` + WHEN 'created' THEN 'created' + WHEN 'starting' THEN 'running' + WHEN 'running' THEN 'running' + WHEN 'stopping' THEN 'stopped' + WHEN 'stopped' THEN 'stopped' + WHEN 'error' THEN 'running' + WHEN 'deleting' THEN 'deleting' + ELSE `desired_status` +END;--> statement-breakpoint +ALTER TABLE `projects` DROP COLUMN `status`; diff --git a/drizzle/meta/0007_snapshot.json b/drizzle/meta/0007_snapshot.json new file mode 100644 index 00000000..bb788aca --- /dev/null +++ b/drizzle/meta/0007_snapshot.json @@ -0,0 +1,958 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "21f3c60d-42fa-4cfa-8139-b2efb2578752", + "prevId": "c7e7f55c-ce33-4306-90ed-80036a7036ae", + "tables": { + "instance_settings": { + "name": "instance_settings", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "base_url": { + "name": "base_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tailscale_enabled": { + "name": "tailscale_enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "tailscale_auth_key": { + "name": "tailscale_auth_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tailscale_hostname": { + "name": "tailscale_hostname", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tailscale_tailnet_name": { + "name": "tailscale_tailnet_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "model_favorites": { + "name": "model_favorites", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model_id": { + "name": "model_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "model_favorites_user_id_idx": { + "name": "model_favorites_user_id_idx", + "columns": [ + "user_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "model_favorites_user_id_users_id_fk": { + "name": "model_favorites_user_id_users_id_fk", + "tableFrom": "model_favorites", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "projects": { + "name": "projects", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "owner_user_id": { + "name": "owner_user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "icon": { + "name": "icon", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'✨'" + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "prompt": { + "name": "prompt", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "dev_port": { + "name": "dev_port", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "desired_status": { + "name": "desired_status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'created'" + }, + "observed_status": { + "name": "observed_status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'unknown'" + }, + "last_reconciled_at": { + "name": "last_reconciled_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "path_on_disk": { + "name": "path_on_disk", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "initial_prompt_sent": { + "name": "initial_prompt_sent", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "initial_prompt_completed": { + "name": "initial_prompt_completed", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "bootstrap_session_id": { + "name": "bootstrap_session_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_prompt_message_id": { + "name": "user_prompt_message_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_prompt_completed": { + "name": "user_prompt_completed", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "bootstrap_agent_status": { + "name": "bootstrap_agent_status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'idle'" + }, + "bootstrap_agent_last_activity_at": { + "name": "bootstrap_agent_last_activity_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "production_port": { + "name": "production_port", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "production_url": { + "name": "production_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "production_status": { + "name": "production_status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'stopped'" + }, + "production_started_at": { + "name": "production_started_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "production_error": { + "name": "production_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "production_hash": { + "name": "production_hash", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "opencode_error_category": { + "name": "opencode_error_category", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "opencode_error_code": { + "name": "opencode_error_code", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "opencode_error_message": { + "name": "opencode_error_message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "opencode_error_source": { + "name": "opencode_error_source", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "opencode_error_at": { + "name": "opencode_error_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "preferred_model": { + "name": "preferred_model", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "projects_slug_unique": { + "name": "projects_slug_unique", + "columns": [ + "slug" + ], + "isUnique": true + }, + "projects_owner_user_id_idx": { + "name": "projects_owner_user_id_idx", + "columns": [ + "owner_user_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "projects_owner_user_id_users_id_fk": { + "name": "projects_owner_user_id_users_id_fk", + "tableFrom": "projects", + "tableTo": "users", + "columnsFrom": [ + "owner_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "queue_jobs": { + "name": "queue_jobs", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'queued'" + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "payload_json": { + "name": "payload_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "priority": { + "name": "priority", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "max_attempts": { + "name": "max_attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 3 + }, + "run_at": { + "name": "run_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "locked_at": { + "name": "locked_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "lock_expires_at": { + "name": "lock_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "locked_by": { + "name": "locked_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "dedupe_key": { + "name": "dedupe_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "dedupe_active": { + "name": "dedupe_active", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "healed_at": { + "name": "healed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "heal_reason": { + "name": "heal_reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cancel_requested_at": { + "name": "cancel_requested_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cancelled_at": { + "name": "cancelled_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "queue_jobs_project_id_idx": { + "name": "queue_jobs_project_id_idx", + "columns": [ + "project_id" + ], + "isUnique": false + }, + "queue_jobs_runnable_idx": { + "name": "queue_jobs_runnable_idx", + "columns": [ + "state", + "run_at", + "lock_expires_at" + ], + "isUnique": false + }, + "queue_jobs_dedupe_idx": { + "name": "queue_jobs_dedupe_idx", + "columns": [ + "dedupe_key", + "dedupe_active" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "queue_settings": { + "name": "queue_settings", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "paused": { + "name": "paused", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "concurrency": { + "name": "concurrency", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 2 + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "sessions": { + "name": "sessions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token_hash": { + "name": "token_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "sessions_token_hash_unique": { + "name": "sessions_token_hash_unique", + "columns": [ + "token_hash" + ], + "isUnique": true + } + }, + "foreignKeys": { + "sessions_user_id_users_id_fk": { + "name": "sessions_user_id_users_id_fk", + "tableFrom": "sessions", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "system_health_snapshots": { + "name": "system_health_snapshots", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "taken_at": { + "name": "taken_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "queue_jobs_queued": { + "name": "queue_jobs_queued", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "queue_jobs_running": { + "name": "queue_jobs_running", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "queue_jobs_failed": { + "name": "queue_jobs_failed", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "queue_orphaned_jobs": { + "name": "queue_orphaned_jobs", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "queue_impossible_jobs": { + "name": "queue_impossible_jobs", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "projects_total": { + "name": "projects_total", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "projects_running": { + "name": "projects_running", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "projects_error": { + "name": "projects_error", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "projects_healthy_mismatch": { + "name": "projects_healthy_mismatch", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "opencode_healthy": { + "name": "opencode_healthy", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "docker_network_exists": { + "name": "docker_network_exists", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "docker_volume_exists": { + "name": "docker_volume_exists", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "violations_found": { + "name": "violations_found", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "violations_healed": { + "name": "violations_healed", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconciliation_duration_ms": { + "name": "reconciliation_duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "system_health_snapshots_taken_at_idx": { + "name": "system_health_snapshots_taken_at_idx", + "columns": [ + "taken_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "user_settings": { + "name": "user_settings", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "openrouter_api_key": { + "name": "openrouter_api_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "default_model": { + "name": "default_model", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "fast_model": { + "name": "fast_model", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "user_settings_user_id_users_id_fk": { + "name": "user_settings_user_id_users_id_fk", + "tableFrom": "user_settings", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "users": { + "name": "users", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "username": { + "name": "username", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "password_hash": { + "name": "password_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json index 14c79eed..f09b7715 100644 --- a/drizzle/meta/_journal.json +++ b/drizzle/meta/_journal.json @@ -50,6 +50,13 @@ "when": 1778777722385, "tag": "0006_keen_domino", "breakpoints": true + }, + { + "idx": 7, + "version": "6", + "when": 1784301722359, + "tag": "0007_bent_nocturne", + "breakpoints": true } ] } \ No newline at end of file diff --git a/src/actions/projects.delete.ts b/src/actions/projects.delete.ts index 0426b844..8f51a4b8 100644 --- a/src/actions/projects.delete.ts +++ b/src/actions/projects.delete.ts @@ -2,7 +2,7 @@ import { ActionError, defineAction } from "astro:actions"; import { z } from "astro/zod"; import { isProjectOwnedByUser, - updateProjectStatus, + updateProjectDesiredStatus, } from "@/server/projects/projects.model"; import { enqueueDeleteAllProjectsForUser, @@ -31,7 +31,7 @@ export const deleteProject = defineAction({ } try { - await updateProjectStatus(input.projectId, "deleting"); + await updateProjectDesiredStatus(input.projectId, "deleting"); } catch {} try { diff --git a/src/actions/projects.lifecycle.ts b/src/actions/projects.lifecycle.ts index 8960c41a..f6338548 100644 --- a/src/actions/projects.lifecycle.ts +++ b/src/actions/projects.lifecycle.ts @@ -67,7 +67,7 @@ export const restart = defineAction({ "@/server/queue/enqueue" ); - const { updateProjectStatus } = await import( + const { updateProjectDesiredStatus } = await import( "@/server/projects/projects.model" ); @@ -76,7 +76,7 @@ export const restart = defineAction({ reason: "user", }); - await updateProjectStatus(input.projectId, "starting"); + await updateProjectDesiredStatus(input.projectId, "running"); return { success: true, jobId: job.id }; }, diff --git a/src/actions/projects.production.ts b/src/actions/projects.production.ts index 4fbf9f6a..ee356ac3 100644 --- a/src/actions/projects.production.ts +++ b/src/actions/projects.production.ts @@ -38,10 +38,13 @@ export const deploy = defineAction({ }); } - if (project.status !== "running") { + if ( + project.desiredStatus !== "running" || + project.observedStatus !== "healthy" + ) { throw new ActionError({ code: "BAD_REQUEST", - message: `Cannot deploy project while it's ${project.status}`, + message: `Cannot deploy project while desired=${project.desiredStatus} and observed=${project.observedStatus}`, }); } diff --git a/src/actions/tailscale.ts b/src/actions/tailscale.ts index 88fa97c0..7453cd77 100644 --- a/src/actions/tailscale.ts +++ b/src/actions/tailscale.ts @@ -8,7 +8,7 @@ import { composeDown } from "@/server/docker/compose"; import { getProjectPreviewPath } from "@/server/projects/paths"; import { getProjectsByUserId, - updateProjectStatus, + updateProjectDesiredStatus, } from "@/server/projects/projects.model"; import { enqueueDockerEnsureRunning } from "@/server/queue/enqueue"; import { @@ -134,7 +134,7 @@ async function getPreviewProjectsNeedingTailscale(userId: string) { const projects = await getProjectsByUserId(userId); const runningProjects = projects.filter( - (project) => project.status === "running", + (project) => project.desiredStatus === "running", ); const checks = await Promise.all( runningProjects.map(async (project) => ({ @@ -256,7 +256,7 @@ export const tailscale = { for (const project of projectsNeedingPreviewRestart) { await composeDown(project.id, getProjectPreviewPath(project.id)); - await updateProjectStatus(project.id, "starting"); + await updateProjectDesiredStatus(project.id, "running"); await enqueueDockerEnsureRunning({ projectId: project.id, reason: "user", diff --git a/src/components/projects/ProjectCard.tsx b/src/components/projects/ProjectCard.tsx index f1a8051c..2524033a 100644 --- a/src/components/projects/ProjectCard.tsx +++ b/src/components/projects/ProjectCard.tsx @@ -94,7 +94,10 @@ export function ProjectCard({ project, onDeleted }: ProjectCardProps) { typeof window === "undefined" ? (productionUrls.preferred ?? project.productionUrl) : getPreferredRuntimeUrl(productionUrls, baseUrl, window.location.origin); - const isPreviewRunning = project.status === "running" && !isStoppingPreview; + const isPreviewRunning = + project.desiredStatus === "running" && + project.observedStatus === "healthy" && + !isStoppingPreview; const isProductionRunning = project.productionStatus === "running" && Boolean(productionUrl) && @@ -290,7 +293,7 @@ export function ProjectCard({ project, onDeleted }: ProjectCardProps) { size="icon" className="h-8 w-8 bg-card/80 text-destructive backdrop-blur-sm hover:text-destructive hover:bg-destructive/10" onClick={handleDeleteClick} - disabled={isDeleting || project.status === "deleting"} + disabled={isDeleting || project.desiredStatus === "deleting"} > {isDeleting ? ( diff --git a/src/pages/api/system/health.ts b/src/pages/api/system/health.ts index 48102c97..87ad63c1 100644 --- a/src/pages/api/system/health.ts +++ b/src/pages/api/system/health.ts @@ -22,18 +22,30 @@ export const GET: APIRoute = async ({ cookies }) => { queueStats.map((s) => [s.state, s.count]), ); - // Project stats - const projectStats = await db + // Project stats expose intent and runtime health separately. A project can be + // desired=running while observed=unhealthy, which is the useful signal here. + const desiredProjectStats = await db .select({ - status: projects.status, + status: projects.desiredStatus, count: count(), }) .from(projects) .where(isNull(projects.deletedAt)) - .groupBy(projects.status); + .groupBy(projects.desiredStatus); + const observedProjectStats = await db + .select({ + status: projects.observedStatus, + count: count(), + }) + .from(projects) + .where(isNull(projects.deletedAt)) + .groupBy(projects.observedStatus); - const projectMap = Object.fromEntries( - projectStats.map((s) => [s.status, s.count]), + const desiredProjectMap = Object.fromEntries( + desiredProjectStats.map((s) => [s.status, s.count]), + ); + const observedProjectMap = Object.fromEntries( + observedProjectStats.map((s) => [s.status, s.count]), ); // Latest snapshot @@ -56,14 +68,20 @@ export const GET: APIRoute = async ({ cookies }) => { cancelled: queueMap.cancelled || 0, }, projects: { - total: Object.values(projectMap).reduce((a, b) => a + b, 0), - created: projectMap.created || 0, - starting: projectMap.starting || 0, - running: projectMap.running || 0, - stopping: projectMap.stopping || 0, - stopped: projectMap.stopped || 0, - error: projectMap.error || 0, - deleting: projectMap.deleting || 0, + total: Object.values(desiredProjectMap).reduce((a, b) => a + b, 0), + desired: { + created: desiredProjectMap.created || 0, + running: desiredProjectMap.running || 0, + stopped: desiredProjectMap.stopped || 0, + deleting: desiredProjectMap.deleting || 0, + }, + observed: { + unknown: observedProjectMap.unknown || 0, + healthy: observedProjectMap.healthy || 0, + unhealthy: observedProjectMap.unhealthy || 0, + missing: observedProjectMap.missing || 0, + crashed: observedProjectMap.crashed || 0, + }, }, infrastructure: { opencodeHealthy, diff --git a/src/server/ai-tools/doce-preview/service.ts b/src/server/ai-tools/doce-preview/service.ts index 28e6a853..c91b3295 100644 --- a/src/server/ai-tools/doce-preview/service.ts +++ b/src/server/ai-tools/doce-preview/service.ts @@ -46,7 +46,7 @@ export async function getDocePreviewStatus( const previewReachable = await checkPreviewReady(projectId); const summary = buildStatusSummary({ - projectStatus: project.status, + projectStatus: `${project.desiredStatus}/${project.observedStatus}`, previewReachable, containers, }); @@ -54,7 +54,7 @@ export async function getDocePreviewStatus( return { ok: true, projectId, - projectStatus: project.status, + projectStatus: `${project.desiredStatus}/${project.observedStatus}`, preview: { reachable: previewReachable, }, diff --git a/src/server/db/schema.ts b/src/server/db/schema.ts index 462194bf..8b549407 100644 --- a/src/server/db/schema.ts +++ b/src/server/db/schema.ts @@ -68,20 +68,6 @@ export const projects = sqliteTable( prompt: text("prompt").notNull(), description: text("description").notNull().default(""), devPort: integer("dev_port").notNull(), - // Deprecated: use desired_status instead. Kept for backward compatibility. - status: text("status", { - enum: [ - "created", - "starting", - "running", - "stopping", - "stopped", - "error", - "deleting", - ], - }) - .notNull() - .default("created"), // Desired state: what we want the project to be desiredStatus: text("desired_status", { enum: ["created", "running", "stopped", "deleting"], diff --git a/src/server/effect/layers.ts b/src/server/effect/layers.ts index 437d88c3..8633a739 100644 --- a/src/server/effect/layers.ts +++ b/src/server/effect/layers.ts @@ -271,11 +271,11 @@ export class DatabaseService extends Context.Tag("DatabaseService")< ) => Effect.Effect; /** - * Update project status + * Update desired project status */ - updateProjectStatus: ( + updateProjectDesiredStatus: ( projectId: string, - status: Project["status"], + status: Project["desiredStatus"], ) => Effect.Effect; /** @@ -316,7 +316,7 @@ export const DatabaseServiceLive = Layer.succeed( DatabaseService.of({ getProjectById: () => Effect.dieMessage("DatabaseServiceLive not implemented"), - updateProjectStatus: () => + updateProjectDesiredStatus: () => Effect.dieMessage("DatabaseServiceLive not implemented"), updateProjectProductionStatus: () => Effect.dieMessage("DatabaseServiceLive not implemented"), diff --git a/src/server/live/manager.ts b/src/server/live/manager.ts index 4b4fc5f4..e365351f 100644 --- a/src/server/live/manager.ts +++ b/src/server/live/manager.ts @@ -20,7 +20,7 @@ import { } from "@/server/projects/health"; import { getProjectById, - updateProjectStatus, + updateProjectDesiredStatus, } from "@/server/projects/projects.model"; import { getProjectRuntimeUrls } from "@/server/projects/projectUrls"; import { @@ -204,13 +204,12 @@ async function ensureRunning( const project = await getProjectById(projectId); if (!project) return; const needsStart = - project.status === "created" || - project.status === "stopped" || - project.status === "error"; + project.desiredStatus === "created" || project.desiredStatus === "stopped"; if (!needsStart) return; state.startedAt = Date.now(); try { + await updateProjectDesiredStatus(projectId, "running"); await enqueueDockerEnsureRunning({ projectId, reason: "presence" }); } catch (error) { logger.error({ error, projectId }, "Failed to enqueue ensure-running"); @@ -273,42 +272,42 @@ async function buildState( activeJobType: activeProductionJob?.type ?? null, }; - let status = project.status; + let status: ProjectLiveState["status"] = project.desiredStatus; let message: string | null = null; - if (previewReady && opencodeReady) { - if (status !== "running") { - await updateProjectStatus(projectId, "running"); + if (project.desiredStatus === "running") { + if (previewReady && opencodeReady) { status = "running"; - } - state.startedAt = null; - } else if (status === "running") { - await updateProjectStatus(projectId, "stopped"); - status = "stopped"; - state.startedAt = Date.now(); - void enqueueDockerEnsureRunning({ projectId, reason: "presence" }).catch( - (error) => logger.error({ error, projectId }, "Recovery enqueue failed"), - ); - message = `Restarting ${[!previewReady ? "preview" : null, !opencodeReady ? "opencode" : null].filter(Boolean).join(" and ")}...`; - } else if ( - status === "starting" || - status === "created" || - status === "stopped" - ) { - status = "starting"; - if (!state.startedAt) state.startedAt = Date.now(); - const elapsed = Date.now() - state.startedAt; - if (elapsed > START_TIMEOUT_MS) { - await updateProjectStatus(projectId, "error"); - status = "error"; state.startedAt = null; + } else if (project.observedStatus === "healthy") { + status = "starting"; + if (!state.startedAt) state.startedAt = Date.now(); + void enqueueDockerEnsureRunning({ projectId, reason: "presence" }).catch( + (error) => + logger.error({ error, projectId }, "Recovery enqueue failed"), + ); + message = `Restarting ${[!previewReady ? "preview" : null, !opencodeReady ? "opencode" : null].filter(Boolean).join(" and ")}...`; + } else if ( + project.observedStatus === "unhealthy" || + project.observedStatus === "crashed" + ) { + status = "error"; message = "Containers failed to start. Open terminal for details."; - } else if (!previewReady && !opencodeReady) { - message = "Starting containers..."; - } else if (!previewReady) { - message = "Waiting for preview server..."; } else { - message = "Waiting for opencode..."; + status = "starting"; + if (!state.startedAt) state.startedAt = Date.now(); + const elapsed = Date.now() - state.startedAt; + if (elapsed > START_TIMEOUT_MS) { + status = "error"; + state.startedAt = null; + message = "Containers failed to start. Open terminal for details."; + } else if (!previewReady && !opencodeReady) { + message = "Starting containers..."; + } else if (!previewReady) { + message = "Waiting for preview server..."; + } else { + message = "Waiting for opencode..."; + } } } diff --git a/src/server/projects/delete.ts b/src/server/projects/delete.ts index e0a6b9ec..9c8b80a5 100644 --- a/src/server/projects/delete.ts +++ b/src/server/projects/delete.ts @@ -7,7 +7,8 @@ import { getProjectById, getProjectsByUserId, hardDeleteProject, - updateProjectStatus, + updateProjectDesiredStatus, + updateProjectObservedStatus, } from "./projects.model"; export interface DeleteProjectResult { @@ -164,24 +165,24 @@ export async function stopProject( const projectPath = getProjectPath(projectId); try { - await updateProjectStatus(projectId, "stopping"); + await updateProjectDesiredStatus(projectId, "stopped"); const { composeStop } = await import("@/server/docker/compose"); const result = await composeStop(projectId, projectPath); if (result.success) { - await updateProjectStatus(projectId, "stopped"); + await updateProjectObservedStatus(projectId, "missing"); logger.info({ projectId }, "Project stopped"); return { success: true }; } else { - await updateProjectStatus(projectId, "error"); + await updateProjectObservedStatus(projectId, "unhealthy"); return { success: false, error: `Docker compose stop failed: ${result.stderr.slice(0, 200)}`, }; } } catch (err) { - await updateProjectStatus(projectId, "error"); + await updateProjectObservedStatus(projectId, "unhealthy"); logger.error({ error: err, projectId }, "Failed to stop project"); return { success: false, diff --git a/src/server/projects/projects.db.ts b/src/server/projects/projects.db.ts index a74adcb3..83971228 100644 --- a/src/server/projects/projects.db.ts +++ b/src/server/projects/projects.db.ts @@ -4,7 +4,8 @@ import { type NewProject, type Project, projects } from "@/server/db/schema"; import type { OpencodeDiagnostic } from "@/server/opencode/diagnostics"; import { emitProjectEvent } from "./events"; -export type ProjectStatus = Project["status"]; +export type DesiredProjectStatus = NonNullable; +export type ObservedProjectStatus = NonNullable; /** * Create a new project in the database. @@ -58,46 +59,29 @@ export async function getProjectsByUserId(userId: string): Promise { and( eq(projects.ownerUserId, userId), isNull(projects.deletedAt), - ne(projects.status, "deleting"), + ne(projects.desiredStatus, "deleting"), ), ) .orderBy(desc(projects.createdAt)); } -/** - * Map legacy status to desired status for backward compatibility. - */ -function statusToDesired(status: ProjectStatus): string { - switch (status) { - case "created": - return "created"; - case "starting": - return "running"; - case "running": - return "running"; - case "stopping": - return "stopped"; - case "stopped": - return "stopped"; - case "error": - return "running"; // Assume user wants it running - case "deleting": - return "deleting"; - } +/** Update the state the user or lifecycle intends the project to reach. */ +export async function updateProjectDesiredStatus( + id: string, + desiredStatus: DesiredProjectStatus, +): Promise { + await db.update(projects).set({ desiredStatus }).where(eq(projects.id, id)); + emitProjectEvent(id); } -/** - * Update a project's status. - * Also updates desired_status for the new self-healing system. - */ -export async function updateProjectStatus( +/** Update the state observed from the project's runtime. */ +export async function updateProjectObservedStatus( id: string, - status: ProjectStatus, + observedStatus: ObservedProjectStatus, ): Promise { - const desiredStatus = statusToDesired(status) as Project["desiredStatus"]; await db .update(projects) - .set({ status, desiredStatus }) + .set({ observedStatus, lastReconciledAt: new Date() }) .where(eq(projects.id, id)); emitProjectEvent(id); } @@ -191,7 +175,9 @@ export async function getRunningProjects(): Promise { return db .select() .from(projects) - .where(and(eq(projects.status, "running"), isNull(projects.deletedAt))); + .where( + and(eq(projects.desiredStatus, "running"), isNull(projects.deletedAt)), + ); } /** diff --git a/src/server/projects/sessionDescription.ts b/src/server/projects/sessionDescription.ts index 17a83f40..7067a813 100644 --- a/src/server/projects/sessionDescription.ts +++ b/src/server/projects/sessionDescription.ts @@ -34,7 +34,7 @@ export async function updateProjectDescriptionFromSessionTitle( if (!description) return false; const project = await getProjectByIdIncludeDeleted(projectId); - if (!project || project.status === "deleting") return false; + if (!project || project.desiredStatus === "deleting") return false; if (project.description === description) return true; await updateProjectDescription(projectId, description); diff --git a/src/server/queue/handlers/dockerComposeUp.ts b/src/server/queue/handlers/dockerComposeUp.ts index d246a25e..bb8ed9be 100644 --- a/src/server/queue/handlers/dockerComposeUp.ts +++ b/src/server/queue/handlers/dockerComposeUp.ts @@ -8,7 +8,8 @@ import { logger } from "@/server/logger"; import { getProjectPreviewPath } from "@/server/projects/paths"; import { getProjectByIdIncludeDeleted, - updateProjectStatus, + updateProjectDesiredStatus, + updateProjectObservedStatus, } from "@/server/projects/projects.model"; import { enqueueDockerWaitReady } from "../enqueue"; import { parsePayload } from "../types"; @@ -19,10 +20,18 @@ const fetchProjectById = (projectId: string) => catch: () => new ProjectNotFoundError({ projectId }), }); -const setProjectStatus = (projectId: string, status: "starting" | "error") => +const setProjectDesiredStatus = (projectId: string) => Effect.tryPromise({ - try: () => updateProjectStatus(projectId, status), - catch: (error) => new Error(`Failed to update project status: ${error}`), + try: () => updateProjectDesiredStatus(projectId, "running"), + catch: (error) => + new Error(`Failed to update desired project status: ${error}`), + }); + +const setProjectObservedStatus = (projectId: string, status: "unhealthy") => + Effect.tryPromise({ + try: () => updateProjectObservedStatus(projectId, status), + catch: (error) => + new Error(`Failed to update observed project status: ${error}`), }); const setupGlobalPnpmVolume = () => @@ -75,14 +84,14 @@ export const handleDockerComposeUp = ( return; } - if (project.status === "deleting") { + if (project.desiredStatus === "deleting") { yield* logInfo("Skipping docker.composeUp for deleting project", { projectId: project.id, }); return; } - yield* setProjectStatus(project.id, "starting"); + yield* setProjectDesiredStatus(project.id); yield* ctx.throwIfCancelRequested(); @@ -100,7 +109,7 @@ export const handleDockerComposeUp = ( result.stderr || result.stdout || "Unknown docker error"; const diagnostic = classifyComposeFailure(errorDetails); const errorMsg = `compose up failed: ${diagnostic.summary} (full logs: ${dockerLogPath})`; - yield* setProjectStatus(project.id, "error"); + yield* setProjectObservedStatus(project.id, "unhealthy"); yield* Effect.fail(new Error(errorMsg)); return; } diff --git a/src/server/queue/handlers/dockerEnsureRunning.ts b/src/server/queue/handlers/dockerEnsureRunning.ts index ba692cc0..3c8b7042 100644 --- a/src/server/queue/handlers/dockerEnsureRunning.ts +++ b/src/server/queue/handlers/dockerEnsureRunning.ts @@ -13,7 +13,8 @@ import { checkPreviewReady } from "@/server/projects/health"; import { getProjectPreviewPath } from "@/server/projects/paths"; import { getProjectByIdIncludeDeleted, - updateProjectStatus, + updateProjectDesiredStatus, + updateProjectObservedStatus, } from "@/server/projects/projects.model"; import { enqueueDockerWaitReady } from "../enqueue"; import { parsePayload } from "../types"; @@ -51,16 +52,16 @@ export function handleDockerEnsureRunning( return; } - if (project.status === "deleting") { + if (project.desiredStatus === "deleting") { return; } yield* Effect.tryPromise({ - try: () => updateProjectStatus(project.id, "starting"), + try: () => updateProjectDesiredStatus(project.id, "running"), catch: (error) => new ProjectError({ projectId: project.id, - operation: "updateProjectStatus", + operation: "updateProjectDesiredStatus", message: error instanceof Error ? error.message : String(error), cause: error, }), @@ -122,11 +123,11 @@ export function handleDockerEnsureRunning( if (!result.success) { yield* Effect.tryPromise({ - try: () => updateProjectStatus(project.id, "error"), + try: () => updateProjectObservedStatus(project.id, "unhealthy"), catch: (error) => new ProjectError({ projectId: project.id, - operation: "updateProjectStatus", + operation: "updateProjectObservedStatus", message: error instanceof Error ? error.message : String(error), cause: error, }), diff --git a/src/server/queue/handlers/dockerStop.ts b/src/server/queue/handlers/dockerStop.ts index 4b605637..b05ce4df 100644 --- a/src/server/queue/handlers/dockerStop.ts +++ b/src/server/queue/handlers/dockerStop.ts @@ -3,7 +3,8 @@ import type { LegacyHandler } from "@/server/effect/handler-adapter"; import { getProjectPreviewPath } from "@/server/projects/paths"; import { getProjectByIdIncludeDeleted, - updateProjectStatus, + updateProjectDesiredStatus, + updateProjectObservedStatus, } from "@/server/projects/projects.model"; import { parsePayload } from "../types"; @@ -15,11 +16,11 @@ export const handleDockerStop: LegacyHandler = async (ctx) => { return; } - if (project.status === "deleting") { + if (project.desiredStatus === "deleting") { return; } - await updateProjectStatus(project.id, "stopping"); + await updateProjectDesiredStatus(project.id, "stopped"); await ctx.throwIfCancelRequested(); @@ -27,9 +28,9 @@ export const handleDockerStop: LegacyHandler = async (ctx) => { const result = await composeStop(project.id, previewPath); if (result.success) { - await updateProjectStatus(project.id, "stopped"); + await updateProjectObservedStatus(project.id, "missing"); } else { - await updateProjectStatus(project.id, "error"); + await updateProjectObservedStatus(project.id, "unhealthy"); throw new Error(`compose stop failed: ${result.stderr.slice(0, 500)}`); } }; diff --git a/src/server/queue/handlers/dockerWaitReady.ts b/src/server/queue/handlers/dockerWaitReady.ts index 7b19bb63..fac03425 100644 --- a/src/server/queue/handlers/dockerWaitReady.ts +++ b/src/server/queue/handlers/dockerWaitReady.ts @@ -10,7 +10,7 @@ import { logger } from "@/server/logger"; import { getProjectPath } from "@/server/projects/paths"; import { getProjectByIdIncludeDeleted, - updateProjectStatus, + updateProjectObservedStatus, } from "@/server/projects/projects.model"; import { enqueueOpencodeSessionCreate } from "../enqueue"; import { parsePayload } from "../types"; @@ -42,7 +42,7 @@ const shouldSkipDeletedProject = ( project: Project, ): Effect.Effect => Effect.sync(() => { - if (project.status === "deleting") { + if (project.desiredStatus === "deleting") { logger.info( { projectId: project.id }, "Skipping docker.waitReady for deleting project", @@ -68,11 +68,11 @@ const handleServicesReady = ( ): Effect.Effect => Effect.gen(function* () { yield* Effect.tryPromise({ - try: () => updateProjectStatus(project.id, "running"), + try: () => updateProjectObservedStatus(project.id, "healthy"), catch: (error) => { logger.error( { projectId: project.id, error }, - "Failed to update project status to running", + "Failed to record healthy project observation", ); }, }).pipe(Effect.orElse(() => Effect.void)); @@ -160,7 +160,7 @@ export const handleDockerWaitReady = ( errorMsg, ); yield* Effect.tryPromise({ - try: () => updateProjectStatus(project.id, "error"), + try: () => updateProjectObservedStatus(project.id, "unhealthy"), catch: () => {}, }).pipe(Effect.orElse(() => Effect.void)); return; diff --git a/src/server/queue/handlers/opencodeSendInitialPrompt.ts b/src/server/queue/handlers/opencodeSendInitialPrompt.ts index eca0db7d..f200feae 100644 --- a/src/server/queue/handlers/opencodeSendInitialPrompt.ts +++ b/src/server/queue/handlers/opencodeSendInitialPrompt.ts @@ -38,7 +38,7 @@ export function handleOpencodeSendInitialPrompt( return; } - if (project.status === "deleting") { + if (project.desiredStatus === "deleting") { logger.info( { projectId: project.id }, "Skipping opencode.sendInitialPrompt for deleting project", diff --git a/src/server/queue/handlers/opencodeSendUserPrompt.ts b/src/server/queue/handlers/opencodeSendUserPrompt.ts index a79bcd7c..d70359a7 100644 --- a/src/server/queue/handlers/opencodeSendUserPrompt.ts +++ b/src/server/queue/handlers/opencodeSendUserPrompt.ts @@ -90,7 +90,7 @@ export function handleOpencodeSendUserPrompt( return; } - if (project.status === "deleting") { + if (project.desiredStatus === "deleting") { logger.info( { projectId: project.id }, "Skipping opencode.sendUserPrompt for deleting project", diff --git a/src/server/queue/handlers/opencodeSessionCreate.ts b/src/server/queue/handlers/opencodeSessionCreate.ts index 8eeb1f37..46e3c28a 100644 --- a/src/server/queue/handlers/opencodeSessionCreate.ts +++ b/src/server/queue/handlers/opencodeSessionCreate.ts @@ -41,7 +41,7 @@ export function handleOpencodeSessionCreate( return; } - if (project.status === "deleting") { + if (project.desiredStatus === "deleting") { logger.info( { projectId: project.id }, "Skipping opencode.sessionCreate for deleting project", diff --git a/src/server/queue/handlers/productionBuild.ts b/src/server/queue/handlers/productionBuild.ts index 057d9216..e7c2a959 100644 --- a/src/server/queue/handlers/productionBuild.ts +++ b/src/server/queue/handlers/productionBuild.ts @@ -42,7 +42,7 @@ export function handleProductionBuild( return undefined as void; } - if (project.status === "deleting") { + if (project.desiredStatus === "deleting") { logger.info( { projectId: project.id }, "Skipping production.build for deleting project", diff --git a/src/server/queue/handlers/productionStart.ts b/src/server/queue/handlers/productionStart.ts index 79ee5333..8cb1eff2 100644 --- a/src/server/queue/handlers/productionStart.ts +++ b/src/server/queue/handlers/productionStart.ts @@ -43,7 +43,7 @@ export function handleProductionStart( return; } - if (project.status === "deleting") { + if (project.desiredStatus === "deleting") { logger.info( { projectId: project.id }, "Skipping production.start for deleting project", diff --git a/src/server/queue/handlers/productionStop.ts b/src/server/queue/handlers/productionStop.ts index 5cca7aec..070f6547 100644 --- a/src/server/queue/handlers/productionStop.ts +++ b/src/server/queue/handlers/productionStop.ts @@ -66,7 +66,7 @@ export function handleProductionStop( yield* removeDockerImages(project.id); yield* removeProductionArtifacts(project.id); - if (project.status !== "deleting") { + if (project.desiredStatus !== "deleting") { yield* Effect.tryPromise({ try: () => updateProductionStatus(project.id, "stopped", { diff --git a/src/server/queue/handlers/productionWaitReady.ts b/src/server/queue/handlers/productionWaitReady.ts index 04c461d4..2bedb0e5 100644 --- a/src/server/queue/handlers/productionWaitReady.ts +++ b/src/server/queue/handlers/productionWaitReady.ts @@ -51,7 +51,7 @@ export function handleProductionWaitReady( return; } - if (project.status === "deleting") { + if (project.desiredStatus === "deleting") { logger.info( { projectId: project.id }, "Skipping production.waitReady for deleting project", diff --git a/src/server/queue/handlers/projectCreate.ts b/src/server/queue/handlers/projectCreate.ts index 89ff8196..7efca220 100644 --- a/src/server/queue/handlers/projectCreate.ts +++ b/src/server/queue/handlers/projectCreate.ts @@ -122,7 +122,8 @@ const createProjectEffect = (params: { description: params.description, devPort: params.devPort, productionPort: params.productionPort, - status: "created", + desiredStatus: "running", + observedStatus: "unknown", pathOnDisk: params.projectPath, }), catch: (error) => toProjectError(error, "createProject", params.id), diff --git a/src/server/queue/handlers/projectDelete.ts b/src/server/queue/handlers/projectDelete.ts index c095170f..26da909b 100644 --- a/src/server/queue/handlers/projectDelete.ts +++ b/src/server/queue/handlers/projectDelete.ts @@ -11,7 +11,7 @@ import { import { getProjectByIdIncludeDeleted, hardDeleteProject, - updateProjectStatus, + updateProjectDesiredStatus, } from "@/server/projects/projects.model"; import { spawnCommand } from "@/server/utils/execAsync"; import { parsePayload } from "../types"; @@ -51,7 +51,7 @@ export const handleProjectDelete: LegacyHandler = async (ctx) => { try { // Step 1: Mark status as "deleting" (best-effort) try { - await updateProjectStatus(project.id, "deleting"); + await updateProjectDesiredStatus(project.id, "deleting"); logger.debug({ projectId: project.id }, "Project marked as deleting"); } catch (error) { logger.warn( diff --git a/src/server/queue/handlers/projectDescriptionSync.ts b/src/server/queue/handlers/projectDescriptionSync.ts index caef0e68..8e0ea124 100644 --- a/src/server/queue/handlers/projectDescriptionSync.ts +++ b/src/server/queue/handlers/projectDescriptionSync.ts @@ -36,7 +36,7 @@ export function handleProjectDescriptionSync( }), }); - if (!project || project.status === "deleting") return; + if (!project || project.desiredStatus === "deleting") return; const sessionId = project.bootstrapSessionId; if (!sessionId) return; diff --git a/src/server/queue/helpers/getProjectOrSkip.ts b/src/server/queue/helpers/getProjectOrSkip.ts index c824db5c..8e082a92 100644 --- a/src/server/queue/helpers/getProjectOrSkip.ts +++ b/src/server/queue/helpers/getProjectOrSkip.ts @@ -19,7 +19,7 @@ export async function getProjectOrSkip( return null; } - if (project.status === "deleting") { + if (project.desiredStatus === "deleting") { logger.info( { projectId: project.id }, `Skipping ${handlerName} for deleting project`, diff --git a/src/server/reconciliation/project.reconcile.ts b/src/server/reconciliation/project.reconcile.ts index efbf4a08..52fa1bff 100644 --- a/src/server/reconciliation/project.reconcile.ts +++ b/src/server/reconciliation/project.reconcile.ts @@ -151,10 +151,12 @@ async function checkProjectHealthSync(project: Project): Promise<{ break; } - // Check for stuck starting + // A running project with no observation yet is still starting. Once an + // observation is unhealthy/crashed it is reported by the desired/observed + // mismatch above instead of relying on the removed transitional status. if ( project.desiredStatus === "running" && - project.status === "starting" && + project.observedStatus === "unknown" && project.createdAt ) { const age = now.getTime() - project.createdAt.getTime(); diff --git a/src/server/reconciliation/reconcile.ts b/src/server/reconciliation/reconcile.ts index 731d8cb3..93ffa4be 100644 --- a/src/server/reconciliation/reconcile.ts +++ b/src/server/reconciliation/reconcile.ts @@ -73,6 +73,7 @@ export async function runReconciliation(): Promise { violationsFound: allViolations.length, actionsApplied: allActions.length, durationMs: completedAt.getTime() - startedAt.getTime(), + projectCounts: metrics.projectCounts, }, "Reconciliation completed", ); @@ -125,6 +126,10 @@ interface HealthMetrics { projectsTotal: number; projectsRunning: number; projectsError: number; + projectCounts: { + desired: Record; + observed: Record; + }; opencodeHealthy: boolean; } @@ -145,18 +150,30 @@ async function gatherHealthMetrics( queueStats.map((s) => [s.state, s.count]), ); - // Count projects by status - const projectStats = await db + // Desired and observed counts answer different operational questions and must + // not be collapsed back into the retired status field. + const desiredProjectStats = await db .select({ - status: projects.status, + status: projects.desiredStatus, count: count(), }) .from(projects) .where(isNull(projects.deletedAt)) - .groupBy(projects.status); + .groupBy(projects.desiredStatus); + const observedProjectStats = await db + .select({ + status: projects.observedStatus, + count: count(), + }) + .from(projects) + .where(isNull(projects.deletedAt)) + .groupBy(projects.observedStatus); - const projectMap = Object.fromEntries( - projectStats.map((s) => [s.status, s.count]), + const desiredProjectMap = Object.fromEntries( + desiredProjectStats.map((s) => [s.status, s.count]), + ); + const observedProjectMap = Object.fromEntries( + observedProjectStats.map((s) => [s.status, s.count]), ); // Check OpenCode health @@ -166,9 +183,14 @@ async function gatherHealthMetrics( queueJobsQueued: queueMap.queued || 0, queueJobsRunning: queueMap.running || 0, queueJobsFailed: queueMap.failed || 0, - projectsTotal: Object.values(projectMap).reduce((a, b) => a + b, 0), - projectsRunning: projectMap.running || 0, - projectsError: projectMap.error || 0, + projectsTotal: Object.values(desiredProjectMap).reduce((a, b) => a + b, 0), + projectsRunning: desiredProjectMap.running || 0, + projectsError: + (observedProjectMap.unhealthy || 0) + (observedProjectMap.crashed || 0), + projectCounts: { + desired: desiredProjectMap, + observed: observedProjectMap, + }, opencodeHealthy, }; }