From bdabe189bb23c474108866be5e863ad64295fef8 Mon Sep 17 00:00:00 2001 From: Y1fe1Zh0u Date: Fri, 24 Jul 2026 10:51:38 +0800 Subject: [PATCH 1/2] Keep fresh deployments compatible with logical deletion The initial revision builds the current ORM schema on empty databases, so the logical-deletion revision must tolerate its columns and indexes already existing. Inspect each object independently and add a migration-only empty-database CI stage so this path fails before full application startup. Constraint: The dynamic initial_schema baseline remains unchanged in this hotfix. Rejected: Rewrite the historical baseline here | requires a full migration-chain schema audit and would broaden an urgent deployment fix. Confidence: high Scope-risk: narrow Reversibility: clean Directive: Keep new schema revisions compatible with fresh metadata creation until initial_schema is replaced by a static baseline. Tested: 12 scoped pytest tests; Ruff on changed Python; shell syntax; Compose config; YAML parsing; git diff check. Not-tested: Local PostgreSQL container run because the Docker daemon is unavailable; the new Drone stage provides this verification. --- .github/drone.yml | 26 ++- .github/scripts/ci_migration_test.sh | 72 +++++++ ...202607221500_add_agent_model_deleted_at.py | 89 ++++++--- .../test_agent_model_deleted_at_migration.py | 188 ++++++++++++++++++ 4 files changed, 343 insertions(+), 32 deletions(-) create mode 100644 .github/scripts/ci_migration_test.sh create mode 100644 backend/tests/test_agent_model_deleted_at_migration.py diff --git a/.github/drone.yml b/.github/drone.yml index a152487bb..4d0fd8ddf 100644 --- a/.github/drone.yml +++ b/.github/drone.yml @@ -5,9 +5,10 @@ # 流程: # 1. 代码克隆 (获取完整历史和 tags) # 2. 构建 Docker 镜像 -# 3. 本地部署测试 (直接利用本地 docker socket 和 docker-compose.ci.yml) -# 4. 本地升级测试 (测试旧版到新版的迁移) -# 5. 打包传输到服务器 (可选) +# 3. 空数据库迁移测试 +# 4. 本地部署测试 (直接利用本地 docker socket 和 docker-compose.ci.yml) +# 5. 本地升级测试 (测试旧版到新版的迁移) +# 6. 打包传输到服务器 (可选) # ============================================================ kind: pipeline type: docker @@ -150,7 +151,20 @@ steps: echo "镜像构建完成 source=$PREVIOUS_RELEASE_TAG target=$DRONE_COMMIT" # -------------------------------------------------------- - # Step 3: 本地部署测试 (Deploy Test) + # Step 3: 空数据库迁移测试 (Migration Test) + # -------------------------------------------------------- + - name: fresh-database-migration-test + image: docker:24.0.6 + privileged: true + volumes: + - name: docker-socket + path: /var/run/docker.sock + commands: + - chmod +x .github/scripts/ci_migration_test.sh + - .github/scripts/ci_migration_test.sh + + # -------------------------------------------------------- + # Step 4: 本地部署测试 (Deploy Test) # -------------------------------------------------------- - name: local-deploy-test image: docker:24.0.6 @@ -163,7 +177,7 @@ steps: - .github/scripts/ci_deploy_test.sh # -------------------------------------------------------- - # Step 4: 本地升级测试 (Upgrade Test) + # Step 5: 本地升级测试 (Upgrade Test) # -------------------------------------------------------- - name: local-upgrade-test image: docker:24.0.6 @@ -176,7 +190,7 @@ steps: - .github/scripts/ci_upgrade_test.sh # -------------------------------------------------------- - # Step 5: (可选) 导出镜像并传输到私有服务器 + # Step 6: (可选) 导出镜像并传输到私有服务器 # 只有在部署测试通过后才会执行到这一步 # -------------------------------------------------------- - name: save-images diff --git a/.github/scripts/ci_migration_test.sh b/.github/scripts/ci_migration_test.sh new file mode 100644 index 000000000..e8286f744 --- /dev/null +++ b/.github/scripts/ci_migration_test.sh @@ -0,0 +1,72 @@ +#!/bin/sh +set -eu + +COMMIT_SHORT="$(printf '%.8s' "$DRONE_COMMIT")" +PROJECT="clawith-ci-$DRONE_BUILD_NUMBER-migration" +NETWORK="$PROJECT-network" +export COMPOSE_PROJECT_NAME="$PROJECT" +export CLAWITH_DOCKER_NETWORK="$NETWORK" +export IMAGE_TAG="ci-$DRONE_BUILD_NUMBER-$COMMIT_SHORT" + +compose() { + docker compose -p "$PROJECT" -f docker-compose.ci.yml "$@" +} + +cleanup() { + STATUS=$? + trap - EXIT + if [ "$STATUS" -ne 0 ]; then + echo "空数据库迁移测试失败，保留诊断输出" + compose logs --no-color --tail=300 postgres 2>/dev/null || true + fi + compose down -v --remove-orphans >/dev/null 2>&1 || true + exit "$STATUS" +} + +wait_healthy() { + CONTAINER_ID="$1" + ATTEMPT=0 + while [ "$(docker inspect --format '{{.State.Health.Status}}' "$CONTAINER_ID" 2>/dev/null || true)" != "healthy" ]; do + ATTEMPT=$((ATTEMPT + 1)) + if [ "$ATTEMPT" -ge 60 ]; then + return 1 + fi + sleep 2 + done +} + +trap cleanup EXIT + +compose down -v --remove-orphans >/dev/null 2>&1 || true +compose up -d postgres +wait_healthy "$(compose ps -q postgres)" + +echo "从空 PostgreSQL 数据库执行 alembic upgrade head" +compose run --rm --no-deps --entrypoint /bin/bash backend \ + -lc 'alembic upgrade head && alembic current --check-heads' + +DELETED_AT_COLUMN_COUNT="$( + compose exec -T postgres psql -U clawith -d clawith -Atc " + SELECT COUNT(*) + FROM information_schema.columns + WHERE table_schema = 'public' + AND table_name IN ('agents', 'llm_models') + AND column_name = 'deleted_at'; + " | tr -d '\r' +)" +[ "$DELETED_AT_COLUMN_COUNT" = "2" ] + +ACTIVE_INDEX_COUNT="$( + compose exec -T postgres psql -U clawith -d clawith -Atc " + SELECT COUNT(*) + FROM pg_indexes + WHERE schemaname = 'public' + AND indexname IN ( + 'ix_agents_active_tenant_created_at', + 'ix_llm_models_active_tenant_created_at' + ); + " | tr -d '\r' +)" +[ "$ACTIVE_INDEX_COUNT" = "2" ] + +echo "空数据库迁移测试通过 columns=$DELETED_AT_COLUMN_COUNT indexes=$ACTIVE_INDEX_COUNT" diff --git a/backend/alembic/versions/202607221500_add_agent_model_deleted_at.py b/backend/alembic/versions/202607221500_add_agent_model_deleted_at.py index 407c10944..dcbd704fd 100644 --- a/backend/alembic/versions/202607221500_add_agent_model_deleted_at.py +++ b/backend/alembic/versions/202607221500_add_agent_model_deleted_at.py @@ -19,33 +19,70 @@ depends_on: str | Sequence[str] | None = None +AGENTS_TABLE = "agents" +LLM_MODELS_TABLE = "llm_models" +DELETED_AT_COLUMN = "deleted_at" +AGENTS_ACTIVE_INDEX = "ix_agents_active_tenant_created_at" +LLM_MODELS_ACTIVE_INDEX = "ix_llm_models_active_tenant_created_at" + + +def _inspector(): + return sa.inspect(op.get_bind()) + + +def _column_exists(table_name: str, column_name: str) -> bool: + return column_name in { + column["name"] for column in _inspector().get_columns(table_name) + } + + +def _index_exists(table_name: str, index_name: str) -> bool: + return index_name in { + index["name"] for index in _inspector().get_indexes(table_name) + } + + def upgrade() -> None: - op.add_column( - "agents", - sa.Column("deleted_at", sa.DateTime(timezone=True), nullable=True), - ) - op.add_column( - "llm_models", - sa.Column("deleted_at", sa.DateTime(timezone=True), nullable=True), - ) - op.create_index( - "ix_agents_active_tenant_created_at", - "agents", - ["tenant_id", "created_at"], - unique=False, - postgresql_where=sa.text("deleted_at IS NULL"), - ) - op.create_index( - "ix_llm_models_active_tenant_created_at", - "llm_models", - ["tenant_id", "created_at"], - unique=False, - postgresql_where=sa.text("deleted_at IS NULL"), - ) + if not _column_exists(AGENTS_TABLE, DELETED_AT_COLUMN): + op.add_column( + AGENTS_TABLE, + sa.Column(DELETED_AT_COLUMN, sa.DateTime(timezone=True), nullable=True), + ) + + if not _column_exists(LLM_MODELS_TABLE, DELETED_AT_COLUMN): + op.add_column( + LLM_MODELS_TABLE, + sa.Column(DELETED_AT_COLUMN, sa.DateTime(timezone=True), nullable=True), + ) + + if not _index_exists(AGENTS_TABLE, AGENTS_ACTIVE_INDEX): + op.create_index( + AGENTS_ACTIVE_INDEX, + AGENTS_TABLE, + ["tenant_id", "created_at"], + unique=False, + postgresql_where=sa.text("deleted_at IS NULL"), + ) + + if not _index_exists(LLM_MODELS_TABLE, LLM_MODELS_ACTIVE_INDEX): + op.create_index( + LLM_MODELS_ACTIVE_INDEX, + LLM_MODELS_TABLE, + ["tenant_id", "created_at"], + unique=False, + postgresql_where=sa.text("deleted_at IS NULL"), + ) def downgrade() -> None: - op.drop_index("ix_llm_models_active_tenant_created_at", table_name="llm_models") - op.drop_index("ix_agents_active_tenant_created_at", table_name="agents") - op.drop_column("llm_models", "deleted_at") - op.drop_column("agents", "deleted_at") + if _index_exists(LLM_MODELS_TABLE, LLM_MODELS_ACTIVE_INDEX): + op.drop_index(LLM_MODELS_ACTIVE_INDEX, table_name=LLM_MODELS_TABLE) + + if _index_exists(AGENTS_TABLE, AGENTS_ACTIVE_INDEX): + op.drop_index(AGENTS_ACTIVE_INDEX, table_name=AGENTS_TABLE) + + if _column_exists(LLM_MODELS_TABLE, DELETED_AT_COLUMN): + op.drop_column(LLM_MODELS_TABLE, DELETED_AT_COLUMN) + + if _column_exists(AGENTS_TABLE, DELETED_AT_COLUMN): + op.drop_column(AGENTS_TABLE, DELETED_AT_COLUMN) diff --git a/backend/tests/test_agent_model_deleted_at_migration.py b/backend/tests/test_agent_model_deleted_at_migration.py new file mode 100644 index 000000000..f263ced70 --- /dev/null +++ b/backend/tests/test_agent_model_deleted_at_migration.py @@ -0,0 +1,188 @@ +"""Deployment contract for Agent and LLM model logical deletion schema.""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path + + +MIGRATION_PATH = ( + Path(__file__).resolve().parents[1] + / "alembic" + / "versions" + / "202607221500_add_agent_model_deleted_at.py" +) + + +def _load_migration(): + spec = importlib.util.spec_from_file_location( + "agent_model_deleted_at_migration", + MIGRATION_PATH, + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class FakeInspector: + def __init__( + self, + *, + columns: dict[str, set[str]], + indexes: dict[str, set[str]], + ): + self.columns = columns + self.indexes = indexes + + def get_columns(self, table_name): + return [{"name": name} for name in self.columns.get(table_name, set())] + + def get_indexes(self, table_name): + return [{"name": name} for name in self.indexes.get(table_name, set())] + + +def _install_inspector(monkeypatch, migration, *, columns, indexes): + inspector = FakeInspector(columns=columns, indexes=indexes) + monkeypatch.setattr(migration, "_inspector", lambda: inspector) + + +def test_revision_follows_experience_revision_head() -> None: + migration = _load_migration() + + assert migration.revision == "add_agent_model_deleted_at" + assert migration.down_revision == "add_experience_revision_drafts" + + +def test_upgrade_adds_all_missing_columns_and_indexes(monkeypatch) -> None: + migration = _load_migration() + calls = [] + _install_inspector( + monkeypatch, + migration, + columns={"agents": {"id"}, "llm_models": {"id"}}, + indexes={"agents": set(), "llm_models": set()}, + ) + monkeypatch.setattr( + migration.op, + "add_column", + lambda *args, **kwargs: calls.append(("add_column", args, kwargs)), + ) + monkeypatch.setattr( + migration.op, + "create_index", + lambda *args, **kwargs: calls.append(("create_index", args, kwargs)), + ) + + migration.upgrade() + + assert [(kind, args[0]) for kind, args, _ in calls] == [ + ("add_column", "agents"), + ("add_column", "llm_models"), + ("create_index", "ix_agents_active_tenant_created_at"), + ("create_index", "ix_llm_models_active_tenant_created_at"), + ] + assert calls[2][1][1:3] == ("agents", ["tenant_id", "created_at"]) + assert calls[3][1][1:3] == ("llm_models", ["tenant_id", "created_at"]) + + +def test_upgrade_is_noop_when_fresh_metadata_already_created_schema( + monkeypatch, +) -> None: + migration = _load_migration() + _install_inspector( + monkeypatch, + migration, + columns={ + "agents": {"id", "deleted_at"}, + "llm_models": {"id", "deleted_at"}, + }, + indexes={ + "agents": {"ix_agents_active_tenant_created_at"}, + "llm_models": {"ix_llm_models_active_tenant_created_at"}, + }, + ) + monkeypatch.setattr( + migration.op, + "add_column", + lambda *args, **kwargs: (_ for _ in ()).throw( + AssertionError("unexpected add_column") + ), + ) + monkeypatch.setattr( + migration.op, + "create_index", + lambda *args, **kwargs: (_ for _ in ()).throw( + AssertionError("unexpected create_index") + ), + ) + + migration.upgrade() + + +def test_upgrade_repairs_partial_schema_independently(monkeypatch) -> None: + migration = _load_migration() + calls = [] + _install_inspector( + monkeypatch, + migration, + columns={ + "agents": {"id", "deleted_at"}, + "llm_models": {"id"}, + }, + indexes={ + "agents": set(), + "llm_models": {"ix_llm_models_active_tenant_created_at"}, + }, + ) + monkeypatch.setattr( + migration.op, + "add_column", + lambda *args, **kwargs: calls.append(("add_column", args, kwargs)), + ) + monkeypatch.setattr( + migration.op, + "create_index", + lambda *args, **kwargs: calls.append(("create_index", args, kwargs)), + ) + + migration.upgrade() + + assert [(kind, args[0]) for kind, args, _ in calls] == [ + ("add_column", "llm_models"), + ("create_index", "ix_agents_active_tenant_created_at"), + ] + + +def test_downgrade_only_drops_existing_objects(monkeypatch) -> None: + migration = _load_migration() + calls = [] + _install_inspector( + monkeypatch, + migration, + columns={ + "agents": {"id", "deleted_at"}, + "llm_models": {"id"}, + }, + indexes={ + "agents": {"ix_agents_active_tenant_created_at"}, + "llm_models": set(), + }, + ) + monkeypatch.setattr( + migration.op, + "drop_index", + lambda *args, **kwargs: calls.append(("drop_index", args, kwargs)), + ) + monkeypatch.setattr( + migration.op, + "drop_column", + lambda *args, **kwargs: calls.append(("drop_column", args, kwargs)), + ) + + migration.downgrade() + + assert [(kind, args[0]) for kind, args, _ in calls] == [ + ("drop_index", "ix_agents_active_tenant_created_at"), + ("drop_column", "agents"), + ] From c2a3f1ff36ee04d40ca5149bad95976dff52f443 Mon Sep 17 00:00:00 2001 From: Y1fe1Zh0u Date: Fri, 24 Jul 2026 11:03:16 +0800 Subject: [PATCH 2/2] Keep upgrade validation usable across the broken release The upgrade job selects v1.11.2 as its source, but that released image cannot bootstrap an empty database because it contains the duplicate-column migration. Build its schema through the parent revision, apply the corrected head with the target image, then continue the existing old-app and target-app upgrade assertions. Constraint: v1.11.2 is the latest release tag and is itself unable to initialize the CI source database. Rejected: Skip the upgrade job or ignore its failure | would leave the PR red and remove existing-database coverage. Confidence: high Scope-risk: narrow Reversibility: clean Directive: Remove the v1.11.2 bootstrap compatibility path after it is no longer a supported upgrade source. Tested: POSIX shell syntax; 12 scoped pytest tests; git diff check. Not-tested: Local Docker execution because the Docker daemon is unavailable; Drone exercises this exact path. --- .github/scripts/ci_upgrade_test.sh | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/.github/scripts/ci_upgrade_test.sh b/.github/scripts/ci_upgrade_test.sh index 945fb3a09..dd1ed9105 100644 --- a/.github/scripts/ci_upgrade_test.sh +++ b/.github/scripts/ci_upgrade_test.sh @@ -46,6 +46,19 @@ wait_healthy() { done } +run_schema_command() { + IMAGE="$1" + COMMAND="$2" + docker run --rm \ + --network "$NETWORK" \ + --entrypoint /bin/bash \ + -e DATABASE_URL=postgresql+asyncpg://clawith:clawith@postgres:5432/clawith \ + -e REDIS_URL=redis://redis:6379/0 \ + -e SECRET_KEY=ci-test-secret \ + -e JWT_SECRET_KEY=ci-test-jwt-secret \ + "$IMAGE" -lc "$COMMAND" +} + trap cleanup EXIT compose down -v --remove-orphans >/dev/null 2>&1 || true @@ -63,6 +76,15 @@ docker image inspect "$OLD_IMAGE" --format '{{ index .Config.Labels "org.opencon OLD_VERSION=$(cat /tmp/old_version | tr -d '\r') echo "启动升级源 version=$OLD_VERSION revision=$OLD_REVISION" +if [ "$OLD_VERSION" = "v1.11.2" ]; then + echo "v1.11.2 无法从空库执行最终 migration，先建立其父 revision" + run_schema_command "$OLD_IMAGE" \ + "alembic upgrade add_experience_revision_drafts" + echo "使用目标镜像执行修复后的最终 migration" + run_schema_command "$NEW_IMAGE" \ + "alembic upgrade head && python -m app.scripts.setup_langgraph_checkpoints" +fi + docker run -d \ --name "$OLD_CONTAINER" \ --network "$NETWORK" \