diff --git a/api/ee/src/apis/fastapi/organizations/models.py b/api/ee/src/apis/fastapi/organizations/models.py index 51c2f06c39..037c57725d 100644 --- a/api/ee/src/apis/fastapi/organizations/models.py +++ b/api/ee/src/apis/fastapi/organizations/models.py @@ -1,5 +1,6 @@ from typing import Optional from datetime import datetime +from uuid import UUID from pydantic import BaseModel, Field @@ -25,7 +26,7 @@ class OrganizationDomainVerify(BaseModel): class OrganizationDomainResponse(BaseModel): """Response model for a domain.""" - id: str + id: UUID slug: str name: Optional[str] @@ -38,7 +39,7 @@ class OrganizationDomainResponse(BaseModel): created_at: datetime updated_at: Optional[datetime] - organization_id: str + organization_id: UUID class Config: from_attributes = True @@ -76,7 +77,7 @@ class OrganizationProviderUpdate(BaseModel): class OrganizationProviderResponse(BaseModel): """Response model for an SSO provider.""" - id: str + id: UUID slug: str name: Optional[str] @@ -89,7 +90,7 @@ class OrganizationProviderResponse(BaseModel): created_at: datetime updated_at: Optional[datetime] - organization_id: str + organization_id: UUID class Config: from_attributes = True diff --git a/api/ee/tests/pytest/unit/test_organization_fastapi_models.py b/api/ee/tests/pytest/unit/test_organization_fastapi_models.py new file mode 100644 index 0000000000..75948e478e --- /dev/null +++ b/api/ee/tests/pytest/unit/test_organization_fastapi_models.py @@ -0,0 +1,48 @@ +from datetime import datetime, timezone +from uuid import uuid4 + +from ee.src.apis.fastapi.organizations.models import ( + OrganizationDomainResponse, + OrganizationProviderResponse, +) +from ee.src.core.organizations.types import OrganizationDomain, OrganizationProvider + + +def test_domain_response_accepts_uuid_backed_domain_dto(): + domain = OrganizationDomain( + id=uuid4(), + organization_id=uuid4(), + slug="example.com", + name="Example", + description="Example domain", + token="verify-me", + flags={"is_verified": False}, + created_at=datetime.now(timezone.utc), + updated_at=None, + ) + + response = OrganizationDomainResponse.model_validate(domain) + + dumped = response.model_dump(mode="json") + assert isinstance(dumped["id"], str) + assert isinstance(dumped["organization_id"], str) + + +def test_provider_response_accepts_uuid_backed_provider_dto(): + provider = OrganizationProvider( + id=uuid4(), + organization_id=uuid4(), + slug="oidc", + name="OIDC", + description="OIDC provider", + settings={"issuer_url": "https://issuer.example.com"}, + flags={"is_active": True, "is_valid": True}, + created_at=datetime.now(timezone.utc), + updated_at=None, + ) + + response = OrganizationProviderResponse.model_validate(provider) + + dumped = response.model_dump(mode="json") + assert isinstance(dumped["id"], str) + assert isinstance(dumped["organization_id"], str) diff --git a/api/oss/src/core/secrets/dtos.py b/api/oss/src/core/secrets/dtos.py index 0a6b60d344..992596075f 100644 --- a/api/oss/src/core/secrets/dtos.py +++ b/api/oss/src/core/secrets/dtos.py @@ -84,31 +84,38 @@ def validate_secret_data_based_on_kind(cls, values: Dict[str, Any]): data = data.model_dump() values["data"] = data + standard_provider_kinds = {provider.value for provider in StandardProviderKind} + custom_provider_kinds = {provider.value for provider in CustomProviderKind} + if kind == SecretKind.PROVIDER_KEY.value: if not isinstance(data, dict): raise ValueError( "The provided request secret dto is not a valid type for StandardProviderDTO" ) - if not isinstance(data["provider"], dict) or "key" not in data["provider"]: + provider = data.get("provider") + if not isinstance(provider, dict) or "key" not in provider: raise ValueError( "The provided request secret dto is missing required fields for StandardProviderSettingsDTO" ) - if data["kind"] not in StandardProviderKind.__members__.values(): + # Accept the legacy provider slug on input, but persist the canonical value. + if data.get("kind") == StandardProviderKind.MISTRALAI.value: + data["kind"] = StandardProviderKind.MISTRAL.value + if data.get("kind") not in standard_provider_kinds: raise ValueError( "The provided kind in data is not a valid StandardProviderKind enum" ) elif kind == SecretKind.CUSTOM_PROVIDER.value: + if not isinstance(data, dict): + raise ValueError( + "The provided request secret dto is not a valid type for CustomProviderDTO" + ) # Fix inconsistent API naming - Users might enter 'togetherai' but the API requires 'together_ai' # This ensures compatibility with LiteLLM which requires the provider in "together_ai" format if data.get("kind", "") == "togetherai": data["kind"] = "together_ai" - if not isinstance(data, dict): - raise ValueError( - "The provided request secret dto is not a valid type for CustomProviderDTO" - ) - if data["kind"] not in CustomProviderKind.__members__.values(): + if data.get("kind") not in custom_provider_kinds: raise ValueError( "The provided kind in data is not a valid CustomProviderKind enum" ) diff --git a/api/oss/src/core/secrets/utils.py b/api/oss/src/core/secrets/utils.py index 9d7e1e838b..d5c7c6ae51 100644 --- a/api/oss/src/core/secrets/utils.py +++ b/api/oss/src/core/secrets/utils.py @@ -7,6 +7,35 @@ from oss.src.models.api.evaluation_model import LMProvidersEnum +_LEGACY_SYSTEM_ENV_NAMES = { + LMProvidersEnum.mistral.value: ("MISTRALAI_API_KEY",), +} + +_PROVIDER_ENV_ALIASES = { + "mistralai": LMProvidersEnum.mistral.value, +} + + +def _get_system_env_secret(secret_name: str) -> str | None: + for env_name in (secret_name, *_LEGACY_SYSTEM_ENV_NAMES.get(secret_name, ())): + env_var = os.getenv(env_name) + if env_var: + return env_var + + return None + + +def _provider_slug_to_env_var(provider_slug: str) -> str: + if not provider_slug: + return "" + + canonical_provider = LMProvidersEnum.__members__.get(provider_slug.replace("_", "")) + if canonical_provider: + return canonical_provider.value + + return _PROVIDER_ENV_ALIASES.get(provider_slug, f"{provider_slug.upper()}_API_KEY") + + async def get_system_llm_providers_secrets() -> Dict[str, Any]: """ Fetches LLM providers secrets from system environment variables. @@ -15,7 +44,7 @@ async def get_system_llm_providers_secrets() -> Dict[str, Any]: secrets = {} for llm_provider in LMProvidersEnum: secret_name = llm_provider.value - env_var = os.getenv(secret_name) + env_var = _get_system_env_secret(secret_name) if env_var: secrets[secret_name] = env_var @@ -46,7 +75,7 @@ async def get_user_llm_providers_secrets(project_id: str) -> Dict[str, Any]: for secret in secrets: kind = secret["data"].get("kind") provider_slug = kind.value if kind else "" - secret_name = f"{provider_slug.upper()}_API_KEY" + secret_name = _provider_slug_to_env_var(provider_slug) if provider_slug: provider = secret["data"].get("provider") readable_secrets[secret_name] = provider.get("key") if provider else None diff --git a/api/oss/src/models/api/evaluation_model.py b/api/oss/src/models/api/evaluation_model.py index f26f109bed..1065cdf15b 100644 --- a/api/oss/src/models/api/evaluation_model.py +++ b/api/oss/src/models/api/evaluation_model.py @@ -153,7 +153,6 @@ class LLMRunRateLimit(BaseModel): class LMProvidersEnum(str, Enum): openai = "OPENAI_API_KEY" mistral = "MISTRAL_API_KEY" - mistralai = "MISTRALAI_API_KEY" cohere = "COHERE_API_KEY" anthropic = "ANTHROPIC_API_KEY" anyscale = "ANYSCALE_API_KEY" diff --git a/api/oss/tests/legacy/conftest.py b/api/oss/tests/legacy/conftest.py index 884ef7a7b1..b46a6cfebc 100644 --- a/api/oss/tests/legacy/conftest.py +++ b/api/oss/tests/legacy/conftest.py @@ -43,7 +43,6 @@ def sample_testset_endpoint_json(): API_KEYS_MAPPING = { "OPENAI_API_KEY": "openai", "MISTRAL_API_KEY": "mistral", - "MISTRALAI_API_KEY": "mistralai", "COHERE_API_KEY": "cohere", "ANTHROPIC_API_KEY": "anthropic", "ANYSCALE_API_KEY": "anyscale", diff --git a/api/oss/tests/pytest/unit/secrets/test_dtos.py b/api/oss/tests/pytest/unit/secrets/test_dtos.py new file mode 100644 index 0000000000..b5ad771009 --- /dev/null +++ b/api/oss/tests/pytest/unit/secrets/test_dtos.py @@ -0,0 +1,60 @@ +import pytest +from pydantic import ValidationError + +from oss.src.core.secrets.dtos import CreateSecretDTO, UpdateSecretDTO + + +def test_create_secret_normalizes_mistralai_standard_provider_payload(): + payload = { + "header": {"name": "Mistral AI", "description": ""}, + "secret": { + "kind": "provider_key", + "data": { + "kind": "mistralai", + "provider": { + "key": "TEST_KEY", + }, + }, + }, + } + + secret = CreateSecretDTO.model_validate(payload) + + assert secret.secret.data.kind == "mistral" + assert secret.secret.data.provider.key == "TEST_KEY" + + +def test_update_secret_normalizes_mistralai_standard_provider_payload(): + payload = { + "secret": { + "kind": "provider_key", + "data": { + "kind": "mistralai", + "provider": { + "key": "TEST_KEY", + }, + }, + }, + } + + secret = UpdateSecretDTO.model_validate(payload) + + assert secret.secret.data.kind == "mistral" + assert secret.secret.data.provider.key == "TEST_KEY" + + +def test_create_secret_rejects_missing_standard_provider_kind(): + payload = { + "header": {"name": "Mistral AI", "description": ""}, + "secret": { + "kind": "provider_key", + "data": { + "provider": { + "key": "TEST_KEY", + }, + }, + }, + } + + with pytest.raises(ValidationError, match="StandardProviderKind"): + CreateSecretDTO.model_validate(payload) diff --git a/api/oss/tests/pytest/unit/secrets/test_utils.py b/api/oss/tests/pytest/unit/secrets/test_utils.py new file mode 100644 index 0000000000..a37c0e692a --- /dev/null +++ b/api/oss/tests/pytest/unit/secrets/test_utils.py @@ -0,0 +1,64 @@ +from types import SimpleNamespace + +import pytest + +from oss.src.core.secrets.enums import StandardProviderKind +from oss.src.core.secrets.utils import ( + get_system_llm_providers_secrets, + get_user_llm_providers_secrets, +) + + +class _FakeVaultService: + def __init__(self, *_args, **_kwargs): + pass + + async def list_secrets(self, project_id): + del project_id + return [ + SimpleNamespace( + kind="provider_key", + model_dump=lambda include=None: { + "data": { + "kind": StandardProviderKind.MISTRALAI, + "provider": {"key": "mistral-key"}, + } + }, + ), + SimpleNamespace( + kind="provider_key", + model_dump=lambda include=None: { + "data": { + "kind": StandardProviderKind.TOGETHERAI, + "provider": {"key": "together-key"}, + } + }, + ), + ] + + +@pytest.mark.asyncio +async def test_get_user_llm_providers_secrets_normalizes_legacy_provider_slugs( + monkeypatch, +): + monkeypatch.setattr("oss.src.core.secrets.utils.VaultService", _FakeVaultService) + + secrets = await get_user_llm_providers_secrets( + "00000000-0000-0000-0000-000000000000" + ) + + assert secrets["MISTRAL_API_KEY"] == "mistral-key" + assert "MISTRALAI_API_KEY" not in secrets + assert secrets["TOGETHERAI_API_KEY"] == "together-key" + assert "TOGETHER_AI_API_KEY" not in secrets + + +@pytest.mark.asyncio +async def test_get_system_llm_providers_secrets_reads_legacy_mistralai_env(monkeypatch): + monkeypatch.delenv("MISTRAL_API_KEY", raising=False) + monkeypatch.setenv("MISTRALAI_API_KEY", "legacy-mistral-key") + + secrets = await get_system_llm_providers_secrets() + + assert secrets["MISTRAL_API_KEY"] == "legacy-mistral-key" + assert "MISTRALAI_API_KEY" not in secrets diff --git a/api/pyproject.toml b/api/pyproject.toml index b61cadadba..c3f243110d 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "api" -version = "0.94.3" +version = "0.94.4" description = "Agenta API" authors = [ { name = "Mahmoud Mabrouk", email = "mahmoud@agenta.ai" }, diff --git a/sdk/agenta/sdk/managers/secrets.py b/sdk/agenta/sdk/managers/secrets.py index 79736ab585..3f97f25651 100644 --- a/sdk/agenta/sdk/managers/secrets.py +++ b/sdk/agenta/sdk/managers/secrets.py @@ -13,7 +13,17 @@ log = get_module_logger(__name__) +_PROVIDER_KIND_ALIASES = { + "mistralai": "mistral", +} + + class SecretsManager: + @staticmethod + def _normalize_provider_kind(provider_kind: str) -> str: + normalized = re.sub(r"[\s-]+", "", provider_kind.lower()) + return _PROVIDER_KIND_ALIASES.get(normalized, normalized) + @staticmethod def get_from_route(scope: str = "all") -> Optional[List[Dict[str, Any]]]: context = RoutingContext.get() @@ -192,9 +202,7 @@ def get_provider_settings(model: str, scope: str = "all") -> Optional[Dict]: # STEP 3: initialize provider settings and simplify provider name provider_settings = dict(model=compatible_provider_model) - request_provider_kind = re.sub( - r"[\s-]+", "", provider.lower() - ) # normalizing other special characters too (azure-openai) + request_provider_kind = SecretsManager._normalize_provider_kind(provider) # STEP 4: get credentials for model for secret in secrets: @@ -204,7 +212,9 @@ def get_provider_settings(model: str, scope: str = "all") -> Optional[Dict]: # i). Extract API key if present # (for standard models -- openai/anthropic/gemini, etc) if secret.get("kind") == "provider_key": - secret_provider_kind = secret_data.get("kind", "") + secret_provider_kind = SecretsManager._normalize_provider_kind( + secret_data.get("kind", "") + ) if request_provider_kind == secret_provider_kind: if "key" in provider_info: @@ -336,9 +346,7 @@ def get_provider_settings_from_workflow( # STEP 3: initialize provider settings and simplify provider name provider_settings = dict(model=compatible_provider_model) - request_provider_kind = re.sub( - r"[\s-]+", "", provider.lower() - ) # normalizing other special characters too (azure-openai) + request_provider_kind = SecretsManager._normalize_provider_kind(provider) # STEP 4: get credentials for model for secret in secrets: @@ -348,7 +356,9 @@ def get_provider_settings_from_workflow( # i). Extract API key if present # (for standard models -- openai/anthropic/gemini, etc) if secret.get("kind") == "provider_key": - secret_provider_kind = secret_data.get("kind", "") + secret_provider_kind = SecretsManager._normalize_provider_kind( + secret_data.get("kind", "") + ) if request_provider_kind == secret_provider_kind: if "key" in provider_info: diff --git a/sdk/agenta/sdk/workflows/runners/daytona.py b/sdk/agenta/sdk/workflows/runners/daytona.py index 8b3aafb18b..adee9fe54e 100644 --- a/sdk/agenta/sdk/workflows/runners/daytona.py +++ b/sdk/agenta/sdk/workflows/runners/daytona.py @@ -128,7 +128,8 @@ def _get_provider_env_vars(self) -> Dict[str, str]: "deepinfra": "DEEPINFRA_API_KEY", "alephalpha": "ALEPHALPHA_API_KEY", "groq": "GROQ_API_KEY", - "mistralai": "MISTRALAI_API_KEY", + "mistral": "MISTRAL_API_KEY", + "mistralai": "MISTRAL_API_KEY", "anthropic": "ANTHROPIC_API_KEY", "perplexityai": "PERPLEXITYAI_API_KEY", # Secret kind is "together_ai" (underscore) even though the env var is TOGETHERAI_API_KEY diff --git a/sdk/oss/tests/legacy/new_tests/conftest.py b/sdk/oss/tests/legacy/new_tests/conftest.py index b6bebae734..cc03ce9ab7 100644 --- a/sdk/oss/tests/legacy/new_tests/conftest.py +++ b/sdk/oss/tests/legacy/new_tests/conftest.py @@ -46,7 +46,6 @@ def sample_testset_endpoint_json(): API_KEYS_MAPPING = { "OPENAI_API_KEY": "openai", "MISTRAL_API_KEY": "mistral", - "MISTRALAI_API_KEY": "mistralai", "COHERE_API_KEY": "cohere", "ANTHROPIC_API_KEY": "anthropic", "ANYSCALE_API_KEY": "anyscale", diff --git a/sdk/oss/tests/pytest/unit/test_mistral_provider_aliases.py b/sdk/oss/tests/pytest/unit/test_mistral_provider_aliases.py new file mode 100644 index 0000000000..2311470eab --- /dev/null +++ b/sdk/oss/tests/pytest/unit/test_mistral_provider_aliases.py @@ -0,0 +1,55 @@ +from types import SimpleNamespace + +from agenta.sdk.contexts.running import RunningContext +from agenta.sdk.managers.secrets import SecretsManager +from agenta.sdk.workflows.runners.daytona import DaytonaRunner + + +def test_secrets_manager_accepts_mistralai_secret_for_mistral_model(monkeypatch): + monkeypatch.setattr( + SecretsManager, + "get_from_route", + staticmethod( + lambda scope="all": [ + { + "kind": "provider_key", + "data": { + "kind": "mistralai", + "provider": {"key": "TEST_KEY"}, + }, + } + ] + ), + ) + + settings = SecretsManager.get_provider_settings("mistral/mistral-small") + + assert settings is not None + assert settings["model"] == "mistral/mistral-small" + assert settings["api_key"] == "TEST_KEY" + + +def test_daytona_runner_exports_canonical_mistral_env_var(monkeypatch): + monkeypatch.setenv("DAYTONA_API_KEY", "test-daytona-key") + runner = DaytonaRunner() + monkeypatch.setattr( + RunningContext, + "get", + staticmethod( + lambda: SimpleNamespace( + vault_secrets=[ + { + "kind": "provider_key", + "data": { + "kind": "mistralai", + "provider": {"key": "TEST_KEY"}, + }, + } + ] + ) + ), + ) + + env_vars = runner._get_provider_env_vars() + + assert env_vars["MISTRAL_API_KEY"] == "TEST_KEY" diff --git a/sdk/pyproject.toml b/sdk/pyproject.toml index 7d03dc7a24..d57fcfa8a5 100644 --- a/sdk/pyproject.toml +++ b/sdk/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "agenta" -version = "0.94.3" +version = "0.94.4" description = "The SDK for agenta is an open-source LLMOps platform." readme = "README.md" authors = [ diff --git a/services/pyproject.toml b/services/pyproject.toml index 7bf8117e63..b2cc931cfd 100644 --- a/services/pyproject.toml +++ b/services/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "services" -version = "0.94.3" +version = "0.94.4" description = "Agenta Services (Chat & Completion)" authors = [ "Mahmoud Mabrouk ", diff --git a/web/ee/package.json b/web/ee/package.json index 6cc13be2f7..090510710c 100644 --- a/web/ee/package.json +++ b/web/ee/package.json @@ -1,6 +1,6 @@ { "name": "@agenta/ee", - "version": "0.94.3", + "version": "0.94.4", "private": true, "engines": { "node": ">=18" diff --git a/web/eslint.config.mjs b/web/eslint.config.mjs index ac93231cb5..657d519421 100644 --- a/web/eslint.config.mjs +++ b/web/eslint.config.mjs @@ -45,12 +45,12 @@ const config = [ "no-restricted-syntax": [ "error", { - selector: 'ExportNamedDeclaration[source.value=/^@agenta/]', + selector: "ExportNamedDeclaration[source.value=/^@agenta/]", message: "Do not re-export from @agenta/* packages. Consumers should import directly from the source package for proper tree-shaking.", }, { - selector: 'ExportAllDeclaration[source.value=/^@agenta/]', + selector: "ExportAllDeclaration[source.value=/^@agenta/]", message: "Do not re-export from @agenta/* packages. Consumers should import directly from the source package for proper tree-shaking.", }, diff --git a/web/oss/package.json b/web/oss/package.json index 7134d80013..8c831d7f5f 100644 --- a/web/oss/package.json +++ b/web/oss/package.json @@ -1,6 +1,6 @@ { "name": "@agenta/oss", - "version": "0.94.3", + "version": "0.94.4", "private": true, "engines": { "node": ">=18" diff --git a/web/oss/src/components/Playground/Components/Modals/TestsetDisconnectConfirmModal/index.tsx b/web/oss/src/components/Playground/Components/Modals/TestsetDisconnectConfirmModal/index.tsx new file mode 100644 index 0000000000..899864a93e --- /dev/null +++ b/web/oss/src/components/Playground/Components/Modals/TestsetDisconnectConfirmModal/index.tsx @@ -0,0 +1,96 @@ +import {loadableController} from "@agenta/entities/loadable" +import {playgroundController} from "@agenta/playground" +import {EnhancedModal, ModalContent} from "@agenta/ui" +import {message} from "@agenta/ui/app-message" +import {Button, Typography} from "antd" +import {useAtomValue, useSetAtom} from "jotai" + +import {initialState, testsetDisconnectConfirmModalAtom} from "./store/state" + +const TestsetDisconnectConfirmModal = () => { + const {open, loadableId, isSaving, intent, meta, onComplete} = useAtomValue( + testsetDisconnectConfirmModalAtom, + ) + const setModalState = useSetAtom(testsetDisconnectConfirmModalAtom) + const disconnectAndReset = useSetAtom(playgroundController.actions.disconnectAndResetToLocal) + const commitChanges = useSetAtom(loadableController.actions.commitChanges) + + const targetName = meta?.targetTestsetName?.trim() || null + const isChangeIntent = intent === "change-testset" + + const title = isChangeIntent + ? targetName + ? `Load ${targetName} test set?` + : "Load different test set?" + : "Save changes?" + + const descriptionLine1 = isChangeIntent + ? targetName + ? `You have unsaved changes. Do you want to save them before loading ${targetName} test set?` + : "You have unsaved changes. Do you want to save them before loading a different test set?" + : "You have unsaved changes. Do you want to save them before disconnecting the testset?" + + const descriptionLine2 = isChangeIntent + ? "Loading testcases from a different testset will remove any previously loaded testcases." + : "Unsaved testcases will convert into local testcases." + + const discardLabel = isChangeIntent ? "Discard & Load" : "Discard & disconnect" + const saveLabel = isChangeIntent ? "Save & load" : "Save & disconnect" + + const handleCancel = () => { + if (isSaving) return + setModalState(initialState) + } + + const handleDiscardAndDisconnect = () => { + if (!loadableId || isSaving) return + disconnectAndReset(loadableId) + onComplete?.() + setModalState(initialState) + } + + const handleSaveAndDisconnect = async () => { + if (!loadableId || isSaving) return + + setModalState((prev) => ({...prev, isSaving: true})) + try { + await commitChanges(loadableId) + disconnectAndReset(loadableId, {preserveRows: true}) + onComplete?.() + setModalState(initialState) + message.success("Testset updated successfully") + } catch (err) { + message.error(err instanceof Error ? err.message : String(err)) + setModalState((prev) => ({...prev, isSaving: false})) + } + } + + return ( + + + + + + } + title={title} + width={500} + > + + {descriptionLine1} + {descriptionLine2} + + + ) +} + +export default TestsetDisconnectConfirmModal diff --git a/web/oss/src/components/Playground/Components/Modals/TestsetDisconnectConfirmModal/store/state.ts b/web/oss/src/components/Playground/Components/Modals/TestsetDisconnectConfirmModal/store/state.ts new file mode 100644 index 0000000000..b6bb024f1f --- /dev/null +++ b/web/oss/src/components/Playground/Components/Modals/TestsetDisconnectConfirmModal/store/state.ts @@ -0,0 +1,27 @@ +import {atom} from "jotai" + +export type TestsetUnsavedChangesIntent = "disconnect" | "change-testset" + +interface TestsetDisconnectConfirmModalState { + open: boolean + loadableId: string | null + isSaving: boolean + intent: TestsetUnsavedChangesIntent + meta?: { + targetTestsetName?: string | null + } + /** Called after the user confirms (save or discard). Lets the opener decide what happens next. */ + onComplete?: () => void +} + +export const initialState: TestsetDisconnectConfirmModalState = { + open: false, + loadableId: null, + isSaving: false, + intent: "disconnect", + meta: undefined, + onComplete: undefined, +} + +export const testsetDisconnectConfirmModalAtom = + atom(initialState) diff --git a/web/oss/src/components/Playground/Components/TestsetDropdown/index.tsx b/web/oss/src/components/Playground/Components/TestsetDropdown/index.tsx index fc02c4d396..3b0eaded6e 100644 --- a/web/oss/src/components/Playground/Components/TestsetDropdown/index.tsx +++ b/web/oss/src/components/Playground/Components/TestsetDropdown/index.tsx @@ -3,19 +3,6 @@ * * Renders a dropdown button in the execution header for testset management. * Adapts based on whether the playground is connected to a local or API-backed testset. - * - * State 1 — Local testset (default): - * Button: "Testset ▼" - * Menu: • Connect testset → opens TestsetSelectionModal (load mode) - * • Add to testset → opens AddToTestsetDrawer with current run results - * - * State 2 — Connected to API-backed testset: - * Button: " ▼" - * Menu: • Sync changes (disabled when no changes) - * • Manage testcases → opens TestsetSelectionModal (edit mode) - * • Change testset → opens TestsetSelectionModal (load mode) - * • Add to testset → opens AddToTestsetDrawer with current run results - * • Disconnect (danger) */ import {useCallback, useEffect, useMemo, useRef, useState} from "react" @@ -34,9 +21,9 @@ import { import { TestsetSelectionModal, type PreviewPanelRenderProps, - type TestsetSelectionMode, type TestsetSelectionPayload, } from "@agenta/playground-ui/components" +import {message} from "@agenta/ui/app-message" import { ArrowsLeftRightIcon, CaretDownIcon, @@ -47,8 +34,8 @@ import { XCircleIcon, } from "@phosphor-icons/react" import type {MenuProps} from "antd" -import {Button, Dropdown, Input, Typography, message} from "antd" -import {atom, useAtomValue, useSetAtom, useStore} from "jotai" +import {Button, Dropdown, Input, Typography} from "antd" +import {atom, useAtom, useAtomValue, useSetAtom, useStore} from "jotai" import dynamic from "next/dynamic" import { @@ -59,7 +46,11 @@ import { import {saveNewTestsetAtom} from "@/oss/state/entities/testset/mutations" import {projectIdAtom} from "@/oss/state/project/selectors/project" +import TestsetDisconnectConfirmModal from "../Modals/TestsetDisconnectConfirmModal" +import {testsetDisconnectConfirmModalAtom} from "../Modals/TestsetDisconnectConfirmModal/store/state" + import {CreateTestsetCardWrapper} from "./CreateTestsetCardWrapper" +import {testsetSelectionModalModeAtom, testsetSyncCommitModalOpenAtom} from "./store/modalState" import {TestsetPreviewPanelWrapper} from "./TestsetPreviewPanelWrapper" // ── Lazy-loaded AddToTestset drawer ──────────────────────────────────────── @@ -161,6 +152,7 @@ export function TestsetDropdown() { const setLoadableName = useSetAtom(loadableController.actions.setName) const initSelectionDraft = useSetAtom(testcaseMolecule.actions.initSelectionDraft) const saveNewTestset = useSetAtom(saveNewTestsetAtom) + const setDisconnectConfirmModalState = useSetAtom(testsetDisconnectConfirmModalAtom) const store = useStore() // ── Derived state ────────────────────────────────────────────────────── @@ -267,7 +259,7 @@ export function TestsetDropdown() { // ── TestsetSelectionModal state ───────────────────────────────────────── // null = closed, "load" = connect/change, "edit" = manage testcases - const [selectionModalMode, setSelectionModalMode] = useState(null) + const [selectionModalMode, setSelectionModalMode] = useAtom(testsetSelectionModalModeAtom) // ── Load/Change mode: connect or replace testset ─────────────────────── const handleLoadConfirm = useCallback( @@ -378,11 +370,42 @@ export function TestsetDropdown() { // ── Disconnect ───────────────────────────────────────────────────────── const handleDisconnect = useCallback(() => { if (!loadableId) return + + if (hasLocalChanges) { + setDisconnectConfirmModalState({ + open: true, + loadableId, + isSaving: false, + intent: "disconnect", + }) + return + } + disconnectAndReset(loadableId) - }, [loadableId, disconnectAndReset]) + }, [loadableId, hasLocalChanges, setDisconnectConfirmModalState, disconnectAndReset]) + + const handleChangeTestset = useCallback(() => { + if (!loadableId) return + + if (hasLocalChanges) { + setDisconnectConfirmModalState({ + open: true, + loadableId, + isSaving: false, + intent: "change-testset", + meta: { + targetTestsetName: null, + }, + onComplete: () => setSelectionModalMode("load"), + }) + return + } + + setSelectionModalMode("load") + }, [loadableId, hasLocalChanges, setDisconnectConfirmModalState, setSelectionModalMode]) // ── Sync changes (EntityCommitModal) ─────────────────────────────────── - const [syncOpen, setSyncOpen] = useState(false) + const [syncOpen, setSyncOpen] = useAtom(testsetSyncCommitModalOpenAtom) const [newTestsetName, setNewTestsetName] = useState("") const [currentSyncMode, setCurrentSyncMode] = useState("commit") const syncModeRef = useRef("commit") @@ -489,7 +512,7 @@ export function TestsetDropdown() { key: "change", icon: , label: "Change testset", - onClick: () => setSelectionModalMode("load"), + onClick: handleChangeTestset, }, { key: "add-to-testset", @@ -513,7 +536,9 @@ export function TestsetDropdown() { hasSuccessfulResults, handleSyncOpen, handleDisconnect, + handleChangeTestset, handleAddToTestset, + handleManageTestcasesClick, ]) if (!loadableId) return null @@ -583,6 +608,9 @@ export function TestsetDropdown() { successMessage="Testset updated successfully" /> + {/* Disconnect with unsaved changes modal */} + + {/* Add to testset drawer — mounted only when open to avoid isDrawerOpenAtom conflicts */} {addToTestsetOpen && ( (null) +export const testsetSyncCommitModalOpenAtom = atom(false) diff --git a/web/oss/src/components/SharedDrawers/AddToTestsetDrawer/atoms/localEntities.ts b/web/oss/src/components/SharedDrawers/AddToTestsetDrawer/atoms/localEntities.ts index 21a84d04f9..e92212c4a1 100644 --- a/web/oss/src/components/SharedDrawers/AddToTestsetDrawer/atoms/localEntities.ts +++ b/web/oss/src/components/SharedDrawers/AddToTestsetDrawer/atoms/localEntities.ts @@ -1,3 +1,4 @@ +import {SYSTEM_FIELDS} from "@agenta/entities/testcase" import {atom} from "jotai" import {testcase} from "@/oss/state/entities/testcase" @@ -322,24 +323,6 @@ export const updateAllLocalEntitiesAtom = atom( // First, mark all non-system columns for removal by setting to undefined if (currentEntity) { - const SYSTEM_FIELDS = new Set([ - "id", - "key", - "testset_id", - "set_id", - "created_at", - "updated_at", - "deleted_at", - "created_by_id", - "updated_by_id", - "deleted_by_id", - "flags", - "tags", - "meta", - "__isSkeleton", - "__isNew", - "testcase_dedup_id", - ]) Object.keys(currentEntity).forEach((key) => { if (!SYSTEM_FIELDS.has(key)) { updates[key] = undefined // Mark for deletion diff --git a/web/oss/src/components/TestcasesTableNew/hooks/constants.ts b/web/oss/src/components/TestcasesTableNew/hooks/constants.ts index c1d1cf3979..00420c5759 100644 --- a/web/oss/src/components/TestcasesTableNew/hooks/constants.ts +++ b/web/oss/src/components/TestcasesTableNew/hooks/constants.ts @@ -16,5 +16,22 @@ export const SYSTEM_COLUMNS = [ "tags", "meta", "__isSkeleton", + "__dedup_id__", "testcase_dedup_id", ] + +const SYSTEM_COLUMN_SET = new Set(SYSTEM_COLUMNS) + +/** + * Returns true when a column key is internal/system, including nested paths. + * Examples: + * - "testcase_dedup_id" -> true + * - "data.testcase_dedup_id" -> true + * - "payload.__dedup_id__" -> true + */ +export const isSystemColumnPath = (columnKey: string): boolean => { + if (!columnKey) return false + + const segments = columnKey.split(".") + return SYSTEM_COLUMN_SET.has(segments[0]) || segments.some((s) => s.startsWith("__")) +} diff --git a/web/oss/src/components/TestcasesTableNew/hooks/useTestcasesTable.ts b/web/oss/src/components/TestcasesTableNew/hooks/useTestcasesTable.ts index f53e5db4b6..dc3a20ffa3 100644 --- a/web/oss/src/components/TestcasesTableNew/hooks/useTestcasesTable.ts +++ b/web/oss/src/components/TestcasesTableNew/hooks/useTestcasesTable.ts @@ -29,6 +29,7 @@ import { testcasesSearchTermAtom, } from "../atoms/tableStore" +import {isSystemColumnPath} from "./constants" import type {TestcaseTableRow, UseTestcasesTableOptions, UseTestcasesTableResult} from "./types" // Re-export types for external consumers @@ -188,6 +189,14 @@ export function useTestcasesTable(options: UseTestcasesTableOptions = {}): UseTe ) const baseColumns = useAtomValue(columnsAtom) // Original columns (for drawer/editing) const columns = useAtomValue(expandedColumnsAtom) // Expanded columns (for table display) + const filteredBaseColumns = useMemo( + () => baseColumns.filter((column) => !isSystemColumnPath(column.key)), + [baseColumns], + ) + const filteredColumns = useMemo( + () => columns.filter((column) => !isSystemColumnPath(column.key)), + [columns], + ) // Check if revision data suggests columns should exist but haven't been derived yet // This catches the gap between data arriving and columns being populated @@ -384,8 +393,8 @@ export function useTestcasesTable(options: UseTestcasesTableOptions = {}): UseTe // Data - row refs (optimized: cells read from entity atoms) rowRefs: displayRowRefs, testcaseIds, // IDs for entity atom access - columns, // Expanded columns for table display - baseColumns, // Original columns for drawer/editing + columns: filteredColumns, // Expanded columns for table display + baseColumns: filteredBaseColumns, // Original columns for drawer/editing // Use combined loading state (includes revisionQuery.isPending for columns) isLoading: combinedIsLoading, error: revisionQuery.error as Error | null, diff --git a/web/oss/src/lib/helpers/llmProviders.ts b/web/oss/src/lib/helpers/llmProviders.ts index 84c7817ce1..0414e02de7 100644 --- a/web/oss/src/lib/helpers/llmProviders.ts +++ b/web/oss/src/lib/helpers/llmProviders.ts @@ -41,7 +41,7 @@ export const transformSecret = (secrets: CustomSecretDTO[] | StandardSecretDTO[] alephalpha: "ALEPHALPHA_API_KEY", groq: "GROQ_API_KEY", mistral: "MISTRAL_API_KEY", - mistralai: "MISTRALAI_API_KEY", + mistralai: "MISTRAL_API_KEY", anthropic: "ANTHROPIC_API_KEY", perplexityai: "PERPLEXITYAI_API_KEY", together_ai: "TOGETHERAI_API_KEY", @@ -85,7 +85,7 @@ export const transformSecret = (secrets: CustomSecretDTO[] | StandardSecretDTO[] export const llmAvailableProviders: LlmProvider[] = [ {title: "OpenAI", key: "", name: "OPENAI_API_KEY"}, - {title: "Mistral AI", key: "", name: "MISTRALAI_API_KEY"}, + {title: "Mistral AI", key: "", name: "MISTRAL_API_KEY"}, {title: "Cohere", key: "", name: "COHERE_API_KEY"}, {title: "Anthropic", key: "", name: "ANTHROPIC_API_KEY"}, {title: "Anyscale", key: "", name: "ANYSCALE_API_KEY"}, diff --git a/web/oss/src/state/app/atoms/vault.ts b/web/oss/src/state/app/atoms/vault.ts index 20d19205e9..1ed3431ed2 100644 --- a/web/oss/src/state/app/atoms/vault.ts +++ b/web/oss/src/state/app/atoms/vault.ts @@ -155,7 +155,9 @@ const getEnvNameMap = (): Record => ({ DEEPINFRA_API_KEY: SecretDTOProvider.DEEPINFRA, ALEPHALPHA_API_KEY: SecretDTOProvider.ALEPHALPHA, GROQ_API_KEY: SecretDTOProvider.GROQ, - MISTRAL_API_KEY: SecretDTOProvider.MISTRALAI, + MISTRAL_API_KEY: SecretDTOProvider.MISTRAL, + // Backward-compatible mapping for legacy Mistral provider name + MISTRALAI_API_KEY: SecretDTOProvider.MISTRAL, ANTHROPIC_API_KEY: SecretDTOProvider.ANTHROPIC, PERPLEXITYAI_API_KEY: SecretDTOProvider.PERPLEXITYAI, TOGETHERAI_API_KEY: SecretDTOProvider.TOGETHERAI, @@ -174,6 +176,13 @@ export const createStandardSecretAtom = atom(null, async (get, set, provider: Ll const updateMutation = get(updateVaultSecretMutationAtom) try { + const providerKind = envNameMap[provider.name as string] + if (!providerKind) { + throw new Error( + `[vault] Unknown provider name "${provider.name}" when creating standard secret`, + ) + } + // Match the original working payload structure exactly const payload = { header: { @@ -183,7 +192,7 @@ export const createStandardSecretAtom = atom(null, async (get, set, provider: Ll secret: { kind: SecretDTOKind.PROVIDER_KEY, data: { - kind: envNameMap[provider.name as string], + kind: providerKind, provider: { key: provider.key, }, diff --git a/web/oss/src/state/entities/testcase/columnState.ts b/web/oss/src/state/entities/testcase/columnState.ts index 8b9d14b919..4966a26f2e 100644 --- a/web/oss/src/state/entities/testcase/columnState.ts +++ b/web/oss/src/state/entities/testcase/columnState.ts @@ -1,3 +1,4 @@ +import {SYSTEM_FIELDS} from "@agenta/entities/testcase" import {atom} from "jotai" import {atomFamily} from "jotai/utils" @@ -159,28 +160,6 @@ export interface Column { name: string } -/** - * System fields to exclude from column derivation - */ -const SYSTEM_FIELDS = new Set([ - "id", - "key", - "testset_id", - "set_id", - "created_at", - "updated_at", - "deleted_at", - "created_by_id", - "updated_by_id", - "deleted_by_id", - "flags", - "tags", - "meta", - "__isSkeleton", - "testcase_dedup_id", - "__dedup_id__", -]) - // ============================================================================ // LOCAL COLUMN STATE (REVISION-SCOPED) // Tracks columns added locally that don't exist in entity data yet @@ -442,6 +421,9 @@ function collectObjectSubKeysRecursive( if (currentDepth >= MAX_COLUMN_DEPTH) return Object.entries(obj).forEach(([subKey, subValue]) => { + // Never expose internal fields as nested columns. + if (subKey.startsWith("__")) return + const fullPath = prefix ? `${prefix}.${subKey}` : subKey // Skip if this path is marked as deleted diff --git a/web/oss/src/state/entities/testcase/schema.ts b/web/oss/src/state/entities/testcase/schema.ts index c41ae41b65..c7408ddb14 100644 --- a/web/oss/src/state/entities/testcase/schema.ts +++ b/web/oss/src/state/entities/testcase/schema.ts @@ -1,3 +1,4 @@ +import {SYSTEM_FIELDS} from "@agenta/entities/testcase" import {z} from "zod" /** @@ -69,6 +70,17 @@ export const flattenedTestcaseSchema = testcaseSchema.omit({data: true}).passthr export type FlattenedTestcase = z.infer & Record +const isRecord = (value: unknown): value is Record => + !!value && typeof value === "object" && !Array.isArray(value) + +const hasWrappedDataShape = (row: Record): boolean => { + if (!isRecord(row.data)) return false + + const keys = Object.keys(row) + + return keys.every((key) => key === "data" || SYSTEM_FIELDS.has(key)) +} + /** * Schema for testcase creation */ @@ -156,6 +168,50 @@ export function flattenTestcase(testcase: Testcase): FlattenedTestcase { } } +/** + * Normalize unknown testcase-like input to flattened testcase shape. + * + * Handles mixed shapes that appear in UI state/cache: + * - Flat row: `{id, input, expected}` + * - Wrapped row: `{id, data: {input, expected}}` + * - Wrapped in testcase key: `{testcase: {...}}` + */ +export function normalizeToFlattenedTestcase(input: unknown): FlattenedTestcase | null { + if (!isRecord(input)) return null + + const base = isRecord(input.testcase) ? input.testcase : input + + if (hasWrappedDataShape(base)) { + const data = isRecord(base.data) ? base.data : {} + const {data: _data, ...rest} = base + + return { + ...data, + ...rest, + } as FlattenedTestcase + } + + return base as FlattenedTestcase +} + +/** + * Extract user-editable testcase fields from mixed testcase row shapes. + * Removes all system/internal fields from the normalized row. + */ +export function extractTestcaseUserData(input: unknown): Record | null { + const normalized = normalizeToFlattenedTestcase(input) + if (!normalized) return null + + const data: Record = {} + for (const [key, value] of Object.entries(normalized)) { + if (!SYSTEM_FIELDS.has(key) && key !== "data") { + data[key] = value + } + } + + return data +} + /** * Transform flattened testcase back to API format */ diff --git a/web/oss/src/state/entities/testcase/testcaseEntity.ts b/web/oss/src/state/entities/testcase/testcaseEntity.ts index 98afa28292..8e4104f65f 100644 --- a/web/oss/src/state/entities/testcase/testcaseEntity.ts +++ b/web/oss/src/state/entities/testcase/testcaseEntity.ts @@ -18,7 +18,12 @@ import { pendingDeletedColumnsAtom, } from "./columnState" import {currentRevisionIdAtom} from "./queries" -import {flattenTestcase, testcaseSchema, type FlattenedTestcase} from "./schema" +import { + flattenTestcase, + normalizeToFlattenedTestcase, + testcaseSchema, + type FlattenedTestcase, +} from "./schema" // ============================================================================ // TESTCASE IDS ATOM @@ -431,7 +436,7 @@ const testcaseDraftState = createEntityDraftState { const queryAtom = testcaseQueryAtomFamily(id) - return atom((get) => get(queryAtom).data ?? null) + return atom((get) => normalizeToFlattenedTestcase(get(queryAtom).data) ?? null) }, // Entire testcase is draftable @@ -445,7 +450,7 @@ const testcaseDraftState = createEntityDraftState // Fall back to server data from query const query = get(testcaseQueryAtomFamily(testcaseId)) - const data = query.data ?? null + const data = normalizeToFlattenedTestcase(query.data) ?? null // Apply pending column changes to server data if (data) { diff --git a/web/oss/src/state/entities/testcase/testcaseMutations.ts b/web/oss/src/state/entities/testcase/testcaseMutations.ts index 79eaa6e1a5..c33785dc47 100644 --- a/web/oss/src/state/entities/testcase/testcaseMutations.ts +++ b/web/oss/src/state/entities/testcase/testcaseMutations.ts @@ -2,7 +2,7 @@ import {atom} from "jotai" import {addColumnAtom, currentColumnsAtom} from "./columnState" import {testsetIdAtom} from "./queries" -import type {FlattenedTestcase} from "./schema" +import {extractTestcaseUserData, type FlattenedTestcase} from "./schema" import { addNewEntityIdAtom, markDeletedAtom, @@ -13,6 +13,10 @@ import { testcaseIdsAtom, } from "./testcaseEntity" +const toCanonicalRowData = (row: Record): Record => { + return extractTestcaseUserData(row) ?? {} +} + // ============================================================================ // DELETE TESTCASES MUTATION // Handles deletion of both new and existing rows @@ -142,6 +146,7 @@ export const createTestcasesAtom = atom( return {ids: [], count: 0, skipped: 0} } + const canonicalRows = rows.map((row) => toCanonicalRowData(row)) const testsetId = testsetIdOverride ?? get(testsetIdAtom) ?? "" const columns = get(currentColumnsAtom) @@ -180,7 +185,7 @@ export const createTestcasesAtom = atom( // Add new columns if needed if (!skipColumnSync) { const existingColumnKeys = new Set(columns.map((c) => c.key)) - for (const row of rows) { + for (const row of canonicalRows) { for (const key of Object.keys(row)) { if (!existingColumnKeys.has(key)) { set(addColumnAtom, key) @@ -195,12 +200,12 @@ export const createTestcasesAtom = atom( let skipped = 0 const timestamp = Date.now() - for (let i = 0; i < rows.length; i++) { - const row = rows[i] + for (let i = 0; i < canonicalRows.length; i++) { + const rowData = canonicalRows[i] // Check deduplication if (existingDataSet) { - const rowDataStr = JSON.stringify(row) + const rowDataStr = JSON.stringify(rowData) if (existingDataSet.has(rowDataStr)) { skipped++ continue @@ -212,7 +217,7 @@ export const createTestcasesAtom = atom( const flattenedRow: FlattenedTestcase = { id: entityId, testset_id: testsetId, - ...row, + ...rowData, } // Register and create draft diff --git a/web/oss/src/state/entities/testset/controller.ts b/web/oss/src/state/entities/testset/controller.ts index 5285bc8a16..2868ba5db1 100644 --- a/web/oss/src/state/entities/testset/controller.ts +++ b/web/oss/src/state/entities/testset/controller.ts @@ -45,6 +45,7 @@ * ``` */ +import {SYSTEM_FIELDS} from "@agenta/entities/testcase" import {atom} from "jotai" import {atomFamily} from "jotai/utils" import {atomWithQuery} from "jotai-tanstack-query" @@ -93,25 +94,6 @@ import {invalidateRevisionsListCache} from "./store" // SYSTEM FIELDS (excluded from column derivation) // ============================================================================ -const SYSTEM_FIELDS = new Set([ - "id", - "key", - "testset_id", - "set_id", - "created_at", - "updated_at", - "deleted_at", - "created_by_id", - "updated_by_id", - "deleted_by_id", - "flags", - "tags", - "meta", - "__isSkeleton", - "testcase_dedup_id", - "__dedup_id__", -]) - // ============================================================================ // REVISION WITH TESTCASES QUERY // Fetches revision with testcases included (for column derivation) diff --git a/web/oss/tests/manual/cell-renderers/test-extract-chat-messages.ts b/web/oss/tests/manual/cell-renderers/test-extract-chat-messages.ts index b3e458ceec..4768f7c5a2 100644 --- a/web/oss/tests/manual/cell-renderers/test-extract-chat-messages.ts +++ b/web/oss/tests/manual/cell-renderers/test-extract-chat-messages.ts @@ -31,9 +31,7 @@ const run = () => { assert.deepEqual(extractChatMessages(nested), [{role: "user", content: "nested"}]) assert.equal(extractChatMessages(deep), null) - assert.deepEqual(extractChatMessages(choices), [ - {role: "assistant", content: "from choices"}, - ]) + assert.deepEqual(extractChatMessages(choices), [{role: "assistant", content: "from choices"}]) assert.deepEqual(extractChatMessages(single), [{role: "assistant", content: "single message"}]) assert.equal(extractChatMessages(plainJson), null) diff --git a/web/oss/tests/playwright/acceptance/app/test.ts b/web/oss/tests/playwright/acceptance/app/test.ts index d6d98dee78..37281f6e1f 100644 --- a/web/oss/tests/playwright/acceptance/app/test.ts +++ b/web/oss/tests/playwright/acceptance/app/test.ts @@ -57,10 +57,7 @@ const testWithAppFixtures = baseTest.extend({ await uiHelpers.typeWithDelay('input[placeholder="Enter a name"]', appName) await page.getByText(appType).first().click() const createAppPromise = page.waitForResponse((response) => { - if ( - !response.url().includes("/apps") || - response.request().method() !== "POST" - ) { + if (!response.url().includes("/apps") || response.request().method() !== "POST") { return false } diff --git a/web/oss/tests/playwright/acceptance/playground/tests.ts b/web/oss/tests/playwright/acceptance/playground/tests.ts index 2074aaee3e..fdefcd368d 100644 --- a/web/oss/tests/playwright/acceptance/playground/tests.ts +++ b/web/oss/tests/playwright/acceptance/playground/tests.ts @@ -85,9 +85,9 @@ const testWithVariantFixtures = baseTest.extend({ await uiHelpers.expectPath(`/apps/${appId}/playground`) } - await expect( - page.getByRole("button", {name: "Run", exact: true}).first(), - ).toBeVisible({timeout: 30000}) + await expect(page.getByRole("button", {name: "Run", exact: true}).first()).toBeVisible({ + timeout: 30000, + }) }) }, diff --git a/web/oss/tests/playwright/acceptance/prompt-registry/index.ts b/web/oss/tests/playwright/acceptance/prompt-registry/index.ts index 8819583b2e..066e09b327 100644 --- a/web/oss/tests/playwright/acceptance/prompt-registry/index.ts +++ b/web/oss/tests/playwright/acceptance/prompt-registry/index.ts @@ -37,9 +37,9 @@ const promptRegistryTests = () => { await uiHelpers.expectPath("/prompts") // Verify the Prompts heading is visible - await expect( - page.getByRole("heading", {name: /prompts/i}).first(), - ).toBeVisible({timeout: 15000}) + await expect(page.getByRole("heading", {name: /prompts/i}).first()).toBeVisible({ + timeout: 15000, + }) // Verify the prompts table is visible (uses div-based rows) const promptsTable = page.getByRole("table").first() diff --git a/web/oss/tests/playwright/acceptance/smoke.spec.ts b/web/oss/tests/playwright/acceptance/smoke.spec.ts index 5182c6e0da..c0011a7985 100644 --- a/web/oss/tests/playwright/acceptance/smoke.spec.ts +++ b/web/oss/tests/playwright/acceptance/smoke.spec.ts @@ -1,6 +1,8 @@ import {test, expect} from "@playwright/test" -test("smoke: auth works and can navigate to apps @scope:auth @coverage:smoke @path:happy @lens:functional @cost:free @license:oss", async ({page}) => { +test("smoke: auth works and can navigate to apps @scope:auth @coverage:smoke @path:happy @lens:functional @cost:free @license:oss", async ({ + page, +}) => { test.setTimeout(10000) await page.goto("/apps") await page.waitForURL("**/apps", {timeout: 5000}) diff --git a/web/oss/tests/playwright/acceptance/testsset/index.ts b/web/oss/tests/playwright/acceptance/testsset/index.ts index 49c752e634..b85333f609 100644 --- a/web/oss/tests/playwright/acceptance/testsset/index.ts +++ b/web/oss/tests/playwright/acceptance/testsset/index.ts @@ -86,7 +86,9 @@ const testsetTests = () => { // 6. Verify testset page await uiHelpers.waitForPath(`/testsets/${testsetId}`) - await expect(page.getByRole("heading", {name: /testset|test set/i}).first()).toBeVisible() + await expect( + page.getByRole("heading", {name: /testset|test set/i}).first(), + ).toBeVisible() const response = await testsetResponsePromise const testset = response.testset diff --git a/web/package.json b/web/package.json index 03afe5e306..5686d2fcd6 100644 --- a/web/package.json +++ b/web/package.json @@ -1,6 +1,6 @@ { "name": "agenta-web", - "version": "0.94.3", + "version": "0.94.4", "workspaces": [ "ee", "oss", diff --git a/web/packages/agenta-entities/src/loadable/controller.ts b/web/packages/agenta-entities/src/loadable/controller.ts index 7c4caab438..1b1b2aa72c 100644 --- a/web/packages/agenta-entities/src/loadable/controller.ts +++ b/web/packages/agenta-entities/src/loadable/controller.ts @@ -40,7 +40,7 @@ import {atomFamily} from "jotai-family" import {queryClientAtom} from "jotai-tanstack-query" import {loadableColumnsFromRunnableAtomFamily} from "../runnable/bridge" -import type {Testcase} from "../testcase/core" +import {SYSTEM_FIELDS, type Testcase} from "../testcase/core" import {testcaseMolecule} from "../testcase/state/molecule" import { setTestcaseIdsAtom, @@ -89,28 +89,6 @@ import {createOutputMappingId, extractPaths} from "./utils" // CONSTANTS // ============================================================================ -/** - * System fields to exclude from column comparisons and row data - * These are entity metadata fields, not actual testcase data - */ -const SYSTEM_FIELDS = new Set([ - "id", - "flags", - "tags", - "meta", - "created_at", - "updated_at", - "deleted_at", - "created_by_id", - "updated_by_id", - "deleted_by_id", - "testset_id", - "set_id", - "testset_variant_id", - "revision_id", - "testcase_dedup_id", -]) - const LOCAL_TESTCASE_PREFIXES = ["new-", "local-"] as const const VERSION_SUFFIX_REGEX = /\s+v\d+\s*$/i diff --git a/web/packages/agenta-entities/src/testcase/core/schema.ts b/web/packages/agenta-entities/src/testcase/core/schema.ts index fa6880656c..a6c6951150 100644 --- a/web/packages/agenta-entities/src/testcase/core/schema.ts +++ b/web/packages/agenta-entities/src/testcase/core/schema.ts @@ -277,6 +277,8 @@ export const SYSTEM_FIELDS = new Set([ "key", "testset_id", "set_id", + "testset_variant_id", + "revision_id", "created_at", "updated_at", "deleted_at", @@ -287,6 +289,8 @@ export const SYSTEM_FIELDS = new Set([ "tags", "meta", "__isSkeleton", + "__isNew", + "__dedup_id__", "testcase_dedup_id", ]) diff --git a/web/packages/agenta-entities/src/testset/core/schema.ts b/web/packages/agenta-entities/src/testset/core/schema.ts index ce29a397a6..fae3b9b14e 100644 --- a/web/packages/agenta-entities/src/testset/core/schema.ts +++ b/web/packages/agenta-entities/src/testset/core/schema.ts @@ -30,6 +30,7 @@ import { safeParseWithLogging, getVersionLabel, } from "../../shared" +import {SYSTEM_FIELDS} from "../../testcase/core" // ============================================================================ // REVISION SCHEMA @@ -240,35 +241,13 @@ export type Variant = z.infer // NORMALIZATION UTILITIES // ============================================================================ -/** - * System/metadata fields to exclude when normalizing testcase data - */ -const TESTCASE_SYSTEM_FIELDS = new Set([ - "id", - "key", - "testset_id", - "set_id", - "created_at", - "updated_at", - "deleted_at", - "created_by_id", - "updated_by_id", - "deleted_by_id", - "flags", - "tags", - "meta", - "__isSkeleton", - "testcase_dedup_id", - "__dedup_id__", -]) - /** * Filter system fields from an object */ function filterSystemFields(obj: Record): Record { const filtered: Record = {} for (const [key, value] of Object.entries(obj)) { - if (!TESTCASE_SYSTEM_FIELDS.has(key)) { + if (!SYSTEM_FIELDS.has(key)) { filtered[key] = value } } @@ -290,7 +269,7 @@ function normalizeTestcase(tc: Record): Record const userData: Record = {} for (const [key, value] of Object.entries(tc)) { - if (!TESTCASE_SYSTEM_FIELDS.has(key) && key !== "data") { + if (!SYSTEM_FIELDS.has(key) && key !== "data") { userData[key] = value } } diff --git a/web/packages/agenta-entities/src/testset/state/mutations.ts b/web/packages/agenta-entities/src/testset/state/mutations.ts index 6bf5ad8c6d..0d341f8c6f 100644 --- a/web/packages/agenta-entities/src/testset/state/mutations.ts +++ b/web/packages/agenta-entities/src/testset/state/mutations.ts @@ -8,6 +8,8 @@ import {projectIdAtom} from "@agenta/shared/state" import {atom} from "jotai" +import {isRecord} from "../../shared" +import {SYSTEM_FIELDS} from "../../testcase/core" // Testcase atoms - import directly from internal modules (not public index) import { currentRevisionIdAtom, @@ -48,8 +50,13 @@ import { // INTERNAL HELPERS // ============================================================================ -// System fields to exclude from column operations -const SYSTEM_FIELDS = new Set(["id", "__id", "__isSkeleton", "key", "created_at", "updated_at"]) +/** Fields that should never appear as user columns inside entity.data */ +const DATA_INTERNAL_FIELDS = new Set([ + "__isSkeleton", + "__isNew", + "__dedup_id__", + "testcase_dedup_id", +]) interface Column { key: string @@ -75,7 +82,7 @@ const currentColumnsAtom = atom((get) => { const entity = get(testcaseEntityAtomFamily(id)) if (!entity?.data) continue for (const key of Object.keys(entity.data)) { - if (!SYSTEM_FIELDS.has(key)) { + if (!DATA_INTERNAL_FIELDS.has(key)) { keySet.add(key) } } @@ -130,27 +137,6 @@ export interface SaveTestsetResult { error?: Error } -const TESTCASE_SYSTEM_FIELDS = new Set([ - "id", - "flags", - "tags", - "meta", - "created_at", - "updated_at", - "deleted_at", - "created_by_id", - "updated_by_id", - "deleted_by_id", - "testset_id", - "set_id", - "testset_variant_id", - "revision_id", - "testcase_dedup_id", -]) - -const isRecord = (value: unknown): value is Record => - !!value && typeof value === "object" && !Array.isArray(value) - const normalizeCommittedRows = ( testcases: unknown, ): {id: string; data: Record}[] => { @@ -169,7 +155,7 @@ const normalizeCommittedRows = ( data = testcase.data } else { for (const [key, value] of Object.entries(testcase)) { - if (!TESTCASE_SYSTEM_FIELDS.has(key) && key !== "data") { + if (!SYSTEM_FIELDS.has(key) && key !== "data") { data[key] = value } } diff --git a/web/packages/agenta-entity-ui/src/selection/adapters/testsetRelationAdapter.ts b/web/packages/agenta-entity-ui/src/selection/adapters/testsetRelationAdapter.ts index b4d1494949..e202cffff6 100644 --- a/web/packages/agenta-entity-ui/src/selection/adapters/testsetRelationAdapter.ts +++ b/web/packages/agenta-entity-ui/src/selection/adapters/testsetRelationAdapter.ts @@ -91,6 +91,8 @@ export const testsetAdapter = createTwoLevelAdapter({ onBeforeLoad: (testsetId: string) => { testsetSelectionConfig.enableRevisionsQuery(testsetId) }, + // Hide v0 revisions in selection UIs (placeholder revisions with no displayable data). + filterItems: (entity: unknown) => (entity as {version?: number}).version !== 0, getLabelNode: (entity: unknown) => { const r = entity as { version?: number diff --git a/web/packages/agenta-playground-ui/src/components/adapters/VariableControlAdapter.tsx b/web/packages/agenta-playground-ui/src/components/adapters/VariableControlAdapter.tsx index 79ac3dc714..5c77140dc3 100644 --- a/web/packages/agenta-playground-ui/src/components/adapters/VariableControlAdapter.tsx +++ b/web/packages/agenta-playground-ui/src/components/adapters/VariableControlAdapter.tsx @@ -1,6 +1,7 @@ import React, {useCallback, useEffect, useMemo, useRef} from "react" import {executionItemController, playgroundController} from "@agenta/playground" +import {isJsonString} from "@agenta/shared/utils" import {getCollapseStyle} from "@agenta/ui/components/presentational" import {TOGGLE_MARKDOWN_VIEW, EditorProvider, useLexicalComposerContext} from "@agenta/ui/editor" import type {EditorProps} from "@agenta/ui/editor" @@ -142,7 +143,18 @@ const VariableControlAdapter: React.FC = ({ // For object/array types, provide a sensible default when value is empty const isJsonType = portType === "object" || portType === "array" const jsonDefault = portType === "array" ? "[]" : "{}" - const effectiveValue = isJsonType && (!value || value === "") ? jsonDefault : value + + // Capture whether the initial value looks like JSON at mount time. + // This is safe because codeOnly is set once before Lexical initialises. + // Switching codeOnly dynamically at runtime crashes Lexical + // (MarkdownShortcuts: missing dependency code), so the flag is immutable. + const initialValueLooksLikeJson = useRef( + typeof value === "string" && !!value && isJsonString(value), + ).current + + const isJsonEditor = isJsonType || initialValueLooksLikeJson + const effectiveValue = + isJsonEditor && isJsonType && (!value || value === "") ? jsonDefault : value // Seed the default back to the store so the execution payload has the correct value useEffect(() => { @@ -236,8 +248,8 @@ const VariableControlAdapter: React.FC = ({ ) } - // Object/array types → JSON code editor - const mergedEditorProps: EditorProps = isJsonType + // Object/array types (and detected JSON strings) → JSON code editor + const mergedEditorProps: EditorProps = isJsonEditor ? {codeOnly: true, language: "json", enableResize: false, boundWidth: true, ...editorProps} : {enableResize: false, boundWidth: true, ...editorProps} @@ -248,9 +260,9 @@ const VariableControlAdapter: React.FC = ({ initialValue={effectiveValue} placeholder={effectivePlaceholder} showToolbar={false} - codeOnly={isJsonType || !!editorProps?.codeOnly} - language={isJsonType ? "json" : undefined} - enableTokens={!isJsonType && !editorProps?.codeOnly} + codeOnly={isJsonEditor || !!editorProps?.codeOnly} + language={isJsonEditor ? "json" : undefined} + enableTokens={!isJsonEditor && !editorProps?.codeOnly} disabled={isEffectivelyDisabled} > @@ -285,7 +297,7 @@ const VariableControlAdapter: React.FC = ({ : viewType === "single" && view !== "focus" ? "" : "bg-transparent", - isJsonType && "!pt-[11px] !pb-0 [&_.agenta-editor-wrapper]:!mb-0", + isJsonEditor && "!pt-[11px] !pb-0 [&_.agenta-editor-wrapper]:!mb-0", className, )} editorProps={mergedEditorProps} diff --git a/web/packages/agenta-playground/src/state/controllers/playgroundController.ts b/web/packages/agenta-playground/src/state/controllers/playgroundController.ts index 3b974b0498..32e1232f24 100644 --- a/web/packages/agenta-playground/src/state/controllers/playgroundController.ts +++ b/web/packages/agenta-playground/src/state/controllers/playgroundController.ts @@ -77,6 +77,7 @@ import { newTestcaseDataHashAtom, } from "../execution/selectors" import {extractAndLoadChatMessagesAtom} from "../helpers/extractAndLoadChatMessages" +import {normalizeTestcaseRowsForLoad} from "../helpers/testcaseRowNormalization" import type {EntitySelection, PlaygroundNode, RunnableType} from "../types" import {getRunnableBridge} from "./runnableBridgeAccess" @@ -449,29 +450,46 @@ const resetAllAtom = atom(null, (_get, set) => { * Disconnect from testset and reset to local mode * * This compound action: - * 1. Calls loadable disconnect (clears connectedSourceId, testcase IDs) - * 2. Regenerates a local testset name from the primary node's label - * 3. Creates an initial empty row for testcases + * 1. Snapshots current rows when `preserveRows` is true (must happen before disconnect) + * 2. Calls loadable disconnect (clears connectedSourceId, testcase IDs) + * 3. Regenerates a local testset name from the primary node's label + * 4. Re-populates with snapshotted rows or creates an initial empty row * - * This ensures the playground returns to the same state as initial setup. + * When `preserveRows` is false (default), the playground returns to the same + * state as initial setup. When true (e.g. after "Save & disconnect"), the + * committed data stays visible as local rows. */ -const disconnectAndResetToLocalAtom = atom(null, (get, set, loadableId: string) => { - const rootNode = get(playgroundNodesAtom).find((n) => n.depth === 0) - if (!rootNode) return +const disconnectAndResetToLocalAtom = atom( + null, + (get, set, loadableId: string, options?: {preserveRows?: boolean}) => { + const rootNode = get(playgroundNodesAtom).find((n) => n.depth === 0) + if (!rootNode) return - // 1. Call loadable disconnect action - set(loadableController.actions.disconnect, loadableId) + // 1. Snapshot current rows before disconnect wipes testcase IDs + const rowsSnapshot = options?.preserveRows + ? get(loadableController.selectors.rows(loadableId)) + : null - // 2. Generate and set local testset name - const localTestsetName = generateLocalTestsetName(rootNode.label) - set(connectedTestsetAtom, { - id: null, // null id indicates it's a local (unsaved) testset - name: localTestsetName, - }) + // 2. Call loadable disconnect action + set(loadableController.actions.disconnect, loadableId) - // 3. Create an initial empty row via loadableController (uses testcaseMolecule) - set(loadableController.actions.addRow, loadableId, {}) -}) + // 3. Generate and set local testset name + const localTestsetName = generateLocalTestsetName(rootNode.label) + set(connectedTestsetAtom, { + id: null, // null id indicates it's a local (unsaved) testset + name: localTestsetName, + }) + + // 4. Re-populate with snapshotted rows or create an initial empty row + if (rowsSnapshot && rowsSnapshot.length > 0) { + for (const row of rowsSnapshot) { + set(loadableController.actions.addRow, loadableId, row.data ?? {}) + } + } else { + set(loadableController.actions.addRow, loadableId, {}) + } + }, +) // ============================================================================ // WP1: TESTSET CONNECTION COMPOUND ACTIONS @@ -491,18 +509,21 @@ const disconnectAndResetToLocalAtom = atom(null, (get, set, loadableId: string) const connectToTestsetAtom = atom(null, (get, set, payload: ConnectToTestsetPayload) => { const {loadableId, revisionId, testcases, testsetName, testsetId, revisionVersion} = payload - // Generate display name from testset name and version + // Generate a fallback display name from the available selection info const displayName = testsetName ? revisionVersion != null - ? `${testsetName} v${revisionVersion}` + ? `${testsetName} (v${revisionVersion})` : testsetName : undefined - // Ensure testcases have IDs - const testcasesWithIds = testcases.map((tc, index) => { - const id = tc.id ?? `testcase-${Date.now()}-${index}` - return {id, ...tc} + const normalizedRows = normalizeTestcaseRowsForLoad(testcases) + + // Ensure testcases have IDs and store them in nested testcase formatat + const testcasesWithIds = normalizedRows.map((row, index) => { + const id = row.id ?? `testcase-${Date.now()}-${index}` + return {id, data: row.data} }) + const flatRows = testcasesWithIds.map(({id, data}) => ({id, ...data})) // Connect to source via loadable controller set( @@ -527,7 +548,7 @@ const connectToTestsetAtom = atom(null, (get, set, payload: ConnectToTestsetPayl if (isChat) { set(extractAndLoadChatMessagesAtom, { loadableId, - testcaseRows: testcasesWithIds as Record[], + testcaseRows: flatRows, skipBlankMessage: true, }) } @@ -544,9 +565,11 @@ const connectToTestsetAtom = atom(null, (get, set, payload: ConnectToTestsetPayl */ const importTestcasesAtom = atom(null, (get, set, payload: ImportTestcasesPayload) => { const {loadableId, testcases} = payload + const normalizedRows = normalizeTestcaseRowsForLoad(testcases) + const flatRows = normalizedRows.map(({id, data}) => (id ? {id, ...data} : {...data})) // Import rows via loadable controller (stays in local mode) - set(loadableController.actions.importRows, loadableId, testcases) + set(loadableController.actions.importRows, loadableId, flatRows) // Extract chat messages from imported testcase rows if in chat mode. // Same reasoning as connectToTestsetAtom — the entity layer stores `messages` @@ -555,7 +578,7 @@ const importTestcasesAtom = atom(null, (get, set, payload: ImportTestcasesPayloa if (isChat) { set(extractAndLoadChatMessagesAtom, { loadableId, - testcaseRows: testcases, + testcaseRows: flatRows, skipBlankMessage: true, }) } diff --git a/web/packages/agenta-playground/src/state/execution/selectors.ts b/web/packages/agenta-playground/src/state/execution/selectors.ts index 7704f550c6..a58b642b27 100644 --- a/web/packages/agenta-playground/src/state/execution/selectors.ts +++ b/web/packages/agenta-playground/src/state/execution/selectors.ts @@ -36,6 +36,18 @@ import {displayedEntityIdsAtom} from "./displayedEntities" import {createExecutionItemHandle, type ExecutionItemLifecycleSnapshot} from "./executionItems" import type {RunStatus} from "./types" +const toDisplayString = (value: unknown): string => { + if (value === undefined || value === null) return "" + if (typeof value === "string") return value + if (typeof value === "number" || typeof value === "boolean") return String(value) + + try { + return JSON.stringify(value) + } catch { + return String(value) + } +} + // ============================================================================ // CONTEXT SELECTORS (derived from playground state) // ============================================================================ @@ -211,7 +223,7 @@ export const rowVariableValueAtomFamily = atomFamily( if (!variableId) return "" const row = get(rowDataWithContextAtomFamily(rowId)) const value = row?.data?.[variableId] - return typeof value === "string" ? value : String(value ?? "") + return toDisplayString(value) }), ) @@ -249,7 +261,7 @@ export const testcaseCellValueAtomFamily = atomFamily( atom((get) => { if (!testcaseId || !column) return "" const value = get(testcaseMolecule.atoms.cell({id: testcaseId, column})) - return value !== undefined && value !== null ? String(value) : "" + return toDisplayString(value) }), (a, b) => a.testcaseId === b.testcaseId && a.column === b.column, ) diff --git a/web/packages/agenta-playground/src/state/helpers/loadTestsetNormalizedMutation.ts b/web/packages/agenta-playground/src/state/helpers/loadTestsetNormalizedMutation.ts index 52599f7765..5c763f3b4e 100644 --- a/web/packages/agenta-playground/src/state/helpers/loadTestsetNormalizedMutation.ts +++ b/web/packages/agenta-playground/src/state/helpers/loadTestsetNormalizedMutation.ts @@ -5,6 +5,7 @@ import {clearAllMessagesAtom} from "../chat/messageReducer" import {derivedLoadableIdAtom, isChatModeAtom} from "../execution/selectors" import {extractAndLoadChatMessagesAtom} from "./extractAndLoadChatMessages" +import {normalizeTestcaseRowsForLoad} from "./testcaseRowNormalization" const MESSAGE_FIELD_KEYS = new Set([ "messages", @@ -38,22 +39,20 @@ export const loadTestsetNormalizedMutationAtom = atom( if (!loadableId) return const dataset = Array.isArray(testsetData) ? testsetData : [] + const normalizedRows = normalizeTestcaseRowsForLoad(dataset) + const flatRows = normalizedRows.map(({id, data}) => (id ? {id, ...data} : {...data})) if (isChatVariant) { set(clearAllMessagesAtom, {loadableId}) - const rowData = (dataset[0] || {}) as Record + const rowData = (normalizedRows[0]?.data || {}) as Record const keys = Object.keys(rowData).filter((k) => !MESSAGE_FIELD_KEYS.has(k)) const updateData: Record = {} for (const key of keys) { const raw = rowData[key] if (raw === undefined) continue - updateData[key] = Array.isArray(raw) - ? JSON.stringify(raw) - : typeof raw === "string" - ? raw - : String(raw ?? "") + updateData[key] = raw } const existingRowIds = get(loadableController.selectors.displayRowIds(loadableId)) @@ -65,19 +64,14 @@ export const loadTestsetNormalizedMutationAtom = atom( } else { set(loadableController.actions.clearRows, loadableId) - for (const row of dataset) { - const rowData = (row || {}) as Record - const keys = Object.keys(rowData).filter((k) => !MESSAGE_FIELD_KEYS.has(k)) + for (const row of normalizedRows) { + const keys = Object.keys(row.data).filter((k) => !MESSAGE_FIELD_KEYS.has(k)) const data: Record = {} for (const key of keys) { - const raw = rowData[key] + const raw = row.data[key] if (raw === undefined) continue - data[key] = Array.isArray(raw) - ? JSON.stringify(raw) - : typeof raw === "string" - ? raw - : String(raw ?? "") + data[key] = raw } set(loadableController.actions.addRow, loadableId, data) @@ -89,7 +83,7 @@ export const loadTestsetNormalizedMutationAtom = atom( // Delegate to the shared chat message extraction helper set(extractAndLoadChatMessagesAtom, { loadableId, - testcaseRows: testsetData, + testcaseRows: flatRows, skipBlankMessage: true, }) }, diff --git a/web/packages/agenta-playground/src/state/helpers/testcaseRowNormalization.ts b/web/packages/agenta-playground/src/state/helpers/testcaseRowNormalization.ts new file mode 100644 index 0000000000..1d1e841983 --- /dev/null +++ b/web/packages/agenta-playground/src/state/helpers/testcaseRowNormalization.ts @@ -0,0 +1,78 @@ +import {isRecord} from "@agenta/entities/shared" +import {SYSTEM_FIELDS} from "@agenta/entities/testcase" + +/** + * Fields that hint at a wrapper object (i.e. system fields minus non-hint keys) + * Used to detect whether a row object is a raw testcase wrapper or actual data. + */ +const NON_HINT_FIELDS = new Set(["id", "key", "__isSkeleton", "__isNew", "__dedup_id__"]) +const WRAPPER_HINT_FIELDS = new Set([...SYSTEM_FIELDS].filter((f) => !NON_HINT_FIELDS.has(f))) + +const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i + +export interface CanonicalTestcaseRow { + id?: string + data: Record +} + +const looksLikeTestcaseId = (value: unknown): boolean => + typeof value === "string" && + (UUID_PATTERN.test(value) || + value.startsWith("new-") || + value.startsWith("local-") || + value.startsWith("testcase-")) + +const unwrapTestcaseObject = (input: Record): Record => { + let current: Record = input + const seen = new Set>() + + while (isRecord(current.testcase) && !seen.has(current)) { + seen.add(current) + current = current.testcase + } + + return current +} + +const hasWrappedDataShape = (row: Record): boolean => { + if (!isRecord(row.data)) return false + + const keys = Object.keys(row) + if (keys.length === 1 && keys[0] === "data") return true + + if (keys.some((key) => WRAPPER_HINT_FIELDS.has(key))) return true + + if (!keys.every((key) => key === "id" || key === "data" || SYSTEM_FIELDS.has(key))) { + return false + } + + if (keys.length === 2 && keys.includes("id") && keys.includes("data")) { + return looksLikeTestcaseId(row.id) + } + + return true +} + +export const extractCanonicalTestcaseRow = (row: Record): CanonicalTestcaseRow => { + const unwrapped = unwrapTestcaseObject(row) + const id = typeof unwrapped.id === "string" ? unwrapped.id : undefined + + const sourceData = hasWrappedDataShape(unwrapped) + ? (unwrapped.data as Record) + : unwrapped + + const data: Record = {} + for (const [key, value] of Object.entries(sourceData)) { + if (!SYSTEM_FIELDS.has(key)) { + data[key] = value + } + } + + return {id, data} +} + +export const normalizeTestcaseRowsForLoad = ( + rows: Record[], +): CanonicalTestcaseRow[] => { + return rows.map((row) => extractCanonicalTestcaseRow(row)) +} diff --git a/web/packages/agenta-playground/src/state/index.ts b/web/packages/agenta-playground/src/state/index.ts index 1e8a7f38d4..5b039b1abc 100644 --- a/web/packages/agenta-playground/src/state/index.ts +++ b/web/packages/agenta-playground/src/state/index.ts @@ -315,6 +315,11 @@ export { type ExtractChatMessagesParams, } from "./helpers/extractAndLoadChatMessages" export {loadTestsetNormalizedMutationAtom} from "./helpers/loadTestsetNormalizedMutation" +export { + extractCanonicalTestcaseRow, + normalizeTestcaseRowsForLoad, + type CanonicalTestcaseRow, +} from "./helpers/testcaseRowNormalization" // Chat ↔ entity sync (writes chat messages back to testcase drafts) export {syncChatMessagesToEntityAtom} from "./helpers/syncChatMessagesToEntity" diff --git a/web/tests/playwright.config.ts b/web/tests/playwright.config.ts index f7f9ddd180..67d35f11f8 100644 --- a/web/tests/playwright.config.ts +++ b/web/tests/playwright.config.ts @@ -5,7 +5,6 @@ import {fileURLToPath} from "url" import {defineConfig} from "@playwright/test" import dotenv from "dotenv" - // Get current directory in ESM const __filename = fileURLToPath(import.meta.url) const __dirname = dirname(__filename) diff --git a/web/tests/playwright/config/testTags.ts b/web/tests/playwright/config/testTags.ts index ae296186d9..62a53fffc8 100644 --- a/web/tests/playwright/config/testTags.ts +++ b/web/tests/playwright/config/testTags.ts @@ -105,9 +105,4 @@ export const createTagString = (type: PlaywrightConfig.TestTagType, value: strin `${TAG_ARGUMENTS[type].prefix}${value}` // Re-export types from the types module for backward compatibility -export type { - TestTagType, - TestTag, - TagArgument, - ProjectFeatureConfig, -} from "./types" +export type {TestTagType, TestTag, TagArgument, ProjectFeatureConfig} from "./types" diff --git a/web/tests/playwright/global-teardown.ts b/web/tests/playwright/global-teardown.ts index 98f71f40c8..4b338a68c1 100644 --- a/web/tests/playwright/global-teardown.ts +++ b/web/tests/playwright/global-teardown.ts @@ -112,9 +112,7 @@ async function deleteEphemeralProject(apiURL: string): Promise { return } - console.log( - `[global-teardown] Deleting ephemeral project: ${projectName} (${projectId})`, - ) + console.log(`[global-teardown] Deleting ephemeral project: ${projectName} (${projectId})`) const statePath = resolve(__dirname, "../state.json") const sessionToken = getSessionToken(statePath) diff --git a/web/tests/playwright/scripts/run-tests.ts b/web/tests/playwright/scripts/run-tests.ts index 4f704b428e..a42a6d4753 100644 --- a/web/tests/playwright/scripts/run-tests.ts +++ b/web/tests/playwright/scripts/run-tests.ts @@ -60,7 +60,9 @@ function parseArgs(args: string[]): ParsedArgs { } // Check if this is a dimension flag - const dimensionMatch = arg.match(/^--?(coverage|lens|path|case|speed|scope|license|cost|plan|role)$/) + const dimensionMatch = arg.match( + /^--?(coverage|lens|path|case|speed|scope|license|cost|plan|role)$/, + ) if (dimensionMatch && i + 1 < args.length) { const dimension = dimensionMatch[1] diff --git a/web/tests/tests/fixtures/base.fixture/apiHelpers/index.ts b/web/tests/tests/fixtures/base.fixture/apiHelpers/index.ts index cfd739b237..337de6fe58 100644 --- a/web/tests/tests/fixtures/base.fixture/apiHelpers/index.ts +++ b/web/tests/tests/fixtures/base.fixture/apiHelpers/index.ts @@ -321,7 +321,9 @@ export const getEvaluationRuns = async (page: Page) => { method: "POST", }) - await page.goto(`${getProjectScopedBasePath(page)}/evaluations`, {waitUntil: "domcontentloaded"}) + await page.goto(`${getProjectScopedBasePath(page)}/evaluations`, { + waitUntil: "domcontentloaded", + }) const evaluationRuns = await evaluationRunsResponse // Fix: Check for .runs array in the response