Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 14 additions & 7 deletions api/oss/src/core/secrets/dtos.py
Original file line number Diff line number Diff line change
Expand Up @@ -84,31 +84,38 @@ def validate_secret_data_based_on_kind(cls, values: Dict[str, Any]):
data = data.model_dump()
values["data"] = data

standard_provider_kinds = {provider.value for provider in StandardProviderKind}
custom_provider_kinds = {provider.value for provider in CustomProviderKind}

if kind == SecretKind.PROVIDER_KEY.value:
if not isinstance(data, dict):
raise ValueError(
"The provided request secret dto is not a valid type for StandardProviderDTO"
)
if not isinstance(data["provider"], dict) or "key" not in data["provider"]:
provider = data.get("provider")
if not isinstance(provider, dict) or "key" not in provider:
raise ValueError(
"The provided request secret dto is missing required fields for StandardProviderSettingsDTO"
)
if data["kind"] not in StandardProviderKind.__members__.values():
# Accept the legacy provider slug on input, but persist the canonical value.
if data.get("kind") == StandardProviderKind.MISTRALAI.value:
data["kind"] = StandardProviderKind.MISTRAL.value
if data.get("kind") not in standard_provider_kinds:
raise ValueError(
"The provided kind in data is not a valid StandardProviderKind enum"
)

elif kind == SecretKind.CUSTOM_PROVIDER.value:
if not isinstance(data, dict):
raise ValueError(
"The provided request secret dto is not a valid type for CustomProviderDTO"
)
# Fix inconsistent API naming - Users might enter 'togetherai' but the API requires 'together_ai'
# This ensures compatibility with LiteLLM which requires the provider in "together_ai" format
if data.get("kind", "") == "togetherai":
data["kind"] = "together_ai"

if not isinstance(data, dict):
raise ValueError(
"The provided request secret dto is not a valid type for CustomProviderDTO"
)
if data["kind"] not in CustomProviderKind.__members__.values():
if data.get("kind") not in custom_provider_kinds:
raise ValueError(
"The provided kind in data is not a valid CustomProviderKind enum"
)
Expand Down
33 changes: 31 additions & 2 deletions api/oss/src/core/secrets/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,35 @@
from oss.src.models.api.evaluation_model import LMProvidersEnum


_LEGACY_SYSTEM_ENV_NAMES = {
LMProvidersEnum.mistral.value: ("MISTRALAI_API_KEY",),
}

_PROVIDER_ENV_ALIASES = {
"mistralai": LMProvidersEnum.mistral.value,
}


def _get_system_env_secret(secret_name: str) -> str | None:
for env_name in (secret_name, *_LEGACY_SYSTEM_ENV_NAMES.get(secret_name, ())):
env_var = os.getenv(env_name)
if env_var:
return env_var

return None


def _provider_slug_to_env_var(provider_slug: str) -> str:
if not provider_slug:
return ""

canonical_provider = LMProvidersEnum.__members__.get(provider_slug.replace("_", ""))
if canonical_provider:
return canonical_provider.value

return _PROVIDER_ENV_ALIASES.get(provider_slug, f"{provider_slug.upper()}_API_KEY")


async def get_system_llm_providers_secrets() -> Dict[str, Any]:
"""
Fetches LLM providers secrets from system environment variables.
Expand All @@ -15,7 +44,7 @@ async def get_system_llm_providers_secrets() -> Dict[str, Any]:
secrets = {}
for llm_provider in LMProvidersEnum:
secret_name = llm_provider.value
env_var = os.getenv(secret_name)
env_var = _get_system_env_secret(secret_name)
if env_var:
secrets[secret_name] = env_var

Expand Down Expand Up @@ -46,7 +75,7 @@ async def get_user_llm_providers_secrets(project_id: str) -> Dict[str, Any]:
for secret in secrets:
kind = secret["data"].get("kind")
provider_slug = kind.value if kind else ""
secret_name = f"{provider_slug.upper()}_API_KEY"
secret_name = _provider_slug_to_env_var(provider_slug)
if provider_slug:
provider = secret["data"].get("provider")
readable_secrets[secret_name] = provider.get("key") if provider else None
Expand Down
1 change: 0 additions & 1 deletion api/oss/src/models/api/evaluation_model.py
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,6 @@ class LLMRunRateLimit(BaseModel):
class LMProvidersEnum(str, Enum):
openai = "OPENAI_API_KEY"
mistral = "MISTRAL_API_KEY"
mistralai = "MISTRALAI_API_KEY"
cohere = "COHERE_API_KEY"
anthropic = "ANTHROPIC_API_KEY"
anyscale = "ANYSCALE_API_KEY"
Expand Down
1 change: 0 additions & 1 deletion api/oss/tests/legacy/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,6 @@ def sample_testset_endpoint_json():
API_KEYS_MAPPING = {
"OPENAI_API_KEY": "openai",
"MISTRAL_API_KEY": "mistral",
"MISTRALAI_API_KEY": "mistralai",
"COHERE_API_KEY": "cohere",
"ANTHROPIC_API_KEY": "anthropic",
"ANYSCALE_API_KEY": "anyscale",
Expand Down
60 changes: 60 additions & 0 deletions api/oss/tests/pytest/unit/secrets/test_dtos.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
import pytest
from pydantic import ValidationError

from oss.src.core.secrets.dtos import CreateSecretDTO, UpdateSecretDTO


def test_create_secret_normalizes_mistralai_standard_provider_payload():
payload = {
"header": {"name": "Mistral AI", "description": ""},
"secret": {
"kind": "provider_key",
"data": {
"kind": "mistralai",
"provider": {
"key": "TEST_KEY",
},
},
},
}

secret = CreateSecretDTO.model_validate(payload)

assert secret.secret.data.kind == "mistral"
assert secret.secret.data.provider.key == "TEST_KEY"


def test_update_secret_normalizes_mistralai_standard_provider_payload():
payload = {
"secret": {
"kind": "provider_key",
"data": {
"kind": "mistralai",
"provider": {
"key": "TEST_KEY",
},
},
},
}

secret = UpdateSecretDTO.model_validate(payload)

assert secret.secret.data.kind == "mistral"
assert secret.secret.data.provider.key == "TEST_KEY"


def test_create_secret_rejects_missing_standard_provider_kind():
payload = {
"header": {"name": "Mistral AI", "description": ""},
"secret": {
"kind": "provider_key",
"data": {
"provider": {
"key": "TEST_KEY",
},
},
},
}

with pytest.raises(ValidationError, match="StandardProviderKind"):
CreateSecretDTO.model_validate(payload)
64 changes: 64 additions & 0 deletions api/oss/tests/pytest/unit/secrets/test_utils.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
from types import SimpleNamespace

import pytest

from oss.src.core.secrets.enums import StandardProviderKind
from oss.src.core.secrets.utils import (
get_system_llm_providers_secrets,
get_user_llm_providers_secrets,
)


class _FakeVaultService:
def __init__(self, *_args, **_kwargs):
pass

async def list_secrets(self, project_id):
del project_id
return [
SimpleNamespace(
kind="provider_key",
model_dump=lambda include=None: {
"data": {
"kind": StandardProviderKind.MISTRALAI,
"provider": {"key": "mistral-key"},
}
},
),
SimpleNamespace(
kind="provider_key",
model_dump=lambda include=None: {
"data": {
"kind": StandardProviderKind.TOGETHERAI,
"provider": {"key": "together-key"},
}
},
),
]


@pytest.mark.asyncio
async def test_get_user_llm_providers_secrets_normalizes_legacy_provider_slugs(
monkeypatch,
):
monkeypatch.setattr("oss.src.core.secrets.utils.VaultService", _FakeVaultService)

secrets = await get_user_llm_providers_secrets(
"00000000-0000-0000-0000-000000000000"
)

assert secrets["MISTRAL_API_KEY"] == "mistral-key"
assert "MISTRALAI_API_KEY" not in secrets
assert secrets["TOGETHERAI_API_KEY"] == "together-key"
assert "TOGETHER_AI_API_KEY" not in secrets


@pytest.mark.asyncio
async def test_get_system_llm_providers_secrets_reads_legacy_mistralai_env(monkeypatch):
monkeypatch.delenv("MISTRAL_API_KEY", raising=False)
monkeypatch.setenv("MISTRALAI_API_KEY", "legacy-mistral-key")

secrets = await get_system_llm_providers_secrets()

assert secrets["MISTRAL_API_KEY"] == "legacy-mistral-key"
assert "MISTRALAI_API_KEY" not in secrets
26 changes: 18 additions & 8 deletions sdk/agenta/sdk/managers/secrets.py
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🟡 Custom provider kind matching broken by asymmetric normalization of 'mistralai' alias

In both get_provider_settings and get_provider_settings_from_workflow, request_provider_kind is normalized via _normalize_provider_kind which maps "mistralai""mistral" (via _PROVIDER_KIND_ALIASES at sdk/agenta/sdk/managers/secrets.py:16-18). However, for custom providers, secret_provider_kind is still computed with only .lower().replace(" ", "") — which does NOT apply the alias, leaving it as "mistralai". This causes the comparison request_provider_kind == secret_provider_kind ("mistral" != "mistralai") to fail, so credentials are never extracted and the function returns None.

Before this PR, request_provider_kind used re.sub(r"[\s-]+", "", provider.lower()) which did NOT apply any alias, so both sides would be "mistralai" and would match. This is a regression for any custom provider configured with CustomProviderKind.MISTRALAI (a valid enum value in api/oss/src/core/secrets/enums.py:40). The same issue is duplicated in get_provider_settings_from_workflow at lines 371-372.

(Refers to lines 227-228)

Open in Devin Review

Was this helpful? React with 👍 or 👎 to provide feedback.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🟡 Same asymmetric custom provider kind normalization in get_provider_settings_from_workflow

Duplicate of the same bug as in get_provider_settings: secret_provider_kind for custom providers uses .lower().replace(" ", "") instead of _normalize_provider_kind, so the "mistralai""mistral" alias is not applied on the secret side while it IS applied on the request side (sdk/agenta/sdk/managers/secrets.py:349). This causes a mismatch and credentials are never matched for custom providers with kind "mistralai".

(Refers to lines 371-372)

Open in Devin Review

Was this helpful? React with 👍 or 👎 to provide feedback.

Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,17 @@
log = get_module_logger(__name__)


_PROVIDER_KIND_ALIASES = {
"mistralai": "mistral",
}


class SecretsManager:
@staticmethod
def _normalize_provider_kind(provider_kind: str) -> str:
normalized = re.sub(r"[\s-]+", "", provider_kind.lower())
return _PROVIDER_KIND_ALIASES.get(normalized, normalized)

@staticmethod
def get_from_route(scope: str = "all") -> Optional[List[Dict[str, Any]]]:
context = RoutingContext.get()
Expand Down Expand Up @@ -192,9 +202,7 @@ def get_provider_settings(model: str, scope: str = "all") -> Optional[Dict]:

# STEP 3: initialize provider settings and simplify provider name
provider_settings = dict(model=compatible_provider_model)
request_provider_kind = re.sub(
r"[\s-]+", "", provider.lower()
) # normalizing other special characters too (azure-openai)
request_provider_kind = SecretsManager._normalize_provider_kind(provider)

# STEP 4: get credentials for model
for secret in secrets:
Expand All @@ -204,7 +212,9 @@ def get_provider_settings(model: str, scope: str = "all") -> Optional[Dict]:
# i). Extract API key if present
# (for standard models -- openai/anthropic/gemini, etc)
if secret.get("kind") == "provider_key":
secret_provider_kind = secret_data.get("kind", "")
secret_provider_kind = SecretsManager._normalize_provider_kind(
secret_data.get("kind", "")
)

if request_provider_kind == secret_provider_kind:
if "key" in provider_info:
Expand Down Expand Up @@ -336,9 +346,7 @@ def get_provider_settings_from_workflow(

# STEP 3: initialize provider settings and simplify provider name
provider_settings = dict(model=compatible_provider_model)
request_provider_kind = re.sub(
r"[\s-]+", "", provider.lower()
) # normalizing other special characters too (azure-openai)
request_provider_kind = SecretsManager._normalize_provider_kind(provider)

# STEP 4: get credentials for model
for secret in secrets:
Expand All @@ -348,7 +356,9 @@ def get_provider_settings_from_workflow(
# i). Extract API key if present
# (for standard models -- openai/anthropic/gemini, etc)
if secret.get("kind") == "provider_key":
secret_provider_kind = secret_data.get("kind", "")
secret_provider_kind = SecretsManager._normalize_provider_kind(
secret_data.get("kind", "")
)

if request_provider_kind == secret_provider_kind:
if "key" in provider_info:
Expand Down
3 changes: 2 additions & 1 deletion sdk/agenta/sdk/workflows/runners/daytona.py
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,8 @@ def _get_provider_env_vars(self) -> Dict[str, str]:
"deepinfra": "DEEPINFRA_API_KEY",
"alephalpha": "ALEPHALPHA_API_KEY",
"groq": "GROQ_API_KEY",
"mistralai": "MISTRALAI_API_KEY",
"mistral": "MISTRAL_API_KEY",
"mistralai": "MISTRAL_API_KEY",
"anthropic": "ANTHROPIC_API_KEY",
"perplexityai": "PERPLEXITYAI_API_KEY",
# Secret kind is "together_ai" (underscore) even though the env var is TOGETHERAI_API_KEY
Expand Down
1 change: 0 additions & 1 deletion sdk/oss/tests/legacy/new_tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,6 @@ def sample_testset_endpoint_json():
API_KEYS_MAPPING = {
"OPENAI_API_KEY": "openai",
"MISTRAL_API_KEY": "mistral",
"MISTRALAI_API_KEY": "mistralai",
"COHERE_API_KEY": "cohere",
"ANTHROPIC_API_KEY": "anthropic",
"ANYSCALE_API_KEY": "anyscale",
Expand Down
55 changes: 55 additions & 0 deletions sdk/oss/tests/pytest/unit/test_mistral_provider_aliases.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
from types import SimpleNamespace

from agenta.sdk.contexts.running import RunningContext
from agenta.sdk.managers.secrets import SecretsManager
from agenta.sdk.workflows.runners.daytona import DaytonaRunner


def test_secrets_manager_accepts_mistralai_secret_for_mistral_model(monkeypatch):
monkeypatch.setattr(
SecretsManager,
"get_from_route",
staticmethod(
lambda scope="all": [
{
"kind": "provider_key",
"data": {
"kind": "mistralai",
"provider": {"key": "TEST_KEY"},
},
}
]
),
)

settings = SecretsManager.get_provider_settings("mistral/mistral-small")

assert settings is not None
assert settings["model"] == "mistral/mistral-small"
assert settings["api_key"] == "TEST_KEY"


def test_daytona_runner_exports_canonical_mistral_env_var(monkeypatch):
monkeypatch.setenv("DAYTONA_API_KEY", "test-daytona-key")
runner = DaytonaRunner()
monkeypatch.setattr(
RunningContext,
"get",
staticmethod(
lambda: SimpleNamespace(
vault_secrets=[
{
"kind": "provider_key",
"data": {
"kind": "mistralai",
"provider": {"key": "TEST_KEY"},
},
}
]
)
),
)

env_vars = runner._get_provider_env_vars()

assert env_vars["MISTRAL_API_KEY"] == "TEST_KEY"
4 changes: 2 additions & 2 deletions web/eslint.config.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -45,12 +45,12 @@ const config = [
"no-restricted-syntax": [
"error",
{
selector: 'ExportNamedDeclaration[source.value=/^@agenta/]',
selector: "ExportNamedDeclaration[source.value=/^@agenta/]",
message:
"Do not re-export from @agenta/* packages. Consumers should import directly from the source package for proper tree-shaking.",
},
{
selector: 'ExportAllDeclaration[source.value=/^@agenta/]',
selector: "ExportAllDeclaration[source.value=/^@agenta/]",
message:
"Do not re-export from @agenta/* packages. Consumers should import directly from the source package for proper tree-shaking.",
},
Expand Down
Loading
Loading