Skip to content

test: add e2e tests for annotation queue flows #4568

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
GanJiaKouN16 wants to merge 8 commits into
base: main
Choose a base branch
from
Open

test: add e2e tests for annotation queue flows #4568

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
6eaecdd
feat: add reset password functionality to workspace UI
GanJiaKouN16 Jun 6, 2026
445fc3f
fix: add shouldIgnoreRowClick guard to unprotected tables
GanJiaKouN16 Jun 6, 2026
585c6ad
fix: surface provider error messages in evaluation table
GanJiaKouN16 Jun 6, 2026
c9d112d
fix: use explicit env var mapping for Together AI provider in vault m…
GanJiaKouN16 Jun 6, 2026
01a6e6d
fix: link evaluator button to playground instead of registry in trace…
GanJiaKouN16 Jun 6, 2026
1706e43
feat: migrate email from SendGrid to SMTP with SendGrid fallback
GanJiaKouN16 Jun 6, 2026
ad6e140
fix: show correct breadcrumb label for SDK evaluations
GanJiaKouN16 Jun 6, 2026
cd01041
test: add e2e tests for annotation queue flows
GanJiaKouN16 Jun 6, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 0 additions & 4 deletions api/ee/src/services/db_manager_ee.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@
import uuid
from datetime import datetime, timezone

import sendgrid
from fastapi import HTTPException

from sqlalchemy import delete, func, update
Expand Down Expand Up @@ -60,9 +59,6 @@
from oss.src.utils.env import env


# Initialize sendgrid api client
sg = sendgrid.SendGridAPIClient(api_key=env.sendgrid.api_key)

log = get_module_logger(__name__)


Expand Down
12 changes: 8 additions & 4 deletions api/ee/src/services/organization_service.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,8 +104,8 @@ async def send_invitation_email(
f"&project_id={project_param}"
)

# If Sendgrid is not configured, return the link for manual sharing (URL-based invitation)
if not env.sendgrid.enabled:
# If email is not configured, return the link for manual sharing (URL-based invitation)
if not env.smtp.enabled and not env.sendgrid.enabled:
return invite_link

html_content = html_template.format(
Expand All @@ -118,8 +118,10 @@ async def send_invitation_email(
),
)

from_address = env.smtp.from_address or env.sendgrid.from_address or "account@hello.agenta.ai"

await email_service.send_email(
from_email="account@hello.agenta.ai",
from_email=from_address,
to_email=email,
subject=f"{user.username} invited you to join {organization.name}",
html_content=html_content,
Expand Down Expand Up @@ -152,10 +154,12 @@ async def notify_org_admin_invitation(workspace: WorkspaceDB, user: UserDB) -> b
),
)

from_address = env.smtp.from_address or env.sendgrid.from_address or "account@hello.agenta.ai"

Comment on lines +157 to +158
Copy link
Copy Markdown

@coderabbitai coderabbitai Bot Jun 6, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟠 Major | ⚡ Quick win

Reconsider hardcoded default sender address.

Lines 157-158 use the same problematic hardcoded default sender "account@hello.agenta.ai". This has the same issues as in send_invitation_email: SPF/DKIM failures, unmonitored replies, and inconsistency with the OSS version.

Consider raising an error when no sender is configured.

🔧 Proposed fix 
-    from_address = env.smtp.from_address or env.sendgrid.from_address or "account@hello.agenta.ai"
-
+    from_address = env.smtp.from_address or env.sendgrid.from_address
+    if not from_address:
+        raise ValueError("Email requires a sender email address (SMTP_FROM_ADDRESS or SENDGRID_FROM_ADDRESS)")
+
     workspace_admins = await db_manager_ee.get_workspace_administrators(workspace)

workspace_admins = await db_manager_ee.get_workspace_administrators(workspace)
for workspace_admin in workspace_admins:
await email_service.send_email(
from_email="account@hello.agenta.ai",
from_email=from_address,
to_email=workspace_admin.email,
subject=f"New Member Joined {organization.name}",
html_content=html_content,
Expand Down
86 changes: 65 additions & 21 deletions api/oss/src/services/email_service.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import os

import sendgrid
from sendgrid.helpers.mail import Mail
import smtplib
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText

from fastapi import HTTPException

Expand All @@ -10,16 +10,25 @@

log = get_logger(__name__)

# Initialize SendGrid only if enabled
if env.sendgrid.enabled:
sg = sendgrid.SendGridAPIClient(api_key=env.sendgrid.api_key)
log.info("✓ SendGrid enabled")
# Determine which email backend to use (SMTP > SendGrid > no-op)
_USE_SMTP = env.smtp.enabled
_USE_SENDGRID = not _USE_SMTP and env.sendgrid.enabled

if _USE_SMTP:
log.info(
"✓ Email enabled via SMTP (%s:%s)", env.smtp.host, env.smtp.port
)
elif _USE_SENDGRID:
import sendgrid

_sg = sendgrid.SendGridAPIClient(api_key=env.sendgrid.api_key)
log.info("✓ Email enabled via SendGrid (legacy)")
else:
sg = None
_sg = None
if env.sendgrid.api_key and not env.sendgrid.from_address:
log.warn("✗ SendGrid disabled: missing sender email address")
log.warn("✗ Email disabled: missing sender email address")
else:
log.warn("✗ SendGrid disabled")
log.warn("✗ Email disabled")


def read_email_template(template_file_path):
Expand All @@ -35,6 +44,46 @@ def read_email_template(template_file_path):
return template_file.read()


def _send_via_smtp(to_email: str, subject: str, html_content: str, from_email: str) -> None:
"""Send email using SMTP."""
msg = MIMEMultipart("alternative")
msg["Subject"] = subject
msg["From"] = from_email
msg["To"] = to_email
msg.attach(MIMEText(html_content, "html"))

smtp_host = env.smtp.host
smtp_port = env.smtp.port

if env.smtp.use_tls:
server = smtplib.SMTP(smtp_host, smtp_port)
server.ehlo()
server.starttls()
server.ehlo()
else:
server = smtplib.SMTP(smtp_host, smtp_port)

try:
if env.smtp.username and env.smtp.password:
server.login(env.smtp.username, env.smtp.password)
server.sendmail(from_email, [to_email], msg.as_string())
finally:
server.quit()

Comment on lines +47 to +72
Copy link
Copy Markdown

@coderabbitai coderabbitai Bot Jun 6, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🔴 Critical | ⚡ Quick win

Add timeout to SMTP connection to prevent hangs.

The SMTP connection lacks a timeout parameter. If the SMTP server is unresponsive, the application will hang indefinitely on the request thread, degrading availability.

🔧 Proposed fix to add timeout 
-    if env.smtp.use_tls:
-        server = smtplib.SMTP(smtp_host, smtp_port)
+    smtp_timeout = 30  # seconds
+    if env.smtp.use_tls:
+        server = smtplib.SMTP(smtp_host, smtp_port, timeout=smtp_timeout)
         server.ehlo()
         server.starttls()
         server.ehlo()
     else:
-        server = smtplib.SMTP(smtp_host, smtp_port)
+        server = smtplib.SMTP(smtp_host, smtp_port, timeout=smtp_timeout)


def _send_via_sendgrid(to_email: str, subject: str, html_content: str, from_email: str) -> None:
"""Send email using SendGrid (legacy fallback)."""
from sendgrid.helpers.mail import Mail

message = Mail(
from_email=from_email,
to_emails=to_email,
subject=subject,
html_content=html_content,
)
_sg.send(message)


async def send_email(
to_email: str, subject: str, html_content: str, from_email: str
) -> bool:
Expand All @@ -54,20 +103,15 @@ async def send_email(
HTTPException: If there is an error sending the email.
"""

# No-op if SendGrid is disabled
if not env.sendgrid.enabled:
log.info(f"[SENDGRID] Email disabled - would send '{subject}' to {to_email}")
if not _USE_SMTP and not _USE_SENDGRID:
log.info(f"[EMAIL] Email disabled - would send '{subject}' to {to_email}")
return True

message = Mail(
from_email=from_email,
to_emails=to_email,
subject=subject,
html_content=html_content,
)

try:
sg.send(message)
if _USE_SMTP:
_send_via_smtp(to_email, subject, html_content, from_email)
else:
_send_via_sendgrid(to_email, subject, html_content, from_email)
return True
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
Comment on lines 116 to 117
Copy link
Copy Markdown

@coderabbitai coderabbitai Bot Jun 6, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟠 Major | ⚡ Quick win

Refine exception handling to preserve error context.

Catching all exceptions with except Exception as e and converting to HTTPException(status_code=500, detail=str(e)) loses important error context and makes debugging harder. SMTP and SendGrid errors have different root causes (network timeouts, authentication failures, rate limits, invalid recipients, etc.) and should be handled with more granularity.

♻️ Suggested improvement 
     try:
         if _USE_SMTP:
             _send_via_smtp(to_email, subject, html_content, from_email)
         else:
             _send_via_sendgrid(to_email, subject, html_content, from_email)
         return True
-    except Exception as e:
-        raise HTTPException(status_code=500, detail=str(e))
+    except smtplib.SMTPException as e:
+        log.error(f"SMTP error sending email to {to_email}: {e}", exc_info=True)
+        raise HTTPException(status_code=500, detail=f"Failed to send email via SMTP: {type(e).__name__}")
+    except Exception as e:
+        log.error(f"Unexpected error sending email to {to_email}: {e}", exc_info=True)
+        raise HTTPException(status_code=500, detail="Failed to send email")

11 changes: 6 additions & 5 deletions api/oss/src/services/organization_service.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -154,8 +154,8 @@ async def send_invitation_email(
f"&project_id={project_param}"
)

# If Sendgrid is not configured, return the link for manual sharing (URL-based invitation)
if not env.sendgrid.enabled:
# If email is not configured, return the link for manual sharing (URL-based invitation)
if not env.smtp.enabled and not env.sendgrid.enabled:
return invite_link

html_template = email_service.read_email_template("./templates/send_email.html")
Expand All @@ -169,11 +169,12 @@ async def send_invitation_email(
),
)

if not env.sendgrid.from_address:
raise ValueError("Sendgrid requires a sender email address to work.")
from_address = env.smtp.from_address or env.sendgrid.from_address
if not from_address:
raise ValueError("Email requires a sender email address to work.")

await email_service.send_email(
from_email=env.sendgrid.from_address,
from_email=from_address,
to_email=email,
subject=f"{user.username} invited you to join their organization",
html_content=html_content,
Expand Down
9 changes: 5 additions & 4 deletions api/oss/src/services/user_service.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ async def generate_user_password_reset_link(user_id: str, admin_user_id: str):
email=user.email,
)

if not env.sendgrid.api_key:
if not env.smtp.enabled and not env.sendgrid.enabled:
return password_reset_link

html_template = email_service.read_email_template("./templates/send_email.html")
Expand All @@ -159,11 +159,12 @@ async def generate_user_password_reset_link(user_id: str, admin_user_id: str):
call_to_action=f"""<p>Click the link below to reset your password:</p><br><a href="{password_reset_link}">Reset Password</a>""",
)

if not env.sendgrid.from_address:
raise ValueError("Sendgrid requires a sender email address to work.")
from_address = env.smtp.from_address or env.sendgrid.from_address
if not from_address:
raise ValueError("Email requires a sender email address to work.")

await email_service.send_email(
from_email=env.sendgrid.from_address,
from_email=from_address,
to_email=user.email,
subject=f"{admin_user.username} requested a password reset for you in their workspace",
html_content=html_content,
Expand Down
45 changes: 34 additions & 11 deletions api/oss/src/utils/env.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -922,12 +922,40 @@ def enabled(self) -> bool:


# ---------------------------------------------------------------------------
# sendgrid
# smtp
# ---------------------------------------------------------------------------


class SmtpConfig(BaseModel):
"""SMTP Email configuration"""

host: str | None = os.getenv("SMTP_HOST")
port: int = int(os.getenv("SMTP_PORT", "587"))
Copy link
Copy Markdown

@coderabbitai coderabbitai Bot Jun 6, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟡 Minor | ⚡ Quick win

Guard against ValueError if SMTP_PORT contains non-numeric input.

If SMTP_PORT is set to a non-numeric string, int(os.getenv(...)) will raise ValueError and crash the application at startup. Consider wrapping in a try-except or using a helper that validates and falls back to the default.

🛡️ Proposed fix with validation 
-    port: int = int(os.getenv("SMTP_PORT", "587"))
+    port: int = (
+        int(port_str)
+        if (port_str := os.getenv("SMTP_PORT", "587")).isdigit()
+        else 587
+    )
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
port: int = int(os.getenv("SMTP_PORT", "587"))
port: int = (
int(port_str)
if (port_str := os.getenv("SMTP_PORT", "587")).isdigit()
else 587
)

username: str | None = os.getenv("SMTP_USERNAME")
password: str | None = os.getenv("SMTP_PASSWORD")
from_address: str | None = (
os.getenv("SMTP_FROM_ADDRESS")
or os.getenv("SENDGRID_FROM_ADDRESS")
or os.getenv("AGENTA_AUTHN_EMAIL_FROM")
or os.getenv("AGENTA_SEND_EMAIL_FROM_ADDRESS")
)
use_tls: bool = os.getenv("SMTP_USE_TLS", "true").lower() in ("true", "1", "yes")

model_config = ConfigDict(extra="ignore")

@property
def enabled(self) -> bool:
"""SMTP enabled if host and from address are present"""
return bool(self.host and self.from_address)


# ---------------------------------------------------------------------------
# sendgrid (legacy — kept for backwards compatibility)
# ---------------------------------------------------------------------------


class SendgridConfig(BaseModel):
"""SendGrid Email configuration"""
"""SendGrid Email configuration (legacy)"""

api_key: str | None = os.getenv("SENDGRID_API_KEY")
from_address: str | None = (
Expand Down Expand Up @@ -1037,15 +1065,9 @@ def email_method(self) -> str:
if env.agenta.access.email_disabled:
return ""

sendgrid_enabled = bool(
os.getenv("SENDGRID_API_KEY")
and (
os.getenv("SENDGRID_FROM_ADDRESS")
or os.getenv("AGENTA_AUTHN_EMAIL_FROM")
or os.getenv("AGENTA_SEND_EMAIL_FROM_ADDRESS")
)
)
return "otp" if sendgrid_enabled else "password"
# SMTP takes priority, then SendGrid fallback
email_configured = env.smtp.enabled or env.sendgrid.enabled
return "otp" if email_configured else "password"

@property
def email_enabled(self) -> bool:
Expand Down Expand Up @@ -1101,6 +1123,7 @@ class EnvironSettings(BaseModel):
posthog: PostHogConfig = PostHogConfig()
redis: RedisConfig = RedisConfig()
sendgrid: SendgridConfig = SendgridConfig()
smtp: SmtpConfig = SmtpConfig()
stripe: StripeConfig = StripeConfig()
supertokens: SuperTokensConfig = SuperTokensConfig()

Expand Down
1 change: 0 additions & 1 deletion api/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@ dependencies = [
"cachetools>=7,<8",
"supertokens-python>=0.31,<0.32",
"openai>=2,<3",
"sendgrid>=6,<7",
"stripe>=15,<16",
"posthog>=7,<8",
"newrelic>=13,<14",
Expand Down
13 changes: 12 additions & 1 deletion docs/docs/self-host/02-configuration.mdx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -254,7 +254,18 @@ This key has no env-var or `env.py` equivalent.
| `REDIS_URI_DURABLE` | `redis.uri_durable` | `redis.uriDurable` |
| `REDIS_URI_VOLATILE` | `redis.uri_volatile` | `redis.uriVolatile` |

## sendgrid
## smtp

| Env var | env.py path | values.yaml path |
|---|---|---|
| `SMTP_HOST` | `smtp.host` | `smtp.host` |
| `SMTP_PORT` | `smtp.port` | `smtp.port` |
| `SMTP_USERNAME` | `smtp.username` | `smtp.username` |
| `SMTP_PASSWORD` | `smtp.password` | `smtp.password` |
| `SMTP_FROM_ADDRESS` | `smtp.from_address` | `smtp.fromAddress` |
| `SMTP_USE_TLS` | `smtp.use_tls` | `smtp.useTls` |

## sendgrid (legacy)

| Env var | env.py path | values.yaml path |
|---|---|---|
Expand Down
12 changes: 11 additions & 1 deletion hosting/docker-compose/ee/env.ee.dev.example
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -215,7 +215,17 @@ POSTHOG_API_KEY=phc_3urGRy5TL1HhaHnRYL0JSHxJxigRVackhphHtozUmdp
# REDIS_URI_VOLATILE=redis://redis-volatile:6379/0

# ================================================================== #
# sendgrid
# smtp
# ================================================================== #
# SMTP_HOST=
# SMTP_PORT=587
# SMTP_USERNAME=
# SMTP_PASSWORD=
# SMTP_FROM_ADDRESS=
# SMTP_USE_TLS=true

# ================================================================== #
# sendgrid (legacy — use SMTP instead)
# ================================================================== #
# SENDGRID_API_KEY=
# SENDGRID_FROM_ADDRESS=
Expand Down
12 changes: 11 additions & 1 deletion hosting/docker-compose/ee/env.ee.gh.example
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -215,7 +215,17 @@ POSTHOG_API_KEY=phc_3urGRy5TL1HhaHnRYL0JSHxJxigRVackhphHtozUmdp
# REDIS_URI_VOLATILE=redis://redis-volatile:6379/0

# ================================================================== #
# sendgrid
# smtp
# ================================================================== #
# SMTP_HOST=
# SMTP_PORT=587
# SMTP_USERNAME=
# SMTP_PASSWORD=
# SMTP_FROM_ADDRESS=
# SMTP_USE_TLS=true

# ================================================================== #
# sendgrid (legacy — use SMTP instead)
# ================================================================== #
# SENDGRID_API_KEY=
# SENDGRID_FROM_ADDRESS=
Expand Down
12 changes: 11 additions & 1 deletion hosting/docker-compose/oss/env.oss.dev.example
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -215,7 +215,17 @@ POSTHOG_API_KEY=phc_hmVSxIjTW1REBHXgj2aw4HW9X6CXb6FzerBgP9XenC7
# REDIS_URI_VOLATILE=redis://redis-volatile:6379/0

# ================================================================== #
# sendgrid
# smtp
# ================================================================== #
# SMTP_HOST=
# SMTP_PORT=587
# SMTP_USERNAME=
# SMTP_PASSWORD=
# SMTP_FROM_ADDRESS=
# SMTP_USE_TLS=true

# ================================================================== #
# sendgrid (legacy — use SMTP instead)
# ================================================================== #
# SENDGRID_API_KEY=
# SENDGRID_FROM_ADDRESS=
Expand Down
12 changes: 11 additions & 1 deletion hosting/docker-compose/oss/env.oss.gh.example
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -215,7 +215,17 @@ POSTHOG_API_KEY=phc_hmVSxIjTW1REBHXgj2aw4HW9X6CXb6FzerBgP9XenC7
# REDIS_URI_VOLATILE=redis://redis-volatile:6379/0

# ================================================================== #
# sendgrid
# smtp
# ================================================================== #
# SMTP_HOST=
# SMTP_PORT=587
# SMTP_USERNAME=
# SMTP_PASSWORD=
# SMTP_FROM_ADDRESS=
# SMTP_USE_TLS=true

# ================================================================== #
# sendgrid (legacy — use SMTP instead)
# ================================================================== #
# SENDGRID_API_KEY=
# SENDGRID_FROM_ADDRESS=
Expand Down
Loading