feat: custom evaluator scaffolding via UI, MCP, and Claude Code skill

Backend:
- Add POST /api/custom-evaluators/scaffold to create boilerplate Python
  file in evaluations/evaluators/custom/ with correct class structure
- Add POST /api/custom-evaluators/register to run the registration logic

Frontend:
- Add "+ Custom Evaluator (Python)" button in evaluators sidebar
- Inline form for name, creates file, shows toast with next steps
- Links to custom evaluator docs

MCP:
- Add scaffold_custom_evaluator and register_custom_evaluator tools

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: radu-mocanu

src/uipath/dev/server/frontend/src/api/eval-client.ts

@@ -150,6 +150,25 @@ export async function updateEvalSetEvaluators(
   });
 }
 
+export async function scaffoldCustomEvaluator(body: {
+  name: string;
+  description?: string;
+}): Promise<{ file_path: string; filename: string; class_name: string; evaluator_id: string }> {
+  return fetchJson(`${BASE}/custom-evaluators/scaffold`, {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify(body),
+  });
+}
+
+export async function registerCustomEvaluator(filename: string): Promise<{ evaluator_id: string; spec_path: string }> {
+  return fetchJson(`${BASE}/custom-evaluators/register`, {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify({ filename }),
+  });
+}
+
 export async function updateLocalEvaluator(
   id: string,
   body: {

src/uipath/dev/server/frontend/src/components/evaluators/EvaluatorsSidebar.tsx

@@ -1,5 +1,8 @@
+import { useState } from "react";
 import { useEvalStore } from "../../store/useEvalStore";
 import { useHashRoute } from "../../hooks/useHashRoute";
+import { useToastStore } from "../../store/useToastStore";
+import { scaffoldCustomEvaluator } from "../../api/eval-client";
 
 const categories: { type: string; label: string; badgeColor: string }[] = [
   { type: "deterministic", label: "Deterministic", badgeColor: "var(--success)" },
@@ -10,6 +13,26 @@ const categories: { type: string; label: string; badgeColor: string }[] = [
 export default function EvaluatorsSidebar() {
   const localEvaluators = useEvalStore((s) => s.localEvaluators);
   const { evaluatorFilter, evaluatorCreateType, navigate } = useHashRoute();
+  const addToast = useToastStore((s) => s.addToast);
+  const [customName, setCustomName] = useState("");
+  const [showCustom, setShowCustom] = useState(false);
+  const [scaffolding, setScaffolding] = useState(false);
+
+  const handleScaffold = async () => {
+    if (!customName.trim()) return;
+    setScaffolding(true);
+    try {
+      const result = await scaffoldCustomEvaluator({ name: customName.trim() });
+      addToast("success", `Created ${result.filename} — implement evaluate() then run: uv run uipath register evaluator ${result.filename}`);
+      setCustomName("");
+      setShowCustom(false);
+    } catch (err: unknown) {
+      const detail = (err as { detail?: string })?.detail;
+      addToast("error", detail ?? "Failed to scaffold evaluator");
+    } finally {
+      setScaffolding(false);
+    }
+  };
 
   // "All" is active when on #/evaluators (no filter, no create, no detail)
   const isAllActive = !evaluatorFilter && !evaluatorCreateType;
@@ -33,6 +56,64 @@ export default function EvaluatorsSidebar() {
         + New Evaluator
       </button>
 
+      {/* Create Custom */}
+      {showCustom ? (
+        <div className="mx-3 mb-1 p-2 rounded border" style={{ borderColor: "var(--border)", background: "var(--bg-secondary)" }}>
+          <input
+            type="text"
+            value={customName}
+            onChange={(e) => setCustomName(e.target.value)}
+            onKeyDown={(e) => { if (e.key === "Enter") handleScaffold(); if (e.key === "Escape") setShowCustom(false); }}
+            placeholder="e.g. DiscountEvaluator"
+            autoFocus
+            className="w-full rounded px-2 py-1 text-[11px] mb-1.5"
+            style={{ background: "var(--bg-primary)", border: "1px solid var(--border)", color: "var(--text-primary)" }}
+          />
+          <div className="flex gap-1.5">
+            <button
+              onClick={handleScaffold}
+              disabled={scaffolding || !customName.trim()}
+              className="flex-1 py-1 text-[10px] font-semibold rounded cursor-pointer disabled:opacity-40"
+              style={{ background: "var(--accent)", color: "var(--bg-primary)", border: "none" }}
+            >
+              {scaffolding ? "Creating..." : "Create"}
+            </button>
+            <button
+              onClick={() => setShowCustom(false)}
+              className="px-2 py-1 text-[10px] rounded cursor-pointer"
+              style={{ color: "var(--text-muted)", background: "transparent", border: "1px solid var(--border)" }}
+            >
+              Cancel
+            </button>
+          </div>
+          <a
+            href="https://uipath.github.io/uipath-python/eval/custom_evaluators/"
+            target="_blank"
+            rel="noopener noreferrer"
+            className="block mt-1.5 text-[10px] no-underline"
+            style={{ color: "var(--accent)" }}
+          >
+            Custom evaluator docs &rarr;
+          </a>
+        </div>
+      ) : (
+        <button
+          onClick={() => setShowCustom(true)}
+          className="mx-3 mb-1 px-3 py-1.5 text-[11px] font-medium rounded border border-dashed transition-colors cursor-pointer"
+          style={{ color: "var(--text-muted)", borderColor: "var(--border)", background: "transparent" }}
+          onMouseEnter={(e) => {
+            e.currentTarget.style.color = "var(--text-primary)";
+            e.currentTarget.style.borderColor = "var(--text-muted)";
+          }}
+          onMouseLeave={(e) => {
+            e.currentTarget.style.color = "var(--text-muted)";
+            e.currentTarget.style.borderColor = "var(--border)";
+          }}
+        >
+          + Custom Evaluator (Python)
+        </button>
+      )}
+
       {/* Categories label */}
       <div className="px-3 pt-3 pb-1 text-[11px] uppercase tracking-widest font-semibold" style={{ color: "var(--text-muted)" }}>
         Categories

src/uipath/dev/server/routes/evaluators.py

@@ -235,6 +235,123 @@ def _discover_local_evaluators() -> list[dict[str, Any]]:
     return results
 
 
+_CUSTOM_EVALUATOR_TEMPLATE = '''\
+from typing import Optional
+
+from uipath.eval.evaluators import (
+    BaseEvaluationCriteria,
+    BaseEvaluator,
+    BaseEvaluatorConfig,
+)
+from uipath.eval.models import (
+    AgentExecution,
+    ErrorEvaluationResult,
+    EvaluationResult,
+    NumericEvaluationResult,
+)
+
+
+class {class_name}Criteria(BaseEvaluationCriteria):
+    """Per-item evaluation criteria for {class_name}."""
+
+    pass  # TODO: add fields like expected_value: str
+
+
+class {class_name}Config(BaseEvaluatorConfig[{class_name}Criteria]):
+    """Configuration for {class_name}."""
+
+    name: str = "{class_name}"
+    default_evaluation_criteria: Optional[{class_name}Criteria] = None
+
+
+class {class_name}(
+    BaseEvaluator[{class_name}Criteria, {class_name}Config, None]
+):
+    """{description}"""
+
+    @classmethod
+    def get_evaluator_id(cls) -> str:
+        return "{class_name}"
+
+    async def evaluate(
+        self,
+        agent_execution: AgentExecution,
+        evaluation_criteria: {class_name}Criteria,
+    ) -> EvaluationResult:
+        try:
+            output = agent_execution.agent_output
+            # TODO: implement evaluation logic
+            score = 0.0
+            return NumericEvaluationResult(score=score, details="Not implemented yet")
+        except Exception as e:
+            return ErrorEvaluationResult(error=str(e))
+'''
+
+
+class ScaffoldCustomEvaluatorBody(BaseModel):
+    """Body for scaffolding a custom evaluator Python file."""
+
+    name: str
+    description: str = ""
+
+
+@router.post("/custom-evaluators/scaffold")
+async def scaffold_custom_evaluator(body: ScaffoldCustomEvaluatorBody) -> dict[str, Any]:
+    """Scaffold a new custom evaluator Python file."""
+    class_name = body.name.replace(" ", "").replace("-", "").replace("_", "")
+    if not class_name.endswith("Evaluator"):
+        class_name += "Evaluator"
+
+    # snake_case filename
+    snake = ""
+    for i, ch in enumerate(class_name):
+        if ch.isupper() and i > 0 and not class_name[i - 1].isupper():
+            snake += "_"
+        snake += ch.lower()
+    filename = f"{snake}.py"
+
+    custom_dir = Path.cwd() / "evaluations" / "evaluators" / "custom"
+    custom_dir.mkdir(parents=True, exist_ok=True)
+
+    filepath = custom_dir / filename
+    if filepath.exists():
+        raise HTTPException(status_code=409, detail=f"File already exists: {filepath}")
+
+    content = _CUSTOM_EVALUATOR_TEMPLATE.format(
+        class_name=class_name,
+        description=body.description or f"Custom evaluator: {body.name}",
+    )
+    filepath.write_text(content, encoding="utf-8")
+
+    return {
+        "file_path": str(filepath),
+        "filename": filename,
+        "class_name": class_name,
+        "evaluator_id": class_name,
+    }
+
+
+class RegisterCustomEvaluatorBody(BaseModel):
+    """Body for registering a custom evaluator."""
+
+    filename: str
+
+
+@router.post("/custom-evaluators/register")
+async def register_custom_evaluator(body: RegisterCustomEvaluatorBody) -> dict[str, Any]:
+    """Register a custom evaluator by running the registration logic."""
+    try:
+        from uipath.eval.evaluators.registration import register_evaluator
+
+        evaluator_id, spec_path = register_evaluator(body.filename)
+        return {
+            "evaluator_id": evaluator_id,
+            "spec_path": spec_path,
+        }
+    except Exception as exc:
+        raise HTTPException(status_code=400, detail=str(exc)) from None
+
+
 class CreateEvaluatorBody(BaseModel):
     """Body for creating a new local evaluator JSON file."""
 

…rver/static/assets/ChatPanel-BNRio83n.js …rver/static/assets/ChatPanel-DJtePF36.jssrc/uipath/dev/server/static/assets/ChatPanel-BNRio83n.js renamed to src/uipath/dev/server/static/assets/ChatPanel-DJtePF36.js src/uipath/dev/server/static/assets/ChatPanel-BNRio83n.js renamed to src/uipath/dev/server/static/assets/ChatPanel-DJtePF36.js

@@ -5,11 +5,11 @@
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <title>UiPath Developer Console</title>
     <link rel="icon" type="image/x-icon" href="/favicon.ico" />
-    <script type="module" crossorigin src="/assets/index-Cd97ab0I.js"></script>
+    <script type="module" crossorigin src="/assets/index-C0mGt6tw.js"></script>
     <link rel="modulepreload" crossorigin href="/assets/vendor-react-VzyiTEsu.js">
     <link rel="modulepreload" crossorigin href="/assets/vendor-reactflow-B_2yZyR4.js">
     <link rel="stylesheet" crossorigin href="/assets/vendor-reactflow-B5DZHykP.css">
-    <link rel="stylesheet" crossorigin href="/assets/index-EAfwuOZs.css">
+    <link rel="stylesheet" crossorigin href="/assets/index-CIbY4UMG.css">
   </head>
   <body>
     <div id="root"></div>