diff --git a/.github/workflows/ui-playwright.yaml b/.github/workflows/ui-playwright.yaml new file mode 100644 index 000000000..90b1db9e3 --- /dev/null +++ b/.github/workflows/ui-playwright.yaml @@ -0,0 +1,56 @@ +name: UI Playwright E2E + +# Page-level browser E2E for the UI against a mocked backend. +# Kept separate from CI so the main workflow does not need paths-filter or +# conditional steps. No cluster required — the stub backend is booted by Playwright. + +on: + push: + branches: [main, "release/**"] + paths: + - "ui/**" + pull_request: + branches: [main, "release/**"] + paths: + - "ui/**" + workflow_dispatch: + +jobs: + playwright: + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version: "24" + cache: "npm" + cache-dependency-path: ui/package-lock.json + + - name: Install dependencies + working-directory: ./ui + env: + CYPRESS_INSTALL_BINARY: 0 + run: npm ci + + - name: Install Playwright browser + working-directory: ./ui + run: npx playwright install --with-deps chromium + + - name: Run Playwright tests + working-directory: ./ui + run: npm run test:pw + + - name: Upload Playwright report + if: always() + uses: actions/upload-artifact@v4 + with: + name: playwright-report + path: | + ui/playwright-report + ui/playwright/test-results + retention-days: 14 diff --git a/ui/.gitignore b/ui/.gitignore index 9352a9952..adb14e302 100644 --- a/ui/.gitignore +++ b/ui/.gitignore @@ -42,3 +42,8 @@ next-env.d.ts # storybook /storybook-static + +# playwright +/playwright-report +/playwright/test-results +/playwright/.cache diff --git a/ui/package-lock.json b/ui/package-lock.json index 545766788..788e11f5b 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -55,6 +55,7 @@ "@chromatic-com/storybook": "^5.2.1", "@eslint/eslintrc": "^3.3.3", "@jest/globals": "^30.4.1", + "@playwright/test": "^1.61.0", "@storybook/addon-a11y": "^10.4.6", "@storybook/addon-docs": "^10.4.6", "@storybook/addon-onboarding": "^10.4.6", @@ -4026,6 +4027,22 @@ "url": "https://opencollective.com/pkgr" } }, + "node_modules/@playwright/test": { + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.61.1.tgz", + "integrity": "sha512-8nKv6+0RJSL9FE4jYOEGXnPeM/Hg12qZpmqzZjRh3qM0Y7c3z1mrOTfFLids72RDQYVh9WpLEfR5WdpNX4fkig==", + "devOptional": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.61.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/@polka/url": { "version": "1.0.0-next.29", "resolved": "https://registry.npmjs.org/@polka/url/-/url-1.0.0-next.29.tgz", @@ -16038,13 +16055,13 @@ } }, "node_modules/playwright": { - "version": "1.61.0", - "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.61.0.tgz", - "integrity": "sha512-Z+7BeeqQPRRzklHsVFP4KTGIyMxKUmfeRA4WisM6G3/XW6nwGeX6fX9qYaDa+CiUqpOkb2f6X3nar05R3kSuJQ==", - "dev": true, + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.61.1.tgz", + "integrity": "sha512-DWnY5o3YbLWK4GovuAVwpqL+1VwGNdUGrRr++8j8PtQQzvAVZUIMjKQ90fY689sEJZJBbZVw1rXaOKSTitkzPQ==", + "devOptional": true, "license": "Apache-2.0", "dependencies": { - "playwright-core": "1.61.0" + "playwright-core": "1.61.1" }, "bin": { "playwright": "cli.js" @@ -16057,10 +16074,10 @@ } }, "node_modules/playwright-core": { - "version": "1.61.0", - "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.61.0.tgz", - "integrity": "sha512-caX7TrY3Ml6egyDX0WUcTHDxodl/b51y5wJOdCEA36QviK/s2g081hvmGs8eaE3DWb6NYZQ6BjO/QkNRPenoPA==", - "dev": true, + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.61.1.tgz", + "integrity": "sha512-h7Qlt6m4REp25qvIdvbDtVmD4LqVXfpRxhORv9L0jzETM05p4fuPJ3dKyuSXQxDSbXnmS79HAgi9589lGSpLkg==", + "devOptional": true, "license": "Apache-2.0", "bin": { "playwright-core": "cli.js" @@ -16073,7 +16090,6 @@ "version": "2.3.2", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", - "dev": true, "hasInstallScript": true, "optional": true, "os": [ diff --git a/ui/package.json b/ui/package.json index aeda2e3a4..55751a765 100644 --- a/ui/package.json +++ b/ui/package.json @@ -12,6 +12,9 @@ "test:watch": "jest --watch", "test:e2e:cypress": "cypress run --e2e", "test:e2e": "start-server-and-test dev http://localhost:8001 test:e2e:cypress", + "test:pw": "playwright test", + "test:pw:ui": "playwright test --ui", + "test:pw:debug": "playwright test --debug", "storybook": "storybook dev -p 6006", "build-storybook": "storybook build", "chromatic": "chromatic --exit-zero-on-changes --storybook-build-dir storybook-static --project-token chpt_3e29f54d624610f" @@ -70,6 +73,7 @@ "@chromatic-com/storybook": "^5.2.1", "@eslint/eslintrc": "^3.3.3", "@jest/globals": "^30.4.1", + "@playwright/test": "^1.61.0", "@storybook/addon-a11y": "^10.4.6", "@storybook/addon-docs": "^10.4.6", "@storybook/addon-onboarding": "^10.4.6", diff --git a/ui/playwright.config.ts b/ui/playwright.config.ts new file mode 100644 index 000000000..ac77c405b --- /dev/null +++ b/ui/playwright.config.ts @@ -0,0 +1,69 @@ +import { defineConfig, devices } from "@playwright/test"; + +/** + * Playwright E2E config for the kagent UI. + * + * Data is fetched server-side (Next.js server actions), so browser-level + * `page.route` cannot intercept `/api/**`. Instead we boot a standalone stub + * backend (playwright/mocks/server.mjs) and point Next at it via + * BACKEND_INTERNAL_URL — `getBackendUrl()` (src/lib/utils.ts) checks that env + * var first. Both servers are started by the `webServer` block below. + * + * See playwright/README.md for the full test strategy. + */ + +const CI = !!process.env.CI; + +const STUB_PORT = 8899; +const STUB_URL = `http://127.0.0.1:${STUB_PORT}`; +const APP_URL = "http://localhost:8001"; + +export default defineConfig({ + testDir: "./playwright/tests", + outputDir: "./playwright/test-results", + // Parallelism stays off until Stage 1 per-test data isolation lands: one + // shared stub backend + one Next server means concurrent tests would race + // against shared state (see README). Flip both `fullyParallel` and `workers` + // together when isolation is in place. + fullyParallel: false, + forbidOnly: CI, + retries: CI ? 1 : 0, + workers: 1, + timeout: 30_000, + expect: { timeout: 10_000 }, + reporter: [["html", { open: "never" }], ["list"]], + use: { + baseURL: APP_URL, + screenshot: "only-on-failure", + trace: "retain-on-failure", + video: "off", + }, + projects: [{ name: "chromium", use: { ...devices["Desktop Chrome"] } }], + webServer: [ + { + command: "node playwright/mocks/server.mjs", + url: `${STUB_URL}/__mock/health`, + reuseExistingServer: !CI, + timeout: 30_000, + stdout: "pipe", + stderr: "pipe", + // Pin the port so a shell-exported STUB_PORT can't make the stub bind + // somewhere other than the health-check / BACKEND_INTERNAL_URL address. + env: { STUB_PORT: String(STUB_PORT) }, + }, + { + command: "npm run dev", + url: APP_URL, + // Never reuse an existing dev server: the BACKEND_INTERNAL_URL below is + // only applied to a server Playwright starts. A reused server (e.g. a + // hand-started `npm run dev`) would silently bypass the stub. Always + // boot our own so the redirect is guaranteed; a busy port fails loudly. + reuseExistingServer: false, + timeout: 120_000, + env: { + // Redirect the server-side backend fetch to our stub. + BACKEND_INTERNAL_URL: `${STUB_URL}/api`, + }, + }, + ], +}); diff --git a/ui/playwright/README.md b/ui/playwright/README.md new file mode 100644 index 000000000..96e8f55a1 --- /dev/null +++ b/ui/playwright/README.md @@ -0,0 +1,87 @@ +# Playwright E2E tests + +Page-level browser end-to-end tests for the kagent UI, run against a **mocked +backend**. This suite fills the one gap nothing else covers: **multi-step user +flows across components**. + +## What this suite does and does not cover + +| Layer | Tool | Not Playwright's job | +|---|---|---| +| Atoms (`src/components/ui/*`) | shadcn primitives | skip | +| Visual / render states | Storybook + Vitest-browser + Chromatic | skip | +| Unit / logic | Jest (`*.test.ts(x)`) | skip | +| Page-load smoke (`h1` renders, page reachable, onboarding steps 1–2, model-edit page opens) | Cypress (`cypress/e2e/smoke.cy.ts`) | skip | +| **Multi-step flows: form submission, payload correctness, streaming, wizard completion, error/edge states** | **Playwright (this suite)** | — | + +Rule: if Cypress / Chromatic / Jest already assert it, Playwright does not. + +## How mocking works (important) + +Nearly every `/api/**` call runs **server-side** inside Next.js server actions +(`src/app/actions/*.ts`, all `"use server"`). The browser sends an opaque RPC +POST to the route; the Node server does the actual backend `fetch`. So browser +`page.route("**/api/**")` intercepts **nothing** on load. + +Instead we run a standalone **stub backend** (`mocks/server.mjs`) and point Next +at it with `BACKEND_INTERNAL_URL` — `getBackendUrl()` (`src/lib/utils.ts`) checks +that env var first. Playwright's `webServer` (in `../playwright.config.ts`) boots +both the stub (`:8899`) and `next dev` (`:8001`). + +The one exception is A2A chat streaming (`POST /a2a/**`, SSE), which **is** +browser-originated and `page.route`-able — used in the chat spec (Stage 2). + +## Layout + +``` +playwright/ + tests/ # *.spec.ts, one per feature area + helpers/ # (Stage 1) reusable drivers: page, forms, select, dialog, nav + mocks/ + server.mjs # stub backend (runtime source of happy-path data) + data.ts # typed spec-side builders (mirror server.mjs shapes) + fixtures/ + test.ts # import { test, expect } from here in every spec + tsconfig.json +``` + +## Running + +```bash +cd ui +npm install +npx playwright install chromium # first time only +npm run test:pw # headless +npm run test:pw:ui # interactive UI mode +npm run test:pw:debug # step-through debugger +``` + +Playwright starts the stub + dev server automatically. To drive the app manually +against the stub: + +```bash +node playwright/mocks/server.mjs & +BACKEND_INTERNAL_URL=http://127.0.0.1:8899/api npm run dev +# open http://localhost:8001 +``` + +## Conventions + +- Import `{ test, expect }` from `../fixtures/test`, never `@playwright/test`. +- One spec file per feature area (`tests/agents/`, `tests/chat/`, …); use + `test.step()` for sub-phases of a multi-step flow. +- **`data-testid` policy:** prefer `getByRole` / `getByLabel`. Add `data-testid` + only where role/text is ambiguous or unstable (list rows, per-item action + buttons, wizard steps, combobox options). Add incrementally — no upfront sweep. + Do **not** remove/rename the existing `data-test` model-edit hooks; Cypress + depends on them. + +## Roadmap + +- **Stage 0 (done):** foundation — config, stub backend, CI, one smoke test. +- **Stage 1:** helper/driver library (`helpers/*`) + per-test scenario overrides + via the stub's `/__mock/scenario` endpoint. Prefer stateless scenario selection + keyed by request content (e.g. `?namespace=empty-ns` → `[]`); fall back to the + control endpoint with `workers: 1` for endpoints lacking a discriminator. +- **Stage 2:** feature flows (gap-scoped), ordered by importance — + Create Agent → Chat/session (A2A SSE mock) → Models → MCP → Onboarding completion. diff --git a/ui/playwright/fixtures/test.ts b/ui/playwright/fixtures/test.ts new file mode 100644 index 000000000..f3f95cb34 --- /dev/null +++ b/ui/playwright/fixtures/test.ts @@ -0,0 +1,29 @@ +// Shared test fixture. Import { test, expect } from here in every spec (not +// directly from @playwright/test) so all specs get the same setup. +// +// For now it resets the stub backend before each test. Reset is a no-op until +// Stage 1 adds per-test scenarios, but establishing the pattern now means specs +// don't change when scenarios land. + +import { test as base, expect } from "@playwright/test"; + +const STUB_URL = "http://127.0.0.1:8899"; + +export const test = base.extend({ + page: async ({ page, request }, use) => { + try { + await request.post(`${STUB_URL}/__mock/reset`); + } catch (err) { + // Stub should always be reachable in CI (managed via webServer). Fail fast. + if (process.env.CI) throw err; + } + // Bypass the first-run onboarding wizard (AppInitializer redirects to it when + // this key is unset). Runs before any page script, on every navigation. + await page.addInitScript(() => { + window.localStorage.setItem("kagent-onboarding", "true"); + }); + await use(page); + }, +}); + +export { expect }; diff --git a/ui/playwright/mocks/data.ts b/ui/playwright/mocks/data.ts new file mode 100644 index 000000000..3fa60f741 --- /dev/null +++ b/ui/playwright/mocks/data.ts @@ -0,0 +1,81 @@ +// Typed, spec-side mock builders. Use these in specs to construct payloads for +// assertions or (from Stage 1) to POST scenarios to the stub's /__mock/scenario +// endpoint. The stub backend's runtime happy-path lives in server.mjs; keep the +// shapes here in sync with it. +// +// Typed against the app's own types (via the @/ alias, see playwright/tsconfig.json) +// so drift from the real API surface fails at compile time. + +import type { + AgentResponse, + ModelConfig, + ProviderModelsResponse, + RemoteMCPServerResponse, + ToolsResponse, +} from "@/types"; + +export type Namespace = { name: string; status: string }; +export type ToolServerListEntry = RemoteMCPServerResponse; + +export function mockAgentResponse( + overrides: Partial = {}, +): AgentResponse { + return { + id: "1", + agent: { + metadata: { name: "e2e-agent", namespace: "default" }, + spec: { type: "Declarative", description: "Seeded E2E agent" }, + }, + model: "gpt-4o", + modelProvider: "OpenAI", + modelConfigRef: "default/default-model-config", + tools: [], + deploymentReady: true, + accepted: true, + ...overrides, + }; +} + +export function mockModelsResponse(): ProviderModelsResponse { + return { openai: [{ name: "gpt-4o", function_calling: true }] }; +} + +export function mockModelConfig( + overrides: Partial = {}, +): ModelConfig { + return { + ref: "default/default-model-config", + spec: { model: "gpt-4o", provider: "OpenAI" }, + ...overrides, + }; +} + +export function mockNamespace(overrides: Partial = {}): Namespace { + return { name: "default", status: "Active", ...overrides }; +} + +export function mockToolServer( + overrides: Partial = {}, +): ToolServerListEntry { + return { + ref: "default/e2e-tool-server", + groupKind: "RemoteMCPServer.kagent.dev", + discoveredTools: [], + ...overrides, + }; +} + +export function mockTool( + overrides: Partial = {}, +): ToolsResponse { + return { + id: "e2e-tool", + server_name: "e2e-tool-server", + created_at: "2026-01-01T00:00:00Z", + updated_at: "2026-01-01T00:00:00Z", + deleted_at: "", + description: "Seeded E2E tool", + group_kind: "RemoteMCPServer.kagent.dev", + ...overrides, + }; +} diff --git a/ui/playwright/mocks/server.mjs b/ui/playwright/mocks/server.mjs new file mode 100644 index 000000000..8181dc6dc --- /dev/null +++ b/ui/playwright/mocks/server.mjs @@ -0,0 +1,123 @@ +// Standalone stub backend for Playwright E2E. +// +// The kagent UI fetches data server-side (Next.js server actions), so the +// backend fetch happens in the Node process, not the browser — browser-level +// page.route cannot mock it. Instead we run this tiny HTTP server and point +// Next at it via BACKEND_INTERNAL_URL (see playwright.config.ts). getBackendUrl() +// in src/lib/utils.ts checks BACKEND_INTERNAL_URL first, so every /api/* call +// lands here. +// +// Dependency-free (Node built-in http only) so it runs without a TS/build step. +// Payloads mirror the shapes in src/types/index.ts; the typed spec-side builders +// live in playwright/mocks/data.ts. Keep the two in sync until Stage 1 unifies +// them behind the /__mock/scenario control endpoint. + +import { createServer } from "node:http"; + +const PORT = Number(process.env.STUB_PORT ?? 8899); + +// region Payloads (happy path) + +// Success envelope used by the Go backend: { message, data }. +const ok = (data, message = "OK") => ({ message, data }); + +const agent = { + id: "1", + agent: { + metadata: { name: "e2e-agent", namespace: "default" }, + spec: { type: "Declarative", description: "Seeded E2E agent" }, + }, + model: "gpt-4o", + modelProvider: "OpenAI", + modelConfigRef: "default/default-model-config", + tools: [], + deploymentReady: true, + accepted: true, +}; + +const modelConfig = { + ref: "default/default-model-config", + spec: { model: "gpt-4o", provider: "OpenAI" }, +}; + +const toolServer = { + ref: "default/e2e-tool-server", + groupKind: "RemoteMCPServer.kagent.dev", + discoveredTools: [], +}; + +const tool = { + id: "e2e-tool", + server_name: "e2e-tool-server", + created_at: "2026-01-01T00:00:00Z", + updated_at: "2026-01-01T00:00:00Z", + deleted_at: "", + description: "Seeded E2E tool", + group_kind: "RemoteMCPServer.kagent.dev", +}; + +const substrateStatus = { + enabled: false, + workerPools: [], + actorTemplates: [], + actors: [], + workers: [], +}; + +// GET route table keyed by pathname (query string stripped before lookup). +const GET_ROUTES = { + "/api/agents": () => ok([agent], "Successfully fetched agents"), + "/api/models": () => ok({ openai: [{ name: "gpt-4o", function_calling: true }] }), + "/api/modelconfigs": () => ok([modelConfig]), + "/api/namespaces": () => ok([{ name: "default", status: "Active" }], "Namespaces fetched successfully"), + "/api/toolservers": () => ok([toolServer]), + "/api/tools": () => ok([tool]), + "/api/substrate/status": () => ok(substrateStatus, "Substrate status fetched"), +}; + +// endregion + +// region Server + +const json = (res, status, body) => { + const payload = JSON.stringify(body); + res.writeHead(status, { + "Content-Type": "application/json", + "Content-Length": Buffer.byteLength(payload), + }); + res.end(payload); +}; + +const server = createServer((req, res) => { + // req.method/url are typed string|undefined; default them so a malformed + // request can't throw on the split below. + const method = req.method ?? "GET"; + const url = req.url ?? "/"; + const pathname = url.split("?")[0]; + + // Control + health endpoints. + if (pathname === "/__mock/health") return json(res, 200, { status: "ok" }); + if (method === "POST" && pathname === "/__mock/reset") { + // No scenario state yet — happy path only. Wired up in Stage 1. + return json(res, 200, { status: "reset" }); + } + if (method === "POST" && pathname === "/__mock/scenario") { + // Placeholder for per-test overrides (Stage 1). + return json(res, 200, { status: "noop" }); + } + + if (method === "GET" && GET_ROUTES[pathname]) { + console.log(`[stub] ${method} ${url} -> 200`); + return json(res, 200, GET_ROUTES[pathname]()); + } + + // Anything unmocked is a real gap — make it loud so we notice leaks. + console.warn(`[stub] UNHANDLED ${method} ${url} -> 404`); + return json(res, 404, { error: `No stub for ${method} ${pathname}` }); +}); + +server.listen(PORT, "127.0.0.1", () => { + console.log(`[stub] kagent mock backend listening on http://127.0.0.1:${PORT}`); +}); + +// endregion diff --git a/ui/playwright/tests/smoke.spec.ts b/ui/playwright/tests/smoke.spec.ts new file mode 100644 index 000000000..3bbc1eba5 --- /dev/null +++ b/ui/playwright/tests/smoke.spec.ts @@ -0,0 +1,20 @@ +import { test, expect } from "../fixtures/test"; + +// Stage 0 smoke test: proves the whole rig works end to end — Playwright boots +// the stub backend + `next dev`, the server-side fetch is redirected to the stub +// (via BACKEND_INTERNAL_URL), and the home page renders the mocked agent list. +test("home page renders the Agents list against the mocked backend", async ({ page }) => { + const fatalErrors: string[] = []; + page.on("pageerror", (err) => fatalErrors.push(err.message)); + + await page.goto("/"); + + // PageHeader renders

Agents

on successful load + // (present in both populated and empty states; absent in loading/error states). + await expect(page.getByRole("heading", { level: 1, name: "Agents" })).toBeVisible(); + + // ErrorState (early-return branch) renders this heading — it must not appear. + await expect(page.getByText("Error Encountered")).toHaveCount(0); + + expect(fatalErrors, `uncaught page errors: ${fatalErrors.join("; ")}`).toEqual([]); +}); diff --git a/ui/playwright/tsconfig.json b/ui/playwright/tsconfig.json new file mode 100644 index 000000000..b84b7846f --- /dev/null +++ b/ui/playwright/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../tsconfig.json", + "compilerOptions": { + "types": ["node", "@playwright/test"] + }, + "include": ["**/*.ts"] +}