Merge pull request #145 from aicodingstack/agent/e2e-changelog

feat(quality): add browser and changelog gates

Author: ericyangpan

.github/workflows/ci.yml

@@ -55,6 +55,8 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
 
       - name: Setup Node.js
         uses: actions/setup-node@v6
@@ -71,6 +73,11 @@ jobs:
       - name: Check data health
         run: npm run data-health:check
 
+      - name: Check manifest changelog
+        run: npm run changelog:check
+        env:
+          CHANGELOG_BASE: ${{ github.event.pull_request.base.sha || github.event.before }}
+
   spell-check:
     name: Spell Check
     runs-on: ubuntu-latest
@@ -123,10 +130,33 @@ jobs:
             .open-next
           retention-days: 7
 
+  e2e:
+    name: Browser Smoke
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v6
+        with:
+          node-version: '22'
+          cache: 'npm'
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Install Chromium
+        run: npx playwright install --with-deps chromium
+
+      - name: Run browser smoke tests
+        run: npm run test:e2e:ci
+
   ci-success:
     name: CI Success
     runs-on: ubuntu-latest
-    needs: [lint, type-check, test, spell-check, build]
+    needs: [lint, type-check, test, spell-check, build, e2e]
     if: always()
     steps:
       - name: Check all jobs

.gitignore

@@ -10,6 +10,8 @@
 
 # testing
 /coverage
+/playwright-report/
+/test-results/
 
 # next.js
 /.next/

data/changelogs.json

@@ -1 +1,63 @@
-{}
+{
+  "version": 1,
+  "entries": [
+    {
+      "id": "2026-07-18-project-review",
+      "date": "2026-07-18",
+      "summary": "Backfilled official-source provenance for priority providers and models",
+      "changes": [
+        {
+          "category": "models",
+          "id": "claude-haiku-4-5",
+          "change": "updated",
+          "fields": ["docsUrl", "confidence", "lastVerifiedAt", "sources", "verifiedBy"]
+        },
+        {
+          "category": "models",
+          "id": "gemini-3-flash",
+          "change": "updated",
+          "fields": ["docsUrl", "confidence", "lastVerifiedAt", "sources", "verifiedBy"]
+        },
+        {
+          "category": "models",
+          "id": "gpt-5-2",
+          "change": "updated",
+          "fields": [
+            "docsUrl",
+            "lifecycle",
+            "confidence",
+            "lastVerifiedAt",
+            "sources",
+            "verifiedBy"
+          ]
+        },
+        {
+          "category": "providers",
+          "id": "anthropic",
+          "change": "updated",
+          "fields": [
+            "applyKeyUrl",
+            "docsUrl",
+            "verified",
+            "confidence",
+            "lastVerifiedAt",
+            "sources",
+            "verifiedBy"
+          ]
+        },
+        {
+          "category": "providers",
+          "id": "google",
+          "change": "updated",
+          "fields": ["docsUrl", "verified", "confidence", "lastVerifiedAt", "sources", "verifiedBy"]
+        },
+        {
+          "category": "providers",
+          "id": "openai",
+          "change": "updated",
+          "fields": ["docsUrl", "verified", "confidence", "lastVerifiedAt", "sources", "verifiedBy"]
+        }
+      ]
+    }
+  ]
+}

docs/PROJECT-REVIEW-2026-07-18.md

@@ -74,7 +74,7 @@ Baseline captured on 2026-07-18:
   integrity, translation placeholders, and missing pricing/benchmark values.
 - [x] Add freshness policies (for example 30/60/90-day review thresholds by field/category).
 - [ ] Automate change discovery from official sources, but keep human review before merge.
-- [ ] Populate a real changelog from manifest diffs instead of keeping an empty static file.
+- [x] Populate a real changelog from manifest diffs instead of keeping an empty static file.
 
 ## P1 — Product Experience
 
@@ -93,7 +93,7 @@ Baseline captured on 2026-07-18:
 ## P1 — Quality Gates
 
 - [x] Add route-contract tests for every manifest category.
-- [ ] Add browser-level smoke tests for search, navigation, detail pages, locale switching, and
+- [x] Add browser-level smoke tests for search, navigation, detail pages, locale switching, and
   comparison selection.
 - [ ] Add automated accessibility checks for dialogs, menus, comparison tables, and mobile nav.
 - [x] Run generation in CI and fail when tracked generated output differs.
@@ -171,3 +171,9 @@ evidence; completed items are reflected in the checklists and implementation log
   a permanent redirect, and removed its duplicate comparison implementation.
 - Added a persistent two-model selection shared by model cards, detail-page comparison actions,
   canonical pair URLs, and the comparison selector; adding a third model replaces the oldest pick.
+- Added Playwright browser smoke coverage for search-to-detail navigation, locale-preserving route
+  switches, and persistent model comparison selections, and made it a required CI job.
+- Replaced the empty changelog placeholder with a deterministic manifest-diff record and added a CI
+  guard requiring future manifest changes to be represented in `data/changelogs.json`.
+- Fixed the search result category badge to use the shared translated category names after browser
+  testing exposed a missing runtime message lookup.

package-lock.json

@@ -20,6 +20,8 @@
     "spell:fix": "cspell '**/*.{ts,tsx,js,jsx,json,md,mdx}' --show-suggestions",
     "test": "vitest",
     "test:ci": "vitest run --reporter=verbose",
+    "test:e2e": "playwright test",
+    "test:e2e:ci": "playwright test --reporter=line",
     "test:validate": "vitest run tests/validate --reporter=verbose",
     "test:urls": "RUN_URL_TESTS=1 vitest run tests/validate/urls.accessibility.test.ts --reporter=verbose",
     "validate:i18n": "npx tsx scripts/validate/validate-i18n.ts",
@@ -31,6 +33,9 @@
     "data-health": "npx tsx scripts/validate/data-health.ts",
     "data-health:check": "npx tsx scripts/validate/data-health.ts --check-snapshot --fail-on=error",
     "data-health:report": "npx tsx scripts/validate/data-health.ts --write",
+    "changelog:check": "npx tsx scripts/validate/changelog.ts --check",
+    "changelog:generate": "npx tsx scripts/validate/changelog.ts --write",
+    "dev:e2e": "next dev --turbopack -p 3100",
     "generate": "npx tsx scripts/generate/index.ts",
     "generate:manifests": "npx tsx scripts/generate/index.ts manifest-indexes",
     "generate:metadata": "npx tsx scripts/generate/index.ts metadata",
@@ -69,6 +74,7 @@
     "@mdx-js/loader": "^3.1.1",
     "@next/bundle-analyzer": "^16.1.2",
     "@next/mdx": "^16.1.2",
+    "@playwright/test": "^1.61.1",
     "@tailwindcss/postcss": "^4.1.17",
     "@types/node": "^22.19.2",
     "@types/react": "^19.2.9",

package.json

@@ -0,0 +1,29 @@
+import { defineConfig, devices } from '@playwright/test'
+
+const port = 3100
+
+export default defineConfig({
+  testDir: './tests/e2e',
+  fullyParallel: true,
+  forbidOnly: Boolean(process.env.CI),
+  retries: process.env.CI ? 2 : 0,
+  workers: process.env.CI ? 2 : undefined,
+  reporter: process.env.CI ? 'line' : 'list',
+  use: {
+    baseURL: `http://127.0.0.1:${port}`,
+    trace: 'retain-on-failure',
+    screenshot: 'only-on-failure',
+  },
+  projects: [
+    {
+      name: 'chromium',
+      use: { ...devices['Desktop Chrome'] },
+    },
+  ],
+  webServer: {
+    command: 'npm run dev:e2e',
+    url: `http://127.0.0.1:${port}`,
+    reuseExistingServer: !process.env.CI,
+    timeout: 120_000,
+  },
+})

playwright.config.ts

@@ -12,8 +12,11 @@ Repository automation is written in TypeScript and executed with `tsx`. Prefer t
 | Sort manifest fields | `npm run refactor:sort-fields` | Reorders manifest JSON using schema order |
 | Fetch GitHub stars | `npm run fetch:github-stars` | Updates `data/github-stars.json` |
 | Validate manifests and data | `npm run test:validate` | Runs the validation test suite |
+| Run browser smoke tests | `npm run test:e2e` | Exercises core journeys in Chromium against a local Next.js server |
 | Check data health | `npm run data-health:check` | Fails on invalid health-report data |
 | Refresh data-health snapshot | `npm run data-health:report` | Writes `data/data-health.json` and `docs/DATA-HEALTH.md` |
+| Check manifest changelog | `npm run changelog:check` | Requires manifest diffs to be represented in `data/changelogs.json` |
+| Generate manifest changelog | `npm run changelog:generate -- --base=<ref> --id=<id> --summary="<text>"` | Adds or replaces one entry from a Git diff |
 | Validate i18n structure | `npm run validate:i18n` | Checks locale alignment and translation shape |
 | Validate i18n usage | `npm run validate:i18n-usage` | Checks translation keys referenced by source |
 | Validate duplicate i18n values | `npm run validate:i18n-duplicates` | Reports duplicated translation content |