diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 2179f02729..0deab54482 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -16,7 +16,6 @@ permissions: jobs: client-conformance: runs-on: ubuntu-latest - continue-on-error: true steps: - uses: actions/checkout@v6 - name: Install pnpm @@ -34,7 +33,6 @@ jobs: server-conformance: runs-on: ubuntu-latest - continue-on-error: true steps: - uses: actions/checkout@v6 - name: Install pnpm @@ -49,3 +47,4 @@ jobs: - run: pnpm install - run: pnpm run build:all - run: pnpm run test:conformance:server + - run: pnpm run test:conformance:server:draft diff --git a/package.json b/package.json index a2cb93f629..d1ecc0c627 100644 --- a/package.json +++ b/package.json @@ -40,6 +40,7 @@ "test:conformance:client:all": "pnpm --filter @modelcontextprotocol/test-conformance run test:conformance:client:all", "test:conformance:client:run": "pnpm --filter @modelcontextprotocol/test-conformance run test:conformance:client:run", "test:conformance:server": "pnpm --filter @modelcontextprotocol/test-conformance run test:conformance:server", + "test:conformance:server:draft": "pnpm --filter @modelcontextprotocol/test-conformance run test:conformance:server:draft", "test:conformance:server:all": "pnpm --filter @modelcontextprotocol/test-conformance run test:conformance:server:all", "test:conformance:server:run": "pnpm --filter @modelcontextprotocol/test-conformance run test:conformance:server:run", "test:conformance:all": "pnpm run test:conformance:client:all && pnpm run test:conformance:server:all" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e81ccf5d81..c13f1ebcde 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1099,8 +1099,8 @@ importers: specifier: workspace:^ version: link:../../packages/client '@modelcontextprotocol/conformance': - specifier: 0.1.15 - version: 0.1.15(@cfworker/json-schema@4.1.1) + specifier: 0.2.0-alpha.1 + version: 0.2.0-alpha.1(@cfworker/json-schema@4.1.1) '@modelcontextprotocol/core': specifier: workspace:^ version: link:../../packages/core @@ -2108,12 +2108,12 @@ packages: '@manypkg/get-packages@1.1.3': resolution: {integrity: sha512-fo+QhuU3qE/2TQMQmbVMqaQ6EWbMhi4ABWP+O4AM1NqPBuy0OrApV5LO6BrrgnhtAHS2NH6RrVk9OL181tTi8A==} - '@modelcontextprotocol/conformance@0.1.15': - resolution: {integrity: sha512-B1eNYpv5kas9YFC40su7SWhKHR2DwYzJRpiX6dfWzDWMcDr71myKVj//PwyqUHC7oucs3EJqcqnwvSuUwvenoQ==} + '@modelcontextprotocol/conformance@0.2.0-alpha.1': + resolution: {integrity: sha512-w5YpjJ7uNXGIWs2r+gMwowNgREykYvDlrFS1Ckl47lVbkZJn2CAytmuV/zCrsCuZkeE7djK1tJkuZ0oa72BhGA==} hasBin: true - '@modelcontextprotocol/sdk@1.28.0': - resolution: {integrity: sha512-gmloF+i+flI8ouQK7MWW4mOwuMh4RePBuPFAEPC6+pdqyWOUMDOixb6qZ69owLJpz6XmyllCouc4t8YWO+E2Nw==} + '@modelcontextprotocol/sdk@1.29.0': + resolution: {integrity: sha512-zo37mZA9hJWpULgkRpowewez1y6ML5GsXJPY8FI0tBBCd77HEvza4jDqRKOXgHNn867PVGCyTdzqpz0izu5ZjQ==} engines: {node: '>=18'} peerDependencies: '@cfworker/json-schema': ^4.1.1 @@ -3639,6 +3639,10 @@ packages: resolution: {integrity: sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg==} engines: {node: '>=18.0.0'} + eventsource-parser@3.0.8: + resolution: {integrity: sha512-70QWGkr4snxr0OXLRWsFLeRBIRPuQOvt4s8QYjmUlmlkyTZkRqS7EDVRZtzU3TiyDbXSzaOeF0XUKy8PchzukQ==} + engines: {node: '>=18.0.0'} + eventsource@3.0.7: resolution: {integrity: sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA==} engines: {node: '>=18.0.0'} @@ -5087,8 +5091,8 @@ packages: resolution: {integrity: sha512-BM/JzwwaRXxrLdElV2Uo6cTLEjhSb3WXboncJamZ15NgUURmvlXvxa6xkwIOILIjPNo9i8ku136ZvWV0Uly8+w==} engines: {node: '>=20.18.1'} - undici@7.24.6: - resolution: {integrity: sha512-Xi4agocCbRzt0yYMZGMA6ApD7gvtUFaxm4ZmeacWI4cZxaF6C+8I8QfofC20NAePiB/IcvZmzkJ7XPa471AEtA==} + undici@7.25.0: + resolution: {integrity: sha512-xXnp4kTyor2Zq+J1FfPI6Eq3ew5h6Vl0F/8d9XU5zZQf1tX9s2Su1/3PiMmUANFULpmksxkClamIZcaUqryHsQ==} engines: {node: '>=20.18.1'} unenv@2.0.0-rc.24: @@ -5315,9 +5319,6 @@ packages: peerDependencies: zod: ^3.25.28 || ^4 - zod@3.25.76: - resolution: {integrity: sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==} - zod@4.3.6: resolution: {integrity: sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg==} @@ -5997,22 +5998,22 @@ snapshots: globby: 11.1.0 read-yaml-file: 1.1.0 - '@modelcontextprotocol/conformance@0.1.15(@cfworker/json-schema@4.1.1)': + '@modelcontextprotocol/conformance@0.2.0-alpha.1(@cfworker/json-schema@4.1.1)': dependencies: - '@modelcontextprotocol/sdk': 1.28.0(@cfworker/json-schema@4.1.1)(zod@3.25.76) + '@modelcontextprotocol/sdk': 1.29.0(@cfworker/json-schema@4.1.1)(zod@4.3.6) '@octokit/rest': 22.0.1 commander: 14.0.3 - eventsource-parser: 3.0.6 + eventsource-parser: 3.0.8 express: 5.2.1 jose: 6.2.2 - undici: 7.24.6 + undici: 7.25.0 yaml: 2.8.3 - zod: 3.25.76 + zod: 4.3.6 transitivePeerDependencies: - '@cfworker/json-schema' - supports-color - '@modelcontextprotocol/sdk@1.28.0(@cfworker/json-schema@4.1.1)(zod@3.25.76)': + '@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.3.6)': dependencies: '@hono/node-server': 1.19.11(hono@4.12.9) ajv: 8.18.0 @@ -6021,7 +6022,7 @@ snapshots: cors: 2.8.6 cross-spawn: 7.0.6 eventsource: 3.0.7 - eventsource-parser: 3.0.6 + eventsource-parser: 3.0.8 express: 5.2.1 express-rate-limit: 8.3.1(express@5.2.1) hono: 4.12.9 @@ -6029,8 +6030,8 @@ snapshots: json-schema-typed: 8.0.2 pkce-challenge: 5.0.1 raw-body: 3.0.2 - zod: 3.25.76 - zod-to-json-schema: 3.25.2(zod@3.25.76) + zod: 4.3.6 + zod-to-json-schema: 3.25.2(zod@4.3.6) optionalDependencies: '@cfworker/json-schema': 4.1.1 transitivePeerDependencies: @@ -7518,6 +7519,8 @@ snapshots: eventsource-parser@3.0.6: {} + eventsource-parser@3.0.8: {} + eventsource@3.0.7: dependencies: eventsource-parser: 3.0.6 @@ -9091,7 +9094,7 @@ snapshots: undici@7.24.4: {} - undici@7.24.6: {} + undici@7.25.0: {} unenv@2.0.0-rc.24: dependencies: @@ -9353,10 +9356,8 @@ snapshots: cookie: 1.1.1 youch-core: 0.3.3 - zod-to-json-schema@3.25.2(zod@3.25.76): + zod-to-json-schema@3.25.2(zod@4.3.6): dependencies: - zod: 3.25.76 - - zod@3.25.76: {} + zod: 4.3.6 zod@4.3.6: {} diff --git a/test/conformance/README.md b/test/conformance/README.md index 7d443f6f73..5b32b5aaf8 100644 --- a/test/conformance/README.md +++ b/test/conformance/README.md @@ -2,6 +2,8 @@ This directory contains conformance test implementations for the TypeScript MCP SDK. +> `@modelcontextprotocol/conformance` is pinned to an exact version (no `^` range) in `package.json`. New conformance releases are adopted by deliberately bumping the pin and updating `expected-failures.yaml` for any new scenarios/checks in the same change. + ## Client Conformance Tests Tests the SDK's client implementation against a conformance test server. diff --git a/test/conformance/expected-failures.yaml b/test/conformance/expected-failures.yaml index 5f4de22088..f4b7ce4213 100644 --- a/test/conformance/expected-failures.yaml +++ b/test/conformance/expected-failures.yaml @@ -1,4 +1,93 @@ # Conformance scenarios not yet implemented in the v2 TypeScript SDK. # CI exits 0 if only these fail, exits 1 on unexpected failures or stale entries. +# +# Baseline established against @modelcontextprotocol/conformance 0.2.0-alpha.1, +# which adds the draft-spec scenario suite (SEP-2575, SEP-2322, SEP-2243, SEP-2549, +# SEP-2468, ...) plus new checks on existing scenarios (SEP-837 application_type +# during DCR). +# +# Entries are grouped by SEP. As each SEP/milestone is implemented in the SDK the +# corresponding scenarios start passing and MUST be removed from this list (the +# runner fails on stale entries), so the baseline burns down per milestone. -client: [] +client: + # --- Draft-spec scenarios (in `--suite draft`, also part of `--suite all`) --- + # SEP-2575 (request metadata / _meta envelope): client does not populate the + # _meta envelope or the MCP-Protocol-Version header semantics yet. + - request-metadata + # SEP-2322 (multi-round-trip requests): client does not echo requestState / + # handle IncompleteResult yet. + - sep-2322-client-request-state + # SEP-2243 (HTTP standardization): no fixture handler / client header support yet. + - http-custom-headers + - http-invalid-tool-headers + # SEP-2106 (JSON Schema $ref handling): client still dereferences network $refs. + - json-schema-ref-no-deref + # SEP-2468 (authorization response iss parameter): not implemented in the client. + - auth/iss-supported + - auth/iss-not-advertised + - auth/iss-supported-missing + - auth/iss-wrong-issuer + - auth/iss-unexpected + - auth/iss-normalized + - auth/metadata-issuer-mismatch + # SEP-2352 (authorization server migration): client does not re-register when + # PRM authorization_servers changes. + - auth/authorization-server-migration + # SEP-2207 (offline_access scope) scenario, currently failing only on the new + # SEP-837 application_type check (see the SEP-837 group below). + - auth/offline-access-not-supported + + # --- Pre-existing scenarios that fail on checks added after conformance 0.1.15 --- + # SEP-837: client MUST send application_type during Dynamic Client Registration. + # Single new check; everything else in these scenarios passes. + - auth/metadata-default + - auth/metadata-var1 + - auth/metadata-var2 + - auth/metadata-var3 + - auth/scope-from-www-authenticate + - auth/scope-from-scopes-supported + - auth/scope-omitted-when-undefined + - auth/scope-step-up + - auth/scope-retry-limit + - auth/token-endpoint-auth-basic + - auth/token-endpoint-auth-post + - auth/token-endpoint-auth-none + - auth/2025-03-26-oauth-metadata-backcompat + # SEP-990 (enterprise-managed authorization extension): no fixture handler / + # client support for the token-exchange + JWT bearer flow. + - auth/enterprise-managed-authorization + +server: + # --- Draft-spec scenarios (in `--suite draft`; the default `active` suite is green) --- + # SEP-2575 (stateless HTTP / _meta envelope): server has no stateless mode, + # _meta-derived capabilities, error-code mappings, or server/discover yet. + - server-stateless + # SEP-2322 (multi-round-trip requests / IncompleteResult): not implemented; + # most scenarios currently fail early with "Session ID required" because the + # fixture only runs in stateful mode. + - input-required-result-basic-elicitation + - input-required-result-basic-sampling + - input-required-result-basic-list-roots + - input-required-result-request-state + - input-required-result-multiple-input-requests + - input-required-result-multi-round + - input-required-result-non-tool-request + - input-required-result-result-type + - input-required-result-tampered-state + - input-required-result-capability-check + # SEP-2549 (caching): no ttlMs/cacheScope support; scenario also hits the + # stateful-mode "Session ID required" error. + - caching + # SEP-2243 (HTTP header standardization): -32001 HeaderMismatch handling and + # case-insensitive/whitespace-trimmed header validation not implemented. + - http-header-validation + - http-custom-header-server-validation + # WARNING-only entries: these scenarios emit no FAILURE checks, only SHOULD-level + # WARNINGs, but the expected-failures evaluator counts WARNINGs as failures. + # SEP-2164: server returns -32002 without the requested URI in error.data. + - sep-2164-resource-not-found + # SEP-2322 SHOULD-level behaviours (re-request missing inputResponses, ignore + # unrecognized inputResponses keys). + - input-required-result-missing-input-response + - input-required-result-ignore-extra-params diff --git a/test/conformance/package.json b/test/conformance/package.json index db0f04a4db..6ce6307f5e 100644 --- a/test/conformance/package.json +++ b/test/conformance/package.json @@ -32,12 +32,13 @@ "test:conformance:client:all": "conformance client --command 'npx tsx ./src/everythingClient.ts' --suite all --expected-failures ./expected-failures.yaml", "test:conformance:client:run": "npx tsx ./src/everythingClient.ts", "test:conformance:server": "scripts/run-server-conformance.sh --expected-failures ./expected-failures.yaml", + "test:conformance:server:draft": "scripts/run-server-conformance.sh --suite draft --expected-failures ./expected-failures.yaml", "test:conformance:server:all": "scripts/run-server-conformance.sh --suite all --expected-failures ./expected-failures.yaml", "test:conformance:server:run": "npx tsx ./src/everythingServer.ts", "test:conformance:all": "pnpm run test:conformance:client:all && pnpm run test:conformance:server:all" }, "devDependencies": { - "@modelcontextprotocol/conformance": "0.1.15", + "@modelcontextprotocol/conformance": "0.2.0-alpha.1", "@modelcontextprotocol/client": "workspace:^", "@modelcontextprotocol/server": "workspace:^", "@modelcontextprotocol/core": "workspace:^",