From 548bd8a89d6d2a795df848560d767880b954d6ed Mon Sep 17 00:00:00 2001 From: Felix Weinberger Date: Sat, 30 May 2026 08:20:15 +0000 Subject: [PATCH 1/6] test(conformance): pin conformance 0.2.0-alpha.1 and enable the draft suites Bump @modelcontextprotocol/conformance from 0.1.15 to the published 0.2.0-alpha.1 release, which adds the draft-spec scenario suites (SEP-2575, SEP-2322, SEP-2243, SEP-2549, SEP-2468, ...) and new checks on existing scenarios (SEP-837 application_type during DCR). The dependency is pinned to an exact version (no ^ range): the conformance framework keeps evolving alongside the draft spec, so new releases are adopted by deliberate version bumps together with expected-failures.yaml updates in the same change. Add a test:conformance:server:draft script that runs the draft-spec server suite against the expected-failures baseline. --- pnpm-lock.yaml | 51 ++++++++++++++++++----------------- test/conformance/README.md | 2 ++ test/conformance/package.json | 3 ++- 3 files changed, 30 insertions(+), 26 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2e0326f2b..97f97675d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1008,8 +1008,8 @@ importers: specifier: workspace:^ version: link:../../packages/client '@modelcontextprotocol/conformance': - specifier: 0.1.15 - version: 0.1.15(@cfworker/json-schema@4.1.1) + specifier: 0.2.0-alpha.1 + version: 0.2.0-alpha.1(@cfworker/json-schema@4.1.1) '@modelcontextprotocol/core': specifier: workspace:^ version: link:../../packages/core @@ -2014,12 +2014,12 @@ packages: '@manypkg/get-packages@1.1.3': resolution: {integrity: sha512-fo+QhuU3qE/2TQMQmbVMqaQ6EWbMhi4ABWP+O4AM1NqPBuy0OrApV5LO6BrrgnhtAHS2NH6RrVk9OL181tTi8A==} - '@modelcontextprotocol/conformance@0.1.15': - resolution: {integrity: sha512-B1eNYpv5kas9YFC40su7SWhKHR2DwYzJRpiX6dfWzDWMcDr71myKVj//PwyqUHC7oucs3EJqcqnwvSuUwvenoQ==} + '@modelcontextprotocol/conformance@0.2.0-alpha.1': + resolution: {integrity: sha512-w5YpjJ7uNXGIWs2r+gMwowNgREykYvDlrFS1Ckl47lVbkZJn2CAytmuV/zCrsCuZkeE7djK1tJkuZ0oa72BhGA==} hasBin: true - '@modelcontextprotocol/sdk@1.28.0': - resolution: {integrity: sha512-gmloF+i+flI8ouQK7MWW4mOwuMh4RePBuPFAEPC6+pdqyWOUMDOixb6qZ69owLJpz6XmyllCouc4t8YWO+E2Nw==} + '@modelcontextprotocol/sdk@1.29.0': + resolution: {integrity: sha512-zo37mZA9hJWpULgkRpowewez1y6ML5GsXJPY8FI0tBBCd77HEvza4jDqRKOXgHNn867PVGCyTdzqpz0izu5ZjQ==} engines: {node: '>=18'} peerDependencies: '@cfworker/json-schema': ^4.1.1 @@ -3545,6 +3545,10 @@ packages: resolution: {integrity: sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg==} engines: {node: '>=18.0.0'} + eventsource-parser@3.0.8: + resolution: {integrity: sha512-70QWGkr4snxr0OXLRWsFLeRBIRPuQOvt4s8QYjmUlmlkyTZkRqS7EDVRZtzU3TiyDbXSzaOeF0XUKy8PchzukQ==} + engines: {node: '>=18.0.0'} + eventsource@3.0.7: resolution: {integrity: sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA==} engines: {node: '>=18.0.0'} @@ -4993,8 +4997,8 @@ packages: resolution: {integrity: sha512-BM/JzwwaRXxrLdElV2Uo6cTLEjhSb3WXboncJamZ15NgUURmvlXvxa6xkwIOILIjPNo9i8ku136ZvWV0Uly8+w==} engines: {node: '>=20.18.1'} - undici@7.24.6: - resolution: {integrity: sha512-Xi4agocCbRzt0yYMZGMA6ApD7gvtUFaxm4ZmeacWI4cZxaF6C+8I8QfofC20NAePiB/IcvZmzkJ7XPa471AEtA==} + undici@7.25.0: + resolution: {integrity: sha512-xXnp4kTyor2Zq+J1FfPI6Eq3ew5h6Vl0F/8d9XU5zZQf1tX9s2Su1/3PiMmUANFULpmksxkClamIZcaUqryHsQ==} engines: {node: '>=20.18.1'} unenv@2.0.0-rc.24: @@ -5221,9 +5225,6 @@ packages: peerDependencies: zod: ^3.25.28 || ^4 - zod@3.25.76: - resolution: {integrity: sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==} - zod@4.3.6: resolution: {integrity: sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg==} @@ -5903,22 +5904,22 @@ snapshots: globby: 11.1.0 read-yaml-file: 1.1.0 - '@modelcontextprotocol/conformance@0.1.15(@cfworker/json-schema@4.1.1)': + '@modelcontextprotocol/conformance@0.2.0-alpha.1(@cfworker/json-schema@4.1.1)': dependencies: - '@modelcontextprotocol/sdk': 1.28.0(@cfworker/json-schema@4.1.1)(zod@3.25.76) + '@modelcontextprotocol/sdk': 1.29.0(@cfworker/json-schema@4.1.1)(zod@4.3.6) '@octokit/rest': 22.0.1 commander: 14.0.3 - eventsource-parser: 3.0.6 + eventsource-parser: 3.0.8 express: 5.2.1 jose: 6.2.2 - undici: 7.24.6 + undici: 7.25.0 yaml: 2.8.3 - zod: 3.25.76 + zod: 4.3.6 transitivePeerDependencies: - '@cfworker/json-schema' - supports-color - '@modelcontextprotocol/sdk@1.28.0(@cfworker/json-schema@4.1.1)(zod@3.25.76)': + '@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.3.6)': dependencies: '@hono/node-server': 1.19.11(hono@4.12.9) ajv: 8.18.0 @@ -5927,7 +5928,7 @@ snapshots: cors: 2.8.6 cross-spawn: 7.0.6 eventsource: 3.0.7 - eventsource-parser: 3.0.6 + eventsource-parser: 3.0.8 express: 5.2.1 express-rate-limit: 8.3.1(express@5.2.1) hono: 4.12.9 @@ -5935,8 +5936,8 @@ snapshots: json-schema-typed: 8.0.2 pkce-challenge: 5.0.1 raw-body: 3.0.2 - zod: 3.25.76 - zod-to-json-schema: 3.25.2(zod@3.25.76) + zod: 4.3.6 + zod-to-json-schema: 3.25.2(zod@4.3.6) optionalDependencies: '@cfworker/json-schema': 4.1.1 transitivePeerDependencies: @@ -7424,6 +7425,8 @@ snapshots: eventsource-parser@3.0.6: {} + eventsource-parser@3.0.8: {} + eventsource@3.0.7: dependencies: eventsource-parser: 3.0.6 @@ -8997,7 +9000,7 @@ snapshots: undici@7.24.4: {} - undici@7.24.6: {} + undici@7.25.0: {} unenv@2.0.0-rc.24: dependencies: @@ -9259,10 +9262,8 @@ snapshots: cookie: 1.1.1 youch-core: 0.3.3 - zod-to-json-schema@3.25.2(zod@3.25.76): + zod-to-json-schema@3.25.2(zod@4.3.6): dependencies: - zod: 3.25.76 - - zod@3.25.76: {} + zod: 4.3.6 zod@4.3.6: {} diff --git a/test/conformance/README.md b/test/conformance/README.md index 7d443f6f7..5b32b5aaf 100644 --- a/test/conformance/README.md +++ b/test/conformance/README.md @@ -2,6 +2,8 @@ This directory contains conformance test implementations for the TypeScript MCP SDK. +> `@modelcontextprotocol/conformance` is pinned to an exact version (no `^` range) in `package.json`. New conformance releases are adopted by deliberately bumping the pin and updating `expected-failures.yaml` for any new scenarios/checks in the same change. + ## Client Conformance Tests Tests the SDK's client implementation against a conformance test server. diff --git a/test/conformance/package.json b/test/conformance/package.json index db0f04a4d..6ce6307f5 100644 --- a/test/conformance/package.json +++ b/test/conformance/package.json @@ -32,12 +32,13 @@ "test:conformance:client:all": "conformance client --command 'npx tsx ./src/everythingClient.ts' --suite all --expected-failures ./expected-failures.yaml", "test:conformance:client:run": "npx tsx ./src/everythingClient.ts", "test:conformance:server": "scripts/run-server-conformance.sh --expected-failures ./expected-failures.yaml", + "test:conformance:server:draft": "scripts/run-server-conformance.sh --suite draft --expected-failures ./expected-failures.yaml", "test:conformance:server:all": "scripts/run-server-conformance.sh --suite all --expected-failures ./expected-failures.yaml", "test:conformance:server:run": "npx tsx ./src/everythingServer.ts", "test:conformance:all": "pnpm run test:conformance:client:all && pnpm run test:conformance:server:all" }, "devDependencies": { - "@modelcontextprotocol/conformance": "0.1.15", + "@modelcontextprotocol/conformance": "0.2.0-alpha.1", "@modelcontextprotocol/client": "workspace:^", "@modelcontextprotocol/server": "workspace:^", "@modelcontextprotocol/core": "workspace:^", From 50268755230393eb190f62cbcd02c5e7102db4a0 Mon Sep 17 00:00:00 2001 From: Felix Weinberger Date: Sat, 30 May 2026 08:20:15 +0000 Subject: [PATCH 2/6] test(conformance): baseline expected failures for the draft-spec suites Record every scenario that currently fails against conformance 0.2.0-alpha.1 in expected-failures.yaml so the client (--suite all) and server (active + draft) runs exit 0, while unexpected regressions and stale entries still fail. Adds a server: section and groups entries by SEP: - client: SEP-2575, SEP-2322, SEP-2243, SEP-2106, SEP-2468, SEP-2352 draft scenarios, plus pre-existing auth scenarios that now fail the new SEP-837 application_type check and the SEP-990 enterprise-managed-auth extension. - server: SEP-2575 stateless, SEP-2322 input-required-result, SEP-2549 caching, SEP-2243 header validation, plus three WARNING-only entries (SEP-2164 / SEP-2322 SHOULD-level checks). Each entry must be removed as the corresponding feature lands (the runner fails on stale entries), so the baseline burns down per milestone. --- test/conformance/expected-failures.yaml | 91 ++++++++++++++++++++++++- 1 file changed, 90 insertions(+), 1 deletion(-) diff --git a/test/conformance/expected-failures.yaml b/test/conformance/expected-failures.yaml index 5f4de2208..f4b7ce421 100644 --- a/test/conformance/expected-failures.yaml +++ b/test/conformance/expected-failures.yaml @@ -1,4 +1,93 @@ # Conformance scenarios not yet implemented in the v2 TypeScript SDK. # CI exits 0 if only these fail, exits 1 on unexpected failures or stale entries. +# +# Baseline established against @modelcontextprotocol/conformance 0.2.0-alpha.1, +# which adds the draft-spec scenario suite (SEP-2575, SEP-2322, SEP-2243, SEP-2549, +# SEP-2468, ...) plus new checks on existing scenarios (SEP-837 application_type +# during DCR). +# +# Entries are grouped by SEP. As each SEP/milestone is implemented in the SDK the +# corresponding scenarios start passing and MUST be removed from this list (the +# runner fails on stale entries), so the baseline burns down per milestone. -client: [] +client: + # --- Draft-spec scenarios (in `--suite draft`, also part of `--suite all`) --- + # SEP-2575 (request metadata / _meta envelope): client does not populate the + # _meta envelope or the MCP-Protocol-Version header semantics yet. + - request-metadata + # SEP-2322 (multi-round-trip requests): client does not echo requestState / + # handle IncompleteResult yet. + - sep-2322-client-request-state + # SEP-2243 (HTTP standardization): no fixture handler / client header support yet. + - http-custom-headers + - http-invalid-tool-headers + # SEP-2106 (JSON Schema $ref handling): client still dereferences network $refs. + - json-schema-ref-no-deref + # SEP-2468 (authorization response iss parameter): not implemented in the client. + - auth/iss-supported + - auth/iss-not-advertised + - auth/iss-supported-missing + - auth/iss-wrong-issuer + - auth/iss-unexpected + - auth/iss-normalized + - auth/metadata-issuer-mismatch + # SEP-2352 (authorization server migration): client does not re-register when + # PRM authorization_servers changes. + - auth/authorization-server-migration + # SEP-2207 (offline_access scope) scenario, currently failing only on the new + # SEP-837 application_type check (see the SEP-837 group below). + - auth/offline-access-not-supported + + # --- Pre-existing scenarios that fail on checks added after conformance 0.1.15 --- + # SEP-837: client MUST send application_type during Dynamic Client Registration. + # Single new check; everything else in these scenarios passes. + - auth/metadata-default + - auth/metadata-var1 + - auth/metadata-var2 + - auth/metadata-var3 + - auth/scope-from-www-authenticate + - auth/scope-from-scopes-supported + - auth/scope-omitted-when-undefined + - auth/scope-step-up + - auth/scope-retry-limit + - auth/token-endpoint-auth-basic + - auth/token-endpoint-auth-post + - auth/token-endpoint-auth-none + - auth/2025-03-26-oauth-metadata-backcompat + # SEP-990 (enterprise-managed authorization extension): no fixture handler / + # client support for the token-exchange + JWT bearer flow. + - auth/enterprise-managed-authorization + +server: + # --- Draft-spec scenarios (in `--suite draft`; the default `active` suite is green) --- + # SEP-2575 (stateless HTTP / _meta envelope): server has no stateless mode, + # _meta-derived capabilities, error-code mappings, or server/discover yet. + - server-stateless + # SEP-2322 (multi-round-trip requests / IncompleteResult): not implemented; + # most scenarios currently fail early with "Session ID required" because the + # fixture only runs in stateful mode. + - input-required-result-basic-elicitation + - input-required-result-basic-sampling + - input-required-result-basic-list-roots + - input-required-result-request-state + - input-required-result-multiple-input-requests + - input-required-result-multi-round + - input-required-result-non-tool-request + - input-required-result-result-type + - input-required-result-tampered-state + - input-required-result-capability-check + # SEP-2549 (caching): no ttlMs/cacheScope support; scenario also hits the + # stateful-mode "Session ID required" error. + - caching + # SEP-2243 (HTTP header standardization): -32001 HeaderMismatch handling and + # case-insensitive/whitespace-trimmed header validation not implemented. + - http-header-validation + - http-custom-header-server-validation + # WARNING-only entries: these scenarios emit no FAILURE checks, only SHOULD-level + # WARNINGs, but the expected-failures evaluator counts WARNINGs as failures. + # SEP-2164: server returns -32002 without the requested URI in error.data. + - sep-2164-resource-not-found + # SEP-2322 SHOULD-level behaviours (re-request missing inputResponses, ignore + # unrecognized inputResponses keys). + - input-required-result-missing-input-response + - input-required-result-ignore-extra-params From fdd11a7d9b3e6f0dbaaa31b94230a3efda639623 Mon Sep 17 00:00:00 2001 From: Felix Weinberger Date: Sat, 30 May 2026 08:20:15 +0000 Subject: [PATCH 3/6] ci(conformance): run the draft-spec server suite Add a workflow step that runs the test:conformance:server:draft script after the existing active-suite step. The client job already covers the draft client scenarios via --suite all; add comments documenting that and the exact-version conformance pin. Both jobs intentionally stay continue-on-error: true for now; the comment in the workflow recommends removing it once the expected-failures baseline has proven stable so that unexpected failures and stale entries actually block. --- .github/workflows/conformance.yml | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 2179f0272..9873b2ee9 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -1,5 +1,14 @@ name: Conformance Tests +# The conformance framework (@modelcontextprotocol/conformance) is pinned to an +# exact version in test/conformance/package.json; new conformance releases are +# adopted by deliberately bumping that pin (+ pnpm-lock.yaml). This workflow +# stays as-is across bumps. +# +# Both jobs are still continue-on-error: true (advisory). Recommended next +# step once the expected-failures baseline has proven stable: remove +# continue-on-error so unexpected failures and stale baseline entries block. + on: push: branches: [main] @@ -30,7 +39,11 @@ jobs: cache-dependency-path: pnpm-lock.yaml - run: pnpm install - run: pnpm run build:all - - run: pnpm run test:conformance:client:all + # --suite all already includes the draft-spec client scenarios + # (SEP-2575, SEP-2322, SEP-2243, SEP-2106, draft auth); known gaps + # are listed in test/conformance/expected-failures.yaml. + - name: Run client conformance (all suites, incl. draft) + run: pnpm run test:conformance:client:all server-conformance: runs-on: ubuntu-latest @@ -48,4 +61,11 @@ jobs: cache-dependency-path: pnpm-lock.yaml - run: pnpm install - run: pnpm run build:all - - run: pnpm run test:conformance:server + # Dated-spec (active) suite: expected to stay fully green. + - name: Run server conformance (active suite) + run: pnpm run test:conformance:server + # Draft-spec suite (SEP-2575 stateless, SEP-2322 MRTR, SEP-2549, + # SEP-2243): known gaps are listed in expected-failures.yaml and + # must be removed from the baseline as each milestone lands. + - name: Run server conformance (draft suite) + run: pnpm --filter @modelcontextprotocol/test-conformance run test:conformance:server:draft From 32792009c4b80e30a62b5cf2da145480874fc620 Mon Sep 17 00:00:00 2001 From: Felix Weinberger Date: Mon, 1 Jun 2026 12:30:08 +0000 Subject: [PATCH 4/6] ci(conformance): make conformance jobs gating The expected-failures baseline is the failure-control mechanism: failures in the baseline pass, unexpected failures and stale baseline entries fail. With that in place, continue-on-error made regressions invisible. --- .github/workflows/conformance.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 9873b2ee9..6f2846648 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -5,9 +5,9 @@ name: Conformance Tests # adopted by deliberately bumping that pin (+ pnpm-lock.yaml). This workflow # stays as-is across bumps. # -# Both jobs are still continue-on-error: true (advisory). Recommended next -# step once the expected-failures baseline has proven stable: remove -# continue-on-error so unexpected failures and stale baseline entries block. +# Both jobs are gating: the expected-failures baseline determines pass/fail. +# A scenario failing outside the baseline, or a stale baseline entry (scenario +# now passing but still listed), fails the job. on: push: @@ -25,7 +25,6 @@ permissions: jobs: client-conformance: runs-on: ubuntu-latest - continue-on-error: true steps: - uses: actions/checkout@v6 - name: Install pnpm @@ -47,7 +46,6 @@ jobs: server-conformance: runs-on: ubuntu-latest - continue-on-error: true steps: - uses: actions/checkout@v6 - name: Install pnpm From ad0f292b1870a0e30ec1e090c33fa7b6564c46c6 Mon Sep 17 00:00:00 2001 From: Felix Weinberger Date: Mon, 1 Jun 2026 12:57:48 +0000 Subject: [PATCH 5/6] ci(conformance): keep workflow steps minimal (bare run commands, no step names or comments) --- .github/workflows/conformance.yml | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 6f2846648..28a242099 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -38,11 +38,7 @@ jobs: cache-dependency-path: pnpm-lock.yaml - run: pnpm install - run: pnpm run build:all - # --suite all already includes the draft-spec client scenarios - # (SEP-2575, SEP-2322, SEP-2243, SEP-2106, draft auth); known gaps - # are listed in test/conformance/expected-failures.yaml. - - name: Run client conformance (all suites, incl. draft) - run: pnpm run test:conformance:client:all + - run: pnpm run test:conformance:client:all server-conformance: runs-on: ubuntu-latest @@ -59,11 +55,5 @@ jobs: cache-dependency-path: pnpm-lock.yaml - run: pnpm install - run: pnpm run build:all - # Dated-spec (active) suite: expected to stay fully green. - - name: Run server conformance (active suite) - run: pnpm run test:conformance:server - # Draft-spec suite (SEP-2575 stateless, SEP-2322 MRTR, SEP-2549, - # SEP-2243): known gaps are listed in expected-failures.yaml and - # must be removed from the baseline as each milestone lands. - - name: Run server conformance (draft suite) - run: pnpm --filter @modelcontextprotocol/test-conformance run test:conformance:server:draft + - run: pnpm run test:conformance:server + - run: pnpm --filter @modelcontextprotocol/test-conformance run test:conformance:server:draft From e9e6a0f1c02052b01cab4fd870b2924c485549e7 Mon Sep 17 00:00:00 2001 From: Felix Weinberger Date: Mon, 1 Jun 2026 14:00:50 +0000 Subject: [PATCH 6/6] ci(conformance): use root script alias for the draft suite; drop workflow header comment --- .github/workflows/conformance.yml | 11 +---------- package.json | 1 + 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 28a242099..0deab5448 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -1,14 +1,5 @@ name: Conformance Tests -# The conformance framework (@modelcontextprotocol/conformance) is pinned to an -# exact version in test/conformance/package.json; new conformance releases are -# adopted by deliberately bumping that pin (+ pnpm-lock.yaml). This workflow -# stays as-is across bumps. -# -# Both jobs are gating: the expected-failures baseline determines pass/fail. -# A scenario failing outside the baseline, or a stale baseline entry (scenario -# now passing but still listed), fails the job. - on: push: branches: [main] @@ -56,4 +47,4 @@ jobs: - run: pnpm install - run: pnpm run build:all - run: pnpm run test:conformance:server - - run: pnpm --filter @modelcontextprotocol/test-conformance run test:conformance:server:draft + - run: pnpm run test:conformance:server:draft diff --git a/package.json b/package.json index a2cb93f62..d1ecc0c62 100644 --- a/package.json +++ b/package.json @@ -40,6 +40,7 @@ "test:conformance:client:all": "pnpm --filter @modelcontextprotocol/test-conformance run test:conformance:client:all", "test:conformance:client:run": "pnpm --filter @modelcontextprotocol/test-conformance run test:conformance:client:run", "test:conformance:server": "pnpm --filter @modelcontextprotocol/test-conformance run test:conformance:server", + "test:conformance:server:draft": "pnpm --filter @modelcontextprotocol/test-conformance run test:conformance:server:draft", "test:conformance:server:all": "pnpm --filter @modelcontextprotocol/test-conformance run test:conformance:server:all", "test:conformance:server:run": "pnpm --filter @modelcontextprotocol/test-conformance run test:conformance:server:run", "test:conformance:all": "pnpm run test:conformance:client:all && pnpm run test:conformance:server:all"