name: Conformance Tests

on:

push:

branches: [main]

pull_request:

workflow_dispatch:

concurrency:

group: conformance-${{ github.ref }}

cancel-in-progress: true

permissions:

contents: read

env:

# Pinned conformance harness package spec (passed verbatim to `npx --yes`).

# Bump deliberately and reconcile the three

# .github/actions/conformance/expected-failures*.yml files in the same change.

CONFORMANCE_PKG: "@modelcontextprotocol/conformance@0.2.0-alpha.11"

# Each job runs three `--suite all` legs, one per baseline file. The two

# `--spec-version <rev>` legs run every scenario the pinned harness marks

# applicable at that spec revision, at that revision's wire (stateful

# handshake for 2025-11-25, stateless per-request _meta for 2026-07-28). Each

# is a superset of the frozen requirement set that `conformance tier-check`

# scores for that revision (`npx $CONFORMANCE_PKG list --requirements <rev>`

# prints it), so an expected-failures.<rev>.yml entry for a scenario in that

# set means tier-check reports the SDK below 100% for <rev>. The bare leg runs

# every shipped scenario at whichever wire the harness defaults it to; it is

# the only leg that reaches extension-tagged scenarios (tasks-*, auth/dpop*,

# ...), which never match a --spec-version filter. For everything else it

# overlaps one of the pinned legs, so a failure it also sees needs its own

# expected-failures.yml entry besides the pinned file's.

jobs:

server-conformance:

runs-on: ubuntu-latest

timeout-minutes: 15

steps:

- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3

with:

persist-credentials: false

- uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0

with:

enable-cache: true

version: 0.9.5

- uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0

with:

node-version: 24

- run: uv sync --frozen --all-extras --package mcp-everything-server

- name: Run server conformance (all suite, 2025-11-25 wire)

run: >-

./.github/actions/conformance/run-server.sh

--suite all

--spec-version 2025-11-25

--expected-failures ./.github/actions/conformance/expected-failures.2025-11-25.yml

--output-dir conformance-results/server-2025-11-25

- name: Run server conformance (all suite, 2026-07-28 wire)

run: >-

./.github/actions/conformance/run-server.sh

--suite all

--spec-version 2026-07-28

--expected-failures ./.github/actions/conformance/expected-failures.2026-07-28.yml

--output-dir conformance-results/server-2026-07-28

- name: Run server conformance (all suite, default wire)

run: >-

./.github/actions/conformance/run-server.sh

--suite all

--expected-failures ./.github/actions/conformance/expected-failures.yml

--output-dir conformance-results/server-all

- name: Upload conformance results

# The log has only summary counts; per-check data is in checks.json.

# always(), not !cancelled(): a timeout-minutes kill is a cancellation,

# and that is when the partial results matter most.

if: always()

uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1

with:

name: server-conformance-results

path: conformance-results/

if-no-files-found: ignore

client-conformance:

runs-on: ubuntu-latest

timeout-minutes: 15

steps:

- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3

with:

persist-credentials: false

- uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0

with:

enable-cache: true

version: 0.9.5

- uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0

with:

node-version: 24

# --compile-bytecode: without it, ~40 concurrently spawned interpreters

# race to byte-compile site-packages during the timing-sensitive window.

- run: uv sync --frozen --all-extras --package mcp --compile-bytecode

- name: Pre-compile bytecode (editable sources)

run: uv run --frozen python -m compileall -q src .github/actions/conformance

# The harness runs a leg's scenarios via unbounded Promise.all; under

# that concurrency the slowest one (sse-retry, which has a real-time SSE

# reconnect wait) needs more than the 30s default budget.

# `.venv/bin/python` (not `uv run`) avoids lockfile re-checks in ~40

# concurrent spawns; run-client.sh re-runs unexpected failures solo.

- name: Run client conformance (all suite, 2025-11-25 wire)

run: >-

./.github/actions/conformance/run-client.sh

--command '.venv/bin/python .github/actions/conformance/client.py'

--suite all

--timeout 60000

--spec-version 2025-11-25

--expected-failures ./.github/actions/conformance/expected-failures.2025-11-25.yml

--output-dir conformance-results/client-2025-11-25

- name: Run client conformance (all suite, 2026-07-28 wire)

run: >-

./.github/actions/conformance/run-client.sh

--command '.venv/bin/python .github/actions/conformance/client.py'

--suite all

--timeout 60000

--spec-version 2026-07-28

--expected-failures ./.github/actions/conformance/expected-failures.2026-07-28.yml

--output-dir conformance-results/client-2026-07-28

- name: Run client conformance (all suite, default wire)

run: >-

./.github/actions/conformance/run-client.sh

--command '.venv/bin/python .github/actions/conformance/client.py'

--suite all

--timeout 60000

--expected-failures ./.github/actions/conformance/expected-failures.yml

--output-dir conformance-results/client-all

- name: Upload conformance results

# The log has only summary counts; per-check data is in checks.json.

# always(), not !cancelled(): a timeout-minutes kill is a cancellation,

# and that is when the partial results matter most.

if: always()

uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1

with:

name: client-conformance-results

path: conformance-results/

if-no-files-found: ignore