name: Conformance Tests on: push: branches: [main] pull_request: workflow_dispatch: concurrency: group: conformance-${{ github.ref }} cancel-in-progress: true permissions: contents: read env: # Pinned conformance harness package spec (passed verbatim to `npx --yes`). # Bump deliberately and reconcile the three # .github/actions/conformance/expected-failures*.yml files in the same change. CONFORMANCE_PKG: "@modelcontextprotocol/conformance@0.2.0-alpha.11" # Each job runs three `--suite all` legs, one per baseline file. The two # `--spec-version ` legs run every scenario the pinned harness marks # applicable at that spec revision, at that revision's wire (stateful # handshake for 2025-11-25, stateless per-request _meta for 2026-07-28). Each # is a superset of the frozen requirement set that `conformance tier-check` # scores for that revision (`npx $CONFORMANCE_PKG list --requirements ` # prints it), so an expected-failures..yml entry for a scenario in that # set means tier-check reports the SDK below 100% for . The bare leg runs # every shipped scenario at whichever wire the harness defaults it to; it is # the only leg that reaches extension-tagged scenarios (tasks-*, auth/dpop*, # ...), which never match a --spec-version filter. For everything else it # overlaps one of the pinned legs, so a failure it also sees needs its own # expected-failures.yml entry besides the pinned file's. jobs: server-conformance: runs-on: ubuntu-latest timeout-minutes: 15 steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 with: enable-cache: true version: 0.9.5 - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: node-version: 24 - run: uv sync --frozen --all-extras --package mcp-everything-server - name: Run server conformance (all suite, 2025-11-25 wire) run: >- ./.github/actions/conformance/run-server.sh --suite all --spec-version 2025-11-25 --expected-failures ./.github/actions/conformance/expected-failures.2025-11-25.yml --output-dir conformance-results/server-2025-11-25 - name: Run server conformance (all suite, 2026-07-28 wire) run: >- ./.github/actions/conformance/run-server.sh --suite all --spec-version 2026-07-28 --expected-failures ./.github/actions/conformance/expected-failures.2026-07-28.yml --output-dir conformance-results/server-2026-07-28 - name: Run server conformance (all suite, default wire) run: >- ./.github/actions/conformance/run-server.sh --suite all --expected-failures ./.github/actions/conformance/expected-failures.yml --output-dir conformance-results/server-all - name: Upload conformance results # The log has only summary counts; per-check data is in checks.json. # always(), not !cancelled(): a timeout-minutes kill is a cancellation, # and that is when the partial results matter most. if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: server-conformance-results path: conformance-results/ if-no-files-found: ignore client-conformance: runs-on: ubuntu-latest timeout-minutes: 15 steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 with: enable-cache: true version: 0.9.5 - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: node-version: 24 # --compile-bytecode: without it, ~40 concurrently spawned interpreters # race to byte-compile site-packages during the timing-sensitive window. - run: uv sync --frozen --all-extras --package mcp --compile-bytecode - name: Pre-compile bytecode (editable sources) run: uv run --frozen python -m compileall -q src .github/actions/conformance # The harness runs a leg's scenarios via unbounded Promise.all; under # that concurrency the slowest one (sse-retry, which has a real-time SSE # reconnect wait) needs more than the 30s default budget. # `.venv/bin/python` (not `uv run`) avoids lockfile re-checks in ~40 # concurrent spawns; run-client.sh re-runs unexpected failures solo. - name: Run client conformance (all suite, 2025-11-25 wire) run: >- ./.github/actions/conformance/run-client.sh --command '.venv/bin/python .github/actions/conformance/client.py' --suite all --timeout 60000 --spec-version 2025-11-25 --expected-failures ./.github/actions/conformance/expected-failures.2025-11-25.yml --output-dir conformance-results/client-2025-11-25 - name: Run client conformance (all suite, 2026-07-28 wire) run: >- ./.github/actions/conformance/run-client.sh --command '.venv/bin/python .github/actions/conformance/client.py' --suite all --timeout 60000 --spec-version 2026-07-28 --expected-failures ./.github/actions/conformance/expected-failures.2026-07-28.yml --output-dir conformance-results/client-2026-07-28 - name: Run client conformance (all suite, default wire) run: >- ./.github/actions/conformance/run-client.sh --command '.venv/bin/python .github/actions/conformance/client.py' --suite all --timeout 60000 --expected-failures ./.github/actions/conformance/expected-failures.yml --output-dir conformance-results/client-all - name: Upload conformance results # The log has only summary counts; per-check data is in checks.json. # always(), not !cancelled(): a timeout-minutes kill is a cancellation, # and that is when the partial results matter most. if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: client-conformance-results path: conformance-results/ if-no-files-found: ignore