modelcontextprotocol--python-sdk/.github/workflows/conformance.yml

name: Conformance Tests

on:
  push:
    branches: [main]
  pull_request:
  workflow_dispatch:

concurrency:
  group: conformance-${{ github.ref }}
  cancel-in-progress: true

permissions:
  contents: read

env:
  # Pinned conformance harness package spec (passed verbatim to `npx --yes`).
  # Bump deliberately and reconcile the three
  # .github/actions/conformance/expected-failures*.yml files in the same change.
  CONFORMANCE_PKG: "@modelcontextprotocol/conformance@0.2.0-alpha.11"

# Each job runs three `--suite all` legs, one per baseline file. The two
# `--spec-version <rev>` legs run every scenario the pinned harness marks
# applicable at that spec revision, at that revision's wire (stateful
# handshake for 2025-11-25, stateless per-request _meta for 2026-07-28). Each
# is a superset of the frozen requirement set that `conformance tier-check`
# scores for that revision (`npx $CONFORMANCE_PKG list --requirements <rev>`
# prints it), so an expected-failures.<rev>.yml entry for a scenario in that
# set means tier-check reports the SDK below 100% for <rev>. The bare leg runs
# every shipped scenario at whichever wire the harness defaults it to; it is
# the only leg that reaches extension-tagged scenarios (tasks-*, auth/dpop*,
# ...), which never match a --spec-version filter. For everything else it
# overlaps one of the pinned legs, so a failure it also sees needs its own
# expected-failures.yml entry besides the pinned file's.

jobs:
  server-conformance:
    runs-on: ubuntu-latest
    timeout-minutes: 15
    steps:
      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
        with:
          persist-credentials: false
      - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
        with:
          enable-cache: true
          version: 0.9.5
      - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
        with:
          node-version: 24
      - run: uv sync --frozen --all-extras --package mcp-everything-server
      - name: Run server conformance (all suite, 2025-11-25 wire)
        run: >-
          ./.github/actions/conformance/run-server.sh
          --suite all
          --spec-version 2025-11-25
          --expected-failures ./.github/actions/conformance/expected-failures.2025-11-25.yml
          --output-dir conformance-results/server-2025-11-25
      - name: Run server conformance (all suite, 2026-07-28 wire)
        run: >-
          ./.github/actions/conformance/run-server.sh
          --suite all
          --spec-version 2026-07-28
          --expected-failures ./.github/actions/conformance/expected-failures.2026-07-28.yml
          --output-dir conformance-results/server-2026-07-28
      - name: Run server conformance (all suite, default wire)
        run: >-
          ./.github/actions/conformance/run-server.sh
          --suite all
          --expected-failures ./.github/actions/conformance/expected-failures.yml
          --output-dir conformance-results/server-all
      - name: Upload conformance results
        # The log has only summary counts; per-check data is in checks.json.
        # always(), not !cancelled(): a timeout-minutes kill is a cancellation,
        # and that is when the partial results matter most.
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: server-conformance-results
          path: conformance-results/
          if-no-files-found: ignore

  client-conformance:
    runs-on: ubuntu-latest
    timeout-minutes: 15
    steps:
      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
        with:
          persist-credentials: false
      - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
        with:
          enable-cache: true
          version: 0.9.5
      - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
        with:
          node-version: 24
      # --compile-bytecode: without it, ~40 concurrently spawned interpreters
      # race to byte-compile site-packages during the timing-sensitive window.
      - run: uv sync --frozen --all-extras --package mcp --compile-bytecode
      - name: Pre-compile bytecode (editable sources)
        run: uv run --frozen python -m compileall -q src .github/actions/conformance
      # The harness runs a leg's scenarios via unbounded Promise.all; under
      # that concurrency the slowest one (sse-retry, which has a real-time SSE
      # reconnect wait) needs more than the 30s default budget.
      # `.venv/bin/python` (not `uv run`) avoids lockfile re-checks in ~40
      # concurrent spawns; run-client.sh re-runs unexpected failures solo.
      - name: Run client conformance (all suite, 2025-11-25 wire)
        run: >-
          ./.github/actions/conformance/run-client.sh
          --command '.venv/bin/python .github/actions/conformance/client.py'
          --suite all
          --timeout 60000
          --spec-version 2025-11-25
          --expected-failures ./.github/actions/conformance/expected-failures.2025-11-25.yml
          --output-dir conformance-results/client-2025-11-25
      - name: Run client conformance (all suite, 2026-07-28 wire)
        run: >-
          ./.github/actions/conformance/run-client.sh
          --command '.venv/bin/python .github/actions/conformance/client.py'
          --suite all
          --timeout 60000
          --spec-version 2026-07-28
          --expected-failures ./.github/actions/conformance/expected-failures.2026-07-28.yml
          --output-dir conformance-results/client-2026-07-28
      - name: Run client conformance (all suite, default wire)
        run: >-
          ./.github/actions/conformance/run-client.sh
          --command '.venv/bin/python .github/actions/conformance/client.py'
          --suite all
          --timeout 60000
          --expected-failures ./.github/actions/conformance/expected-failures.yml
          --output-dir conformance-results/client-all
      - name: Upload conformance results
        # The log has only summary counts; per-check data is in checks.json.
        # always(), not !cancelled(): a timeout-minutes kill is a cancellation,
        # and that is when the partial results matter most.
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: client-conformance-results
          path: conformance-results/
          if-no-files-found: ignore