De-flake conformance CI: solo re-verification, spawn-storm reduction, result artifacts (#3043)

.github/actions/conformance/run-client.sh

@@ -0,0 +1,104 @@
+#!/bin/bash
+# Run a client conformance suite, re-verifying unexpected failures solo.
+# Concurrent suite runs on a 2-vCPU runner can push scenarios with real-time
+# waits past tolerance; solo, a real failure fails again while a contention
+# artifact passes. Failures that only reproduce under concurrency are excused.
+set -uo pipefail
+
+: "${CONFORMANCE_PKG:?set CONFORMANCE_PKG (pinned in .github/workflows/conformance.yml)}"
+# One attempt: a solo failure on the quiet runner disproves the contention
+# hypothesis; a second try would be the blind retry this script avoids.
+SOLO_ATTEMPTS="${CONFORMANCE_SOLO_ATTEMPTS:-1}"
+
+# Relative args resolve from the repo root; same contract as run-server.sh.
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+cd "$SCRIPT_DIR/../../.." || exit 1
+
+log="$(mktemp)"
+trap 'rm -f "$log"' EXIT
+
+npx --yes "$CONFORMANCE_PKG" client "$@" 2>&1 | tee "$log"
+rc=${PIPESTATUS[0]}
+if [ "$rc" -eq 0 ]; then
+    exit 0
+fi
+
+plain="$(sed 's/\x1b\[[0-9;]*m//g' "$log")"
+
+# If the harness's summary wording changes, the list comes up empty and the
+# original exit code passes through - never a false green.
+mapfile -t scenarios < <(
+    printf '%s\n' "$plain" |
+        sed -n '/^Unexpected failures (not in baseline):$/,/^$/p' |
+        sed -n 's/^  ✗ //p'
+)
+if [ "${#scenarios[@]}" -eq 0 ]; then
+    exit "$rc"
+fi
+for scenario in "${scenarios[@]}"; do
+    if ! [[ "$scenario" =~ ^[A-Za-z0-9/_-]+$ ]]; then
+        echo "Extracted unexpected-failure name '${scenario}' does not look like a scenario name; passing the suite failure through." >&2
+        exit "$rc"
+    fi
+done
+
+# A stale baseline entry is a configuration error a solo rerun cannot excuse.
+# Here-string, not a pipe: grep -q quitting early would SIGPIPE printf and,
+# under pipefail, skip this guard exactly when the pattern is present.
+if grep -q '^Stale baseline entries' <<<"$plain"; then
+    echo "Suite also reported stale baseline entries; not retrying." >&2
+    exit "$rc"
+fi
+
+# Drop the suite-only flags: --scenario replaces --suite, and solo runs are
+# judged directly rather than against the baseline.
+rerun_args=()
+output_dir=""
+skip_next=0
+expect_output_dir=0
+for arg in "$@"; do
+    if [ "$skip_next" -eq 1 ]; then
+        if [ "$expect_output_dir" -eq 1 ]; then
+            output_dir="$arg"
+        fi
+        skip_next=0
+        expect_output_dir=0
+        continue
+    fi
+    case "$arg" in
+    --output-dir)
+        skip_next=1
+        expect_output_dir=1
+        ;;
+    --suite | --expected-failures) skip_next=1 ;;
+    --output-dir=*) output_dir="${arg#--output-dir=}" ;;
+    --suite=* | --expected-failures=*) ;;
+    *) rerun_args+=("$arg") ;;
+    esac
+done
+if [ -n "$output_dir" ]; then
+    rerun_args+=(--output-dir "${output_dir}-solo")
+fi
+
+for scenario in "${scenarios[@]}"; do
+    passed=0
+    for attempt in $(seq 1 "$SOLO_ATTEMPTS"); do
+        echo ""
+        echo "Re-running '${scenario}' solo (attempt ${attempt}/${SOLO_ATTEMPTS})..."
+        if npx --yes "$CONFORMANCE_PKG" client --scenario "$scenario" "${rerun_args[@]}"; then
+            passed=1
+            break
+        fi
+    done
+    if [ "$passed" -ne 1 ]; then
+        echo "'${scenario}' still fails when run alone: real failure, not suite contention." >&2
+        exit 1
+    fi
+done
+
+if [ -n "$output_dir" ]; then
+    mkdir -p "$output_dir"
+    printf '%s\n' "${scenarios[@]}" > "$output_dir/FLAKE_RESCUED"
+fi
+echo "All ${#scenarios[@]} unexpected failure(s) passed when re-run solo; the suite failures were parallel-run contention."
+exit 0

.github/workflows/conformance.yml

@@ -64,17 +64,20 @@ jobs:
           ./.github/actions/conformance/run-server.sh
           --suite active
           --expected-failures ./.github/actions/conformance/expected-failures.yml
+          --output-dir conformance-results/server-active
       - name: Run server conformance (draft suite)
         run: >-
           ./.github/actions/conformance/run-server.sh
           --suite draft
           --expected-failures ./.github/actions/conformance/expected-failures.yml
+          --output-dir conformance-results/server-draft
       - name: Run server conformance (2026-07-28 wire, all suite)
         run: >-
           ./.github/actions/conformance/run-server.sh
           --suite all
           --spec-version 2026-07-28
           --expected-failures ./.github/actions/conformance/expected-failures.2026-07-28.yml
+          --output-dir conformance-results/server-2026-07-28
       - name: Run server conformance (all suite, extension scenarios)
         # A bare `--suite all` (no --spec-version) selects every scenario
         # shipped with the pinned harness — including the extension-tagged
@@ -91,6 +94,15 @@ jobs:
           ./.github/actions/conformance/run-server.sh
           --suite all
           --expected-failures ./.github/actions/conformance/expected-failures.yml
+          --output-dir conformance-results/server-all
+      - name: Upload conformance results
+        # The log has only summary counts; per-check data is in checks.json.
+        if: failure()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: server-conformance-results
+          path: conformance-results/
+          if-no-files-found: ignore
 
   client-conformance:
     runs-on: ubuntu-latest
@@ -118,22 +130,39 @@ jobs:
               echo "CONFORMANCE_PKG=file:/tmp/conformance.tgz" >> "$GITHUB_ENV"
               ;;
           esac
-      - run: uv sync --frozen --all-extras --package mcp
+      # --compile-bytecode: without it, ~40 concurrently spawned interpreters
+      # race to byte-compile site-packages during the timing-sensitive window.
+      - run: uv sync --frozen --all-extras --package mcp --compile-bytecode
+      - name: Pre-compile bytecode (editable sources)
+        run: uv run --frozen python -m compileall -q src .github/actions/conformance
       - name: Run client conformance (all suite)
         # The harness runs all scenarios via unbounded Promise.all; with 40
         # scenarios on a 2-core runner the slowest one (sse-retry, which has a
         # real-time SSE reconnect wait) needs more than the 30s default budget.
+        # `.venv/bin/python` (not `uv run`) avoids lockfile re-checks in ~40
+        # concurrent spawns; run-client.sh re-runs unexpected failures solo.
         run: >-
-          npx --yes "$CONFORMANCE_PKG" client
-          --command 'uv run --frozen python .github/actions/conformance/client.py'
+          ./.github/actions/conformance/run-client.sh
+          --command '.venv/bin/python .github/actions/conformance/client.py'
           --suite all
           --timeout 60000
           --expected-failures ./.github/actions/conformance/expected-failures.yml
+          --output-dir conformance-results/client-all
       - name: Run client conformance (2026-07-28 wire, all suite)
         run: >-
-          npx --yes "$CONFORMANCE_PKG" client
-          --command 'uv run --frozen python .github/actions/conformance/client.py'
+          ./.github/actions/conformance/run-client.sh
+          --command '.venv/bin/python .github/actions/conformance/client.py'
           --suite all
           --timeout 60000
           --spec-version 2026-07-28
           --expected-failures ./.github/actions/conformance/expected-failures.2026-07-28.yml
+          --output-dir conformance-results/client-2026-07-28
+      - name: Upload conformance results
+        # The log has only summary counts; per-check data is in checks.json.
+        # Also on FLAKE_RESCUED: rescued-flake evidence is otherwise discarded.
+        if: failure() || hashFiles('conformance-results/**/FLAKE_RESCUED') != ''
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: client-conformance-results
+          path: conformance-results/
+          if-no-files-found: ignore

.gitignore

@@ -173,3 +173,6 @@ cython_debug/
 
 # claude code
 results/
+
+# conformance CI local runs
+conformance-results/

examples/servers/everything-server/mcp_everything_server/server.py

@@ -192,10 +192,12 @@ async def test_tool_with_progress(ctx: Context) -> str:
 async def test_sampling(prompt: str, ctx: Context) -> str:
     """Tests server-initiated sampling (LLM completion request)"""
     try:
-        # Request sampling from client
+        # Request sampling from client. Without related_request_id the request goes
+        # to the standalone GET stream and is silently dropped if it is not open yet.
         result = await ctx.session.create_message(  # pyright: ignore[reportDeprecated]
             messages=[SamplingMessage(role="user", content=TextContent(type="text", text=prompt))],
             max_tokens=100,
+            related_request_id=ctx.request_id,
         )
 
         # Since we're not passing tools param, result.content is single content