harness: label provider conformance phases

internal/harness/provider-conformance/README.md

@@ -20,7 +20,11 @@ agent test and the harnesses in `internal/harness`:
   preserving run metadata.
 
 The command also emits the registered provider capability matrix so the run shows
-which providers advertise model, image, video, and streaming support.
+which providers advertise model, image, video, and streaming support. Console output
+and summary artifacts label each harness with the phase it is proving (for example,
+model call + tool call, workflow event + tool call, or streaming fallback + tool
+call), so provider failures identify the failed lifecycle phase instead of only the
+provider name.
 
 ## Local usage
 

internal/harness/provider-conformance/main.go

@@ -36,6 +36,14 @@ import (
 
 const defaultHarnesses = "agent,universe,agent-flow,plan-delegate,a2a-stream-fallback"
 
+var harnessPhases = map[string]string{
+	"agent":               "model call + tool call",
+	"universe":            "service discovery + tool call",
+	"agent-flow":          "workflow event + tool call",
+	"plan-delegate":       "plan persistence + delegation + tool call",
+	"a2a-stream-fallback": "streaming fallback + tool call",
+}
+
 var providerEnv = map[string]string{
 	"anthropic":  "ANTHROPIC_API_KEY",
 	"openai":     "OPENAI_API_KEY",
@@ -92,15 +100,16 @@ func main() {
 		}
 
 		for _, harness := range harnesses {
-			fmt.Printf("\n==> %s / %s\n", provider, harness)
+			phase := harnessPhase(harness)
+			fmt.Printf("\n==> %s / %s (%s)\n", provider, harness, phase)
 			if err := runHarness(provider, harness, *timeoutFlag); err != nil {
-				fmt.Printf("FAIL %s / %s: %v\n", provider, harness, err)
+				fmt.Printf("FAIL %s / %s (%s): %v\n", provider, harness, phase, err)
 				failed++
-				results = append(results, conformanceResult{Provider: provider, Harness: harness, Status: statusFailed, Error: err.Error()})
+				results = append(results, conformanceResult{Provider: provider, Harness: harness, Phase: phase, Status: statusFailed, Error: err.Error()})
 				continue
 			}
 			ran++
-			results = append(results, conformanceResult{Provider: provider, Harness: harness, Status: statusPassed})
+			results = append(results, conformanceResult{Provider: provider, Harness: harness, Phase: phase, Status: statusPassed})
 		}
 	}
 
@@ -140,6 +149,7 @@ const (
 type conformanceResult struct {
 	Provider string `json:"provider"`
 	Harness  string `json:"harness,omitempty"`
+	Phase    string `json:"phase,omitempty"`
 	Status   string `json:"status"`
 	Error    string `json:"error,omitempty"`
 }
@@ -176,14 +186,18 @@ func writeSummaryMarkdown(path string, summary conformanceSummary) error {
 	b.WriteString("## Capability matrix\n\n")
 	b.WriteString(capabilityMarkdown(summary.Capabilities))
 	b.WriteString("\n## Harness results\n\n")
-	b.WriteString("| Provider | Harness | Status | Detail |\n")
-	b.WriteString("| --- | --- | --- | --- |\n")
+	b.WriteString("| Provider | Harness | Phase | Status | Detail |\n")
+	b.WriteString("| --- | --- | --- | --- | --- |\n")
 	for _, result := range summary.Results {
 		harness := result.Harness
 		if harness == "" {
 			harness = "—"
 		}
-		fmt.Fprintf(&b, "| %s | %s | %s | %s |\n", result.Provider, harness, result.Status, markdownCell(result.Error))
+		phase := result.Phase
+		if phase == "" {
+			phase = "—"
+		}
+		fmt.Fprintf(&b, "| %s | %s | %s | %s | %s |\n", result.Provider, harness, markdownCell(phase), result.Status, markdownCell(result.Error))
 	}
 	return os.WriteFile(path, []byte(b.String()), 0o644)
 }
@@ -209,6 +223,13 @@ func markdownList(values []string) string {
 	return strings.Join(escaped, ", ")
 }
 
+func harnessPhase(harness string) string {
+	if phase, ok := harnessPhases[harness]; ok {
+		return phase
+	}
+	return "harness"
+}
+
 func markdownCell(s string) string {
 	if s == "" {
 		return "—"

internal/harness/provider-conformance/main_test.go

@@ -99,7 +99,7 @@ func TestWriteSummaryMarkdown(t *testing.T) {
 	summary := conformanceSummary{
 		Capabilities: []ai.CapabilityRow{{Provider: "mock", Capabilities: ai.Capabilities{Model: true}}},
 		Results: []conformanceResult{
-			{Provider: "mock", Harness: "agent-flow", Status: statusPassed},
+			{Provider: "mock", Harness: "agent-flow", Phase: harnessPhase("agent-flow"), Status: statusPassed},
 			{Provider: "live", Status: statusSkipped, Error: "missing | key"},
 		},
 		Passed:  1,
@@ -120,8 +120,8 @@ func TestWriteSummaryMarkdown(t *testing.T) {
 		"Providers: —.",
 		"Harnesses: —.",
 		"| mock | ✅ | — | — | — |",
-		"| mock | agent-flow | passed | — |",
-		"| live | — | skipped | missing \\| key |",
+		"| mock | agent-flow | workflow event + tool call | passed | — |",
+		"| live | — | — | skipped | missing \\| key |",
 	} {
 		if !strings.Contains(got, want) {
 			t.Fatalf("summary markdown = %q, want %q", got, want)
@@ -129,6 +129,15 @@ func TestWriteSummaryMarkdown(t *testing.T) {
 	}
 }
 
+func TestHarnessPhaseLabelsKnownHarnesses(t *testing.T) {
+	if got := harnessPhase("a2a-stream-fallback"); got != "streaming fallback + tool call" {
+		t.Fatalf("harnessPhase() = %q, want streaming fallback phase", got)
+	}
+	if got := harnessPhase("custom"); got != "harness" {
+		t.Fatalf("harnessPhase(custom) = %q, want fallback phase", got)
+	}
+}
+
 func TestMarkdownListEscapesBackticks(t *testing.T) {
 	got := markdownList([]string{"agent", "bad`name"})
 	want := "`agent`, `bad\\`name`"