Improve agent telemetry error classification

agent/checkpoint_test.go

@@ -13,7 +13,7 @@ import (
 
 func TestResumeCompletedCheckpointDoesNotReplayModel(t *testing.T) {
 	ctx := context.Background()
-	cp := flow.StoreCheckpoint(store.NewStore(), "durable-agent")
+	cp := flow.StoreCheckpoint(store.NewMemoryStore(), "durable-agent")
 	calls := 0
 	fakeGen = func(ctx context.Context, opts ai.Options, req *ai.Request) (*ai.Response, error) {
 		calls++
@@ -52,7 +52,7 @@ func TestResumeCompletedCheckpointDoesNotReplayModel(t *testing.T) {
 
 func TestResumeFailedCheckpointDoesNotReplayCompletedTool(t *testing.T) {
 	ctx := context.Background()
-	cp := flow.StoreCheckpoint(store.NewStore(), "tool-resume-agent")
+	cp := flow.StoreCheckpoint(store.NewMemoryStore(), "tool-resume-agent")
 	toolRuns := 0
 	first := true
 	fakeGen = func(ctx context.Context, opts ai.Options, req *ai.Request) (*ai.Response, error) {
@@ -170,7 +170,7 @@ func countMemoryContent(messages []ai.Message, needle string) int {
 
 func TestPendingReturnsUnfinishedAgentRuns(t *testing.T) {
 	ctx := context.Background()
-	cp := flow.StoreCheckpoint(store.NewStore(), "pending-agent")
+	cp := flow.StoreCheckpoint(store.NewMemoryStore(), "pending-agent")
 	run := flow.Run{ID: "run-1", Flow: "pending-agent", Status: "failed", State: flow.State{Stage: agentAskStep, Data: []byte("retry me")}}
 	if err := cp.Save(ctx, run); err != nil {
 		t.Fatalf("Save: %v", err)
@@ -187,7 +187,7 @@ func TestPendingReturnsUnfinishedAgentRuns(t *testing.T) {
 
 func TestPendingSkipsTerminalCanceledAndExpiredAgentRuns(t *testing.T) {
 	ctx := context.Background()
-	cp := flow.StoreCheckpoint(store.NewStore(), "terminal-agent")
+	cp := flow.StoreCheckpoint(store.NewMemoryStore(), "terminal-agent")
 	for _, run := range []flow.Run{
 		{ID: "active", Flow: "terminal-agent", Status: "failed", State: flow.State{Stage: agentAskStep, Data: []byte("retry me")}},
 		{ID: "done", Flow: "terminal-agent", Status: "done", State: flow.State{Stage: agentAskStep, Data: []byte("done")}},
@@ -216,7 +216,7 @@ func TestPendingSkipsTerminalCanceledAndExpiredAgentRuns(t *testing.T) {
 
 func TestHumanInputPauseResumesSameRunWithInput(t *testing.T) {
 	ctx := context.Background()
-	cp := flow.StoreCheckpoint(store.NewStore(), "input-agent")
+	cp := flow.StoreCheckpoint(store.NewMemoryStore(), "input-agent")
 	calls := 0
 	fakeGen = func(ctx context.Context, opts ai.Options, req *ai.Request) (*ai.Response, error) {
 		calls++
@@ -274,7 +274,7 @@ func TestHumanInputPauseResumesSameRunWithInput(t *testing.T) {
 
 func TestHumanInputResumeHonorsCanceledContextAndLeavesRunPending(t *testing.T) {
 	ctx := context.Background()
-	cp := flow.StoreCheckpoint(store.NewStore(), "input-cancel-agent")
+	cp := flow.StoreCheckpoint(store.NewMemoryStore(), "input-cancel-agent")
 	fakeGen = func(ctx context.Context, opts ai.Options, req *ai.Request) (*ai.Response, error) {
 		if opts.ToolHandler != nil {
 			opts.ToolHandler(ctx, ai.ToolCall{ID: "input-1", Name: toolHumanInput, Input: map[string]any{"prompt": "Approve deploy?"}})
@@ -319,7 +319,7 @@ func TestHumanInputResumeHonorsCanceledContextAndLeavesRunPending(t *testing.T)
 
 func TestApprovalDenialPausesCheckpointedRunAndResumeContinues(t *testing.T) {
 	ctx := context.Background()
-	cp := flow.StoreCheckpoint(store.NewStore(), "approval-agent")
+	cp := flow.StoreCheckpoint(store.NewMemoryStore(), "approval-agent")
 	calls := 0
 	fakeGen = func(ctx context.Context, opts ai.Options, req *ai.Request) (*ai.Response, error) {
 		calls++

agent/otel.go

@@ -38,6 +38,7 @@ const (
 	AttrGuardrailBlock = "agent.guardrail.block"
 	AttrRefusal        = "agent.refusal"
 	AttrInputChars     = "agent.input.chars"
+	AttrErrorKind      = "agent.error.kind"
 )
 
 type RunEvent struct {
@@ -57,6 +58,7 @@ type RunEvent struct {
 	Tokens      Usage     `json:"tokens,omitempty"`
 	Refused     string    `json:"refused,omitempty"`
 	Error       string    `json:"error,omitempty"`
+	ErrorKind   string    `json:"error_kind,omitempty"`
 	InputChars  int       `json:"input_chars,omitempty"`
 }
 
@@ -110,7 +112,7 @@ func (a *agentImpl) startRun(ctx context.Context, message string) (context.Conte
 		return ctx, func(err error) {
 			latency := time.Since(start).Milliseconds()
 			if err != nil {
-				a.recordRunEvent(RunEvent{Time: time.Now(), RunID: info.RunID, ParentID: info.ParentID, Agent: info.Agent, Kind: "error", LatencyMS: latency, Error: err.Error()})
+				a.recordRunEvent(RunEvent{Time: time.Now(), RunID: info.RunID, ParentID: info.ParentID, Agent: info.Agent, Kind: "error", LatencyMS: latency, Error: err.Error(), ErrorKind: string(ai.ClassifyError(err))})
 				return
 			}
 			a.recordRunEvent(RunEvent{Time: time.Now(), RunID: info.RunID, ParentID: info.ParentID, Agent: info.Agent, Kind: "done", LatencyMS: latency})
@@ -124,9 +126,10 @@ func (a *agentImpl) startRun(ctx context.Context, message string) (context.Conte
 		latency := time.Since(start).Milliseconds()
 		span.SetAttributes(attribute.Int64(AttrLatencyMS, latency))
 		if err != nil {
+			span.SetAttributes(attribute.String(AttrErrorKind, string(ai.ClassifyError(err))))
 			span.RecordError(err)
 			span.SetStatus(codes.Error, err.Error())
-			a.recordSpanEvent(span, RunEvent{Time: time.Now(), RunID: info.RunID, ParentID: info.ParentID, Agent: info.Agent, Kind: "error", LatencyMS: latency, Error: err.Error()})
+			a.recordSpanEvent(span, RunEvent{Time: time.Now(), RunID: info.RunID, ParentID: info.ParentID, Agent: info.Agent, Kind: "error", LatencyMS: latency, Error: err.Error(), ErrorKind: string(ai.ClassifyError(err))})
 		} else {
 			span.SetStatus(codes.Ok, "")
 			a.recordSpanEvent(span, RunEvent{Time: time.Now(), RunID: info.RunID, ParentID: info.ParentID, Agent: info.Agent, Kind: "done", LatencyMS: latency})
@@ -157,6 +160,7 @@ func (m *tracedModel) Generate(ctx context.Context, req *ai.Request, opts ...ai.
 		e := RunEvent{Time: time.Now(), RunID: info.RunID, ParentID: info.ParentID, Agent: info.Agent, Kind: "model", Provider: provider, Model: model, Attempt: info.Attempt, MaxAttempts: info.MaxAttempts, LatencyMS: dur, Tokens: usage}
 		if err != nil {
 			e.Error = err.Error()
+			e.ErrorKind = string(ai.ClassifyError(err))
 		}
 		m.a.recordRunEvent(e)
 		return resp, err
@@ -185,6 +189,7 @@ func (m *tracedModel) Generate(ctx context.Context, req *ai.Request, opts ...ai.
 	}
 	span.SetAttributes(attrs...)
 	if err != nil {
+		span.SetAttributes(attribute.String(AttrErrorKind, string(ai.ClassifyError(err))))
 		span.RecordError(err)
 		span.SetStatus(codes.Error, err.Error())
 	} else {
@@ -194,6 +199,7 @@ func (m *tracedModel) Generate(ctx context.Context, req *ai.Request, opts ...ai.
 	e := RunEvent{Time: time.Now(), RunID: info.RunID, ParentID: info.ParentID, Agent: info.Agent, Kind: "model", Provider: provider, Model: model, Attempt: info.Attempt, MaxAttempts: info.MaxAttempts, LatencyMS: dur, Tokens: usage}
 	if err != nil {
 		e.Error = err.Error()
+		e.ErrorKind = string(ai.ClassifyError(err))
 	}
 	m.a.recordSpanEvent(span, e)
 	return resp, err
@@ -220,7 +226,8 @@ func (a *agentImpl) traceTool(next ai.ToolHandler) ai.ToolHandler {
 		if a.opts.TraceProvider == nil {
 			res := next(ctx, call)
 			dur := time.Since(start).Milliseconds()
-			a.recordRunEvent(RunEvent{Time: time.Now(), RunID: info.RunID, ParentID: info.ParentID, Agent: info.Agent, Kind: "tool", Name: call.Name, LatencyMS: dur, Refused: res.Refused, Error: resultError(res)})
+			resErr := resultError(res)
+			a.recordRunEvent(RunEvent{Time: time.Now(), RunID: info.RunID, ParentID: info.ParentID, Agent: info.Agent, Kind: "tool", Name: call.Name, LatencyMS: dur, Refused: res.Refused, Error: resErr, ErrorKind: classifyToolError(resErr)})
 			return res
 		}
 
@@ -237,8 +244,11 @@ func (a *agentImpl) traceTool(next ai.ToolHandler) ai.ToolHandler {
 		if res.Refused != "" {
 			attrs = append(attrs, attribute.Bool(AttrGuardrailBlock, true), attribute.String(AttrRefusal, res.Refused))
 		}
-		span.SetAttributes(attrs...)
 		resErr := resultError(res)
+		if kind := classifyToolError(resErr); kind != "" {
+			attrs = append(attrs, attribute.String(AttrErrorKind, kind))
+		}
+		span.SetAttributes(attrs...)
 		if res.Refused != "" {
 			span.SetStatus(codes.Error, res.Refused)
 		} else if resErr != "" {
@@ -247,7 +257,7 @@ func (a *agentImpl) traceTool(next ai.ToolHandler) ai.ToolHandler {
 			span.SetStatus(codes.Ok, "")
 		}
 		span.End()
-		a.recordSpanEvent(span, RunEvent{Time: time.Now(), RunID: info.RunID, ParentID: info.ParentID, Agent: info.Agent, Kind: "tool", Name: call.Name, LatencyMS: dur, Refused: res.Refused, Error: resErr})
+		a.recordSpanEvent(span, RunEvent{Time: time.Now(), RunID: info.RunID, ParentID: info.ParentID, Agent: info.Agent, Kind: "tool", Name: call.Name, LatencyMS: dur, Refused: res.Refused, Error: resErr, ErrorKind: classifyToolError(resErr)})
 		return res
 	}
 }
@@ -264,6 +274,19 @@ func resultError(res ai.ToolResult) string {
 	return ""
 }
 
+func classifyToolError(err string) string {
+	switch {
+	case err == "":
+		return ""
+	case strings.Contains(strings.ToLower(err), "context canceled"):
+		return string(ai.ErrorKindCanceled)
+	case strings.Contains(strings.ToLower(err), "deadline exceeded"):
+		return string(ai.ErrorKindTimeout)
+	default:
+		return string(ai.ErrorKindProvider)
+	}
+}
+
 func (a *agentImpl) recordSpanEvent(span trace.Span, e RunEvent) {
 	if sc := span.SpanContext(); sc.IsValid() {
 		e.TraceID = sc.TraceID().String()
@@ -309,6 +332,9 @@ func runEventAttributes(e RunEvent) []attribute.KeyValue {
 	if e.Error != "" {
 		attrs = append(attrs, attribute.String("agent.error", e.Error))
 	}
+	if e.ErrorKind != "" {
+		attrs = append(attrs, attribute.String(AttrErrorKind, e.ErrorKind))
+	}
 	return attrs
 }
 

agent/otel_test.go

@@ -241,7 +241,7 @@ func TestAgentOpenTelemetrySpansModelFailure(t *testing.T) {
 				sawRunError = true
 			}
 		case spanNameModelCall:
-			if attrs[AttrAgentName] == "failing-runner" && attrs[AttrAttempt] == "1" && s.Status().Code == codesError {
+			if attrs[AttrAgentName] == "failing-runner" && attrs[AttrAttempt] == "1" && attrs[AttrErrorKind] == string(ai.ErrorKindUnknown) && s.Status().Code == codesError {
 				sawModelError = true
 			}
 		}
@@ -263,7 +263,7 @@ func TestAgentOpenTelemetrySpansModelFailure(t *testing.T) {
 	}
 	var sawModelEvent bool
 	for _, event := range events {
-		if event.Kind == "model" && event.Attempt == 1 && event.MaxAttempts == 1 && event.Error != "" {
+		if event.Kind == "model" && event.Attempt == 1 && event.MaxAttempts == 1 && event.Error != "" && event.ErrorKind == string(ai.ErrorKindUnknown) {
 			sawModelEvent = true
 		}
 	}