Compare commits

...

11 Commits

Author SHA1 Message Date
Evan Mattson cd5d282827 Merge branch 'main' into feature/python-subworkflow-checkpoint 2026-07-24 12:25:13 +09:00
Tao Chen 535690cd1d Add warning 2026-07-23 15:25:58 -07:00
Tao Chen 85fde62a76 Fix syntax check 2026-07-20 14:14:24 -07:00
Tao Chen 85eb53d412 Address comments 2026-07-20 13:27:35 -07:00
Tao Chen 2d7c8da6b0 Merge branch 'main' into feature/python-subworkflow-checkpoint 2026-07-20 11:16:36 -07:00
Tao Chen e0b0b79d9e Do not allow checkpoint storage in sub workflow 2026-07-14 14:30:46 -07:00
Tao Chen a4f6c26990 Clean up 2026-07-14 11:28:48 -07:00
Tao Chen 1389f304f2 Drop redundant decode in WorkflowExecutor.on_checkpoint_restore
The storage backend already materializes the full checkpoint on load (FileCheckpointStorage decodes recursively; InMemoryCheckpointStorage deep-copies), so the embedded sub_workflow_checkpoint (and legacy execution_contexts) arrive already decoded - like every other executor's on_checkpoint_restore state. Remove the no-op decode_checkpoint_value calls and the now-unused import.
2026-07-14 11:11:49 -07:00
Tao Chen 93719f4a34 Remove per-execution bookkeeping from WorkflowExecutor
The sub-workflow is a single shared instance, so per-execution ExecutionContext/request routing never provided real isolation. Delegate request/response tracking to the sub-workflow itself: can_handle accepts targeted propagated responses, _handle_response validates against the sub-workflow's pending requests and forwards responses immediately, and on_checkpoint_save embeds only the sub-workflow checkpoint (on_checkpoint_restore keeps a legacy reader for older checkpoints). Also emit the fresh-message/checkpoint-while-pending warning from FunctionalWorkflow.run to match Workflow.run.
2026-07-14 10:46:14 -07:00
Tao Chen a486374fd8 Move checkpoint-object construction into the runner context
Add RunnerContext.create_checkpoint_object alongside create_checkpoint (create_checkpoint now delegates to it and persists), so Runner.capture_checkpoint_object builds the snapshot via the context instead of a one-off get_messages peek primitive. In-flight messages are captured non-destructively (per-source lists copied). The checkpoint-less capturing contexts (azurefunctions, durabletask) raise NotImplementedError to match create_checkpoint.
2026-07-13 15:57:37 -07:00
Tao Chen e78604103d Fix sub-workflow checkpoint restore to preserve sub-workflow state
Add Runner.capture_checkpoint_object/restore_from_checkpoint_object (quiescent-only nested checkpoint) and embed a sub_workflow_checkpoint in WorkflowExecutor.on_checkpoint_save/on_checkpoint_restore so a resumed parent restores each sub-workflow's mid-progress state instead of only replaying pending request-info events. Keeps a backward-compat fallback when sub_workflow_checkpoint is absent.
2026-07-13 15:18:53 -07:00
11 changed files with 605 additions and 246 deletions
@@ -115,6 +115,18 @@ class CapturingRunnerContext(RunnerContext):
"""Checkpointing not supported in activity context."""
raise NotImplementedError("Checkpointing is not supported in Azure Functions activity context")
async def build_checkpoint(
self,
workflow_name: str,
graph_signature_hash: str,
state: State,
previous_checkpoint_id: str | None,
iteration_count: int,
metadata: dict[str, Any] | None = None,
) -> WorkflowCheckpoint:
"""Checkpointing not supported in activity context."""
raise NotImplementedError("Checkpointing is not supported in Azure Functions activity context")
async def load_checkpoint(self, checkpoint_id: str) -> WorkflowCheckpoint | None:
"""Checkpointing not supported in activity context."""
raise NotImplementedError("Checkpointing is not supported in Azure Functions activity context")
@@ -811,6 +811,24 @@ class FunctionalWorkflow:
execution is not allowed).
"""
self._validate_run_params(message, responses, checkpoint_id)
# Warn (but don't block) when a fresh message or a checkpoint restore begins while a prior
# run left request_info events pending. Mirrors Workflow.run. Delivering responses is the
# normal way to complete the pending cycle and is intentionally not warned.
if (message is not None or checkpoint_id is not None) and self._last_pending_request_ids:
logger.warning(
"Workflow %s received %s while %d request_info event(s) are still pending from an "
"unfinished request/response cycle; %s. Deliver responses (responses=...) to complete "
"the pending cycle before starting new input.",
self.name,
"a fresh message" if message is not None else "a checkpoint restore",
len(self._last_pending_request_ids),
(
"those requests remain answerable, but this run advances workflow state, so a "
"response that arrives later may apply to a workflow that has moved on"
if message is not None
else "those pending requests will be overwritten by the checkpoint's state"
),
)
if responses and checkpoint_id is None:
# Require at least one response key to match a currently-pending
# request; prevents silent replay against stale state while still
@@ -245,7 +245,11 @@ class RunnerImpl:
self._state.commit()
async def create_checkpoint_if_enabled(self) -> None:
"""Create a checkpoint if checkpointing is enabled and attach a label and metadata."""
"""Create a checkpoint and save the checkpoint to the configured storage if one is configured.
Note:
1. This method has no effect if checkpointing is not enabled in the context.
"""
if not self._ctx.has_checkpointing():
return
@@ -340,6 +344,57 @@ class RunnerImpl:
logger.error(f"Failed to restore from checkpoint {checkpoint_id}: {e}")
raise WorkflowCheckpointException(f"Failed to restore from checkpoint {checkpoint_id}") from e
async def build_checkpoint(self) -> WorkflowCheckpoint:
"""Create a checkpoint object.
Returns:
A ``WorkflowCheckpoint``.
"""
# Persist executor snapshots into committed shared state before exporting it.
await self._prepare_checkpoint_state()
return await self._ctx.build_checkpoint(
self._workflow_name,
self._graph_signature_hash,
self._state,
None,
self._iteration,
)
async def restore_checkpoint(self, checkpoint: WorkflowCheckpoint) -> None:
"""Restore runner state from an in-memory ``WorkflowCheckpoint`` object.
Unlike :meth:`restore_from_checkpoint`, this does not load from a storage
backend; it applies a checkpoint the caller already holds - for example, a
child workflow checkpoint embedded in a parent ``WorkflowExecutor``'s state.
Restores shared state, executor snapshots, in-flight messages, and pending
request_info events, then marks the runner as resumed.
Args:
checkpoint: The checkpoint whose state should be restored.
Raises:
WorkflowCheckpointException: If the checkpoint's graph signature does not
match this runner's workflow, or if restoration otherwise fails.
"""
if self._graph_signature_hash != checkpoint.graph_signature_hash:
raise WorkflowCheckpointException(
"Workflow graph has changed since the checkpoint was created. "
"Please rebuild the original workflow before resuming."
)
try:
# Clear first so import_state (which merges) does not leak stale keys from a
# prior run on this Workflow instance.
self._state.clear()
self._state.import_state(checkpoint.state)
await self._restore_executor_states()
await self._ctx.apply_checkpoint(checkpoint)
self._mark_resumed(checkpoint)
except Exception as e:
logger.error(f"Failed to restore from checkpoint {checkpoint.checkpoint_id}: {e}")
raise WorkflowCheckpointException(f"Failed to restore from checkpoint {checkpoint.checkpoint_id}") from e
async def _save_executor_states(self) -> None:
"""Populate executor state by calling checkpoint hooks on executors."""
for exec_id, executor in self._executors.items():
@@ -195,6 +195,34 @@ class RunnerContext(Protocol):
"""
...
async def build_checkpoint(
self,
workflow_name: str,
graph_signature_hash: str,
state: State,
previous_checkpoint_id: CheckpointID | None,
iteration_count: int,
metadata: dict[str, Any] | None = None,
) -> WorkflowCheckpoint:
"""Build a checkpoint and return it for the caller to own.
The checkpoint is constructed in memory and handed back to the caller; nothing is
persisted and no checkpoint storage is required.
Args:
workflow_name: The name of the workflow for which the checkpoint is being created.
graph_signature_hash: Hash of the workflow graph topology to
validate checkpoint compatibility during restore.
state: The state to include in the checkpoint.
previous_checkpoint_id: The ID of the previous checkpoint, if any, to form a checkpoint chain.
iteration_count: The current iteration count of the workflow.
metadata: Optional metadata to associate with the checkpoint.
Returns:
A ``WorkflowCheckpoint`` of the current context state.
"""
...
async def create_checkpoint(
self,
workflow_name: str,
@@ -204,7 +232,7 @@ class RunnerContext(Protocol):
iteration_count: int,
metadata: dict[str, Any] | None = None,
) -> CheckpointID:
"""Create a checkpoint of the current workflow state.
"""Persist a checkpoint of the current workflow state to configured storage and return its ID.
Args:
workflow_name: The name of the workflow for which the checkpoint is being created.
@@ -219,6 +247,9 @@ class RunnerContext(Protocol):
Returns:
The ID of the created checkpoint.
Raises:
ValueError: If checkpoint storage is not configured.
"""
...
@@ -381,6 +412,27 @@ class InProcRunnerContext:
def has_checkpointing(self) -> bool:
return self._get_effective_checkpoint_storage() is not None
async def build_checkpoint(
self,
workflow_name: str,
graph_signature_hash: str,
state: State,
previous_checkpoint_id: CheckpointID | None,
iteration_count: int,
metadata: dict[str, Any] | None = None,
) -> WorkflowCheckpoint:
return WorkflowCheckpoint(
workflow_name=workflow_name,
graph_signature_hash=graph_signature_hash,
previous_checkpoint_id=previous_checkpoint_id,
# Copy the per-source lists so the snapshot is isolated from later context mutations.
messages={source_id: list(messages) for source_id, messages in self._messages.items()},
state=state.export_state(),
pending_request_info_events=dict(self._pending_request_info_events),
iteration_count=iteration_count,
metadata=metadata or {},
)
async def create_checkpoint(
self,
workflow_name: str,
@@ -394,15 +446,13 @@ class InProcRunnerContext:
if not storage:
raise ValueError("Checkpoint storage not configured")
checkpoint = WorkflowCheckpoint(
workflow_name=workflow_name,
graph_signature_hash=graph_signature_hash,
previous_checkpoint_id=previous_checkpoint_id,
messages=dict(self._messages),
state=state.export_state(),
pending_request_info_events=dict(self._pending_request_info_events),
iteration_count=iteration_count,
metadata=metadata or {},
checkpoint = await self.build_checkpoint(
workflow_name,
graph_signature_hash,
state,
previous_checkpoint_id,
iteration_count,
metadata,
)
checkpoint_id = await storage.save(checkpoint)
logger.debug(f"Created checkpoint {checkpoint_id}")
@@ -814,10 +814,9 @@ class Workflow(DictConvertible):
# runner context has fully drained from any prior run. If it still
# has in-flight executor messages, the prior run didn't complete -
# the caller must either resume from a checkpoint or wait for the
# prior run to drain. (Pending request_info events are intentionally
# NOT blocked here: a follow-up run with message=... is the normal
# way to deliver a response to those pending requests, e.g. via
# WorkflowAgent._process_pending_requests.)
# prior run to drain. Pending request_info events are intentionally
# NOT blocked here (they are answered via a follow-up ``responses=...``
# run); the warning below surfaces the abandon/overwrite cases instead.
# NOTE: _validate_run_params already enforces that ``message`` is
# mutually exclusive with both ``checkpoint_id`` and ``responses``,
# so we don't need to re-check those here.
@@ -830,6 +829,33 @@ class Workflow(DictConvertible):
"checkpointing; there is no in-process recovery path."
)
# Warn (but don't block) when a fresh message or a checkpoint restore begins while the
# workflow still has pending request_info events from an unfinished request/response
# cycle. A fresh ``message`` does NOT drop those pending requests - they remain pending and
# can still be answered later - but the new run advances executor and shared state, so when
# a response for an earlier request eventually arrives the workflow may have moved on,
# yielding inconsistent results. A ``checkpoint_id`` restore instead replaces the context's
# pending requests with the checkpoint's state. Delivering ``responses`` is the normal way to
# answer pending requests and is intentionally not warned. Mirrors the WorkflowExecutor
# warning for overlapping sub-workflow executions.
if message is not None or checkpoint_id is not None:
pending_request_info_events = await self._runner.context.get_pending_request_info_events()
if pending_request_info_events:
logger.warning(
"Workflow %s received %s while %d request_info event(s) are still pending from an "
"unfinished request/response cycle; %s. Deliver responses (responses=...) to complete "
"the pending cycle before starting new input.",
self.id,
"a fresh message" if message is not None else "a checkpoint restore",
len(pending_request_info_events),
(
"those requests remain pending, but this run advances executor and shared state, "
"so a response that arrives later may apply to a workflow that has moved on"
if message is not None
else "those pending requests will be overwritten by the checkpoint's state"
),
)
initial_executor_fn = self._resolve_execution_mode(message, responses, checkpoint_id, checkpoint_storage)
async for event in self._run_workflow_with_tracing(
@@ -4,14 +4,12 @@ import asyncio
import logging
import sys
import types
import uuid
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any
if TYPE_CHECKING:
from ._workflow import Workflow
from ._checkpoint_encoding import decode_checkpoint_value
from ._const import GLOBAL_KWARGS_KEY, WORKFLOW_RUN_KWARGS_KEY
from ._events import (
WorkflowEvent,
@@ -36,7 +34,12 @@ logger = logging.getLogger(__name__)
@dataclass
class ExecutionContext:
"""Context for tracking a single sub-workflow execution."""
"""Legacy per-execution bookkeeping.
Retained only to decode checkpoints written before the sub-workflow's own checkpoint was
embedded (see ``WorkflowExecutor.on_checkpoint_restore``). It is no longer used at runtime -
the wrapped sub-workflow is the single source of truth for its pending requests.
"""
# The ID of the execution context
execution_id: str
@@ -161,11 +164,9 @@ class WorkflowExecutor(Executor):
# The response handler expects a SubWorkflowResponseMessage wrapping the response data.
### State Management
WorkflowExecutor maintains execution state across request/response cycles:
- Tracks pending requests by request_id
- Accumulates responses until all expected responses are received
- Resumes sub-workflow execution with complete response batch
- Handles concurrent executions and multiple pending requests
WorkflowExecutor keeps no request/response bookkeeping of its own. The wrapped sub-workflow
is the single source of truth for its pending requests; responses are forwarded to it and
validated against its own pending request_info events.
## Type System Integration
WorkflowExecutor inherits its type signature from the wrapped workflow:
@@ -194,46 +195,21 @@ class WorkflowExecutor(Executor):
- Converts to error event in parent context
- Provides detailed error information including sub-workflow ID
## Concurrent Execution Support
WorkflowExecutor fully supports multiple concurrent sub-workflow executions:
### Per-Execution State Isolation
Each sub-workflow invocation creates an isolated ExecutionContext:
## Overlapping Executions
A ``WorkflowExecutor`` wraps a single shared sub-workflow instance and keeps no per-execution
state. If a new input arrives while the sub-workflow still has pending request_info events from
an unfinished request/response cycle, the new input advances the shared sub-workflow state and
can interfere with that cycle - a response arriving later may apply to a sub-workflow that has
moved on. This is allowed but logs a warning, and is only safe when the wrapped workflow (and
its executors) are stateless.
.. code-block:: python
# Multiple concurrent invocations are supported
workflow_executor = WorkflowExecutor(my_workflow, id="concurrent_executor")
# Each invocation gets its own execution context
# Execution 1: processes input_1 independently
# Execution 2: processes input_2 independently
# No state interference between executions
### Request/Response Coordination
Responses are correctly routed to the originating execution:
- Each execution tracks its own pending requests and expected responses
- Request-to-execution mapping ensures responses reach the correct sub-workflow
- Response accumulation is isolated per execution
- Automatic cleanup when execution completes
### Memory Management
- Unlimited concurrent executions supported
- Each execution has unique UUID-based identification
- Cleanup of completed execution contexts
- Thread-safe state management for concurrent access
### Important Considerations
**Shared Workflow Instance**: All concurrent executions use the same underlying workflow instance.
For proper isolation, ensure that the wrapped workflow and its executors are stateless.
.. code-block:: python
# Avoid: Stateful executor with instance variables
# Avoid: stateful executor whose instance variables are shared across overlapping runs
class StatefulExecutor(Executor):
def __init__(self):
super().__init__(id="stateful")
self.data = [] # This will be shared across concurrent executions!
self.data = [] # Shared across overlapping sub-workflow executions!
## Integration with Parent Workflows
Parent workflows can intercept sub-workflow requests:
@@ -255,12 +231,18 @@ class WorkflowExecutor(Executor):
# Forward to external handler
await ctx.request_info(request.source_event, response_type=request.source_event.response_type)
## Checkpointing
The provided sub workflow may not have its own checkpoint storage. The sub workflow checkpointed states will
be managed by the parent workflow.
## Implementation Notes
- Sub-workflows run to completion before processing their results
- Event processing is atomic - all outputs are forwarded before requests
- Response accumulation ensures sub-workflows receive complete response batches
- Execution state is maintained for proper resumption after external requests
- Concurrent executions are fully isolated and do not interfere with each other
- Sub-workflows run to completion (or to idle-with-pending-requests) before their results are processed
- Event processing is ordered - outputs are forwarded before requests
- Responses are forwarded to the sub-workflow as they arrive; the sub-workflow tracks its own
pending requests and resumes when they are answered
- The WorkflowExecutor keeps no per-execution bookkeeping; the sub-workflow is the single source
of truth for its pending requests. Starting a new execution while the sub-workflow still has
pending requests logs a warning and is only safe when the wrapped workflow is stateless
"""
def __init__(
@@ -274,19 +256,18 @@ class WorkflowExecutor(Executor):
"""Initialize the WorkflowExecutor.
Args:
workflow: The workflow to execute as a sub-workflow.
workflow: The workflow to execute as a sub-workflow. This workflow instance (including
the executor instances within it) must be unique. If the same instances are shared
across multiple WorkflowExecutor instances, it may lead to incorrect behavior.
id: Unique identifier for this executor.
allow_direct_output: Whether to allow direct output from the sub-workflow.
By default, outputs from the sub-workflow are sent to
other executors in the parent workflow as messages.
When this is set to true, the outputs are yielded
directly from the WorkflowExecutor to the parent
workflow's event stream.
propagate_request: Whether to propagate requests from the sub-workflow to the
parent workflow. If set to true, requests from the sub-workflow
will be propagated as the original WorkflowEvent to the parent
workflow. Otherwise, they will be wrapped in a SubWorkflowRequestMessage,
which should be handled by an executor in the parent workflow.
allow_direct_output: Whether to allow direct output from the sub-workflow. By default,
outputs from the sub-workflow are sent to other executors in the parent workflow as
messages. When this is set to true, the outputs are yielded directly from the
WorkflowExecutor to the parent workflow's event stream.
propagate_request: Whether to propagate requests from the sub-workflow to the parent
workflow. If set to true, requests from the sub-workflow will be propagated as the
original WorkflowEvent to the parent workflow. Otherwise, they will be wrapped in a
SubWorkflowRequestMessage, which should be handled by an executor in the parent workflow.
Keyword Args:
**kwargs: Additional keyword arguments passed to the parent constructor.
@@ -294,13 +275,17 @@ class WorkflowExecutor(Executor):
super().__init__(id, **kwargs)
self.workflow = workflow
self.allow_direct_output = allow_direct_output
# Track execution contexts for concurrent sub-workflow executions
self._execution_contexts: dict[str, ExecutionContext] = {} # execution_id -> ExecutionContext
# Map request_id to execution_id for response routing
self._request_to_execution: dict[str, str] = {} # request_id -> execution_id
self._propagate_request = propagate_request
if self.workflow._runner_context.has_checkpointing(): # type: ignore
logger.warning(
"Sub workflow %s has its own checkpoint storage configured. "
"Sub workflow states are checkpointed by the parent workflow at superstep boundaries. "
"Additional checkpointing is only needed if you need to persist sub workflow state "
"independently of the parent workflow. ",
self.workflow.id,
)
@property
def input_types(self) -> list[type[Any] | types.UnionType]:
"""Get the input types based on the underlying workflow's input types plus WorkflowExecutor-specific types.
@@ -351,11 +336,10 @@ class WorkflowExecutor(Executor):
# Always handle SubWorkflowResponseMessage
return True
if (
message.original_request_info_event is not None
and message.original_request_info_event.request_id in self._request_to_execution
):
# Handle propagated responses for known requests
if message.original_request_info_event is not None:
# A propagated response is target-routed back to the executor that issued the request,
# so if one reaches this WorkflowExecutor it belongs to our sub-workflow. _handle_response
# validates it against the sub-workflow's pending requests and ignores anything unknown.
return True
# For other messages, only handle if the wrapped workflow can accept them as input
@@ -372,58 +356,52 @@ class WorkflowExecutor(Executor):
input_data: The input data to send to the sub-workflow.
ctx: The workflow context from the parent.
"""
# Create execution context for this sub-workflow run
execution_id = str(uuid.uuid4())
execution_context = ExecutionContext(
execution_id=execution_id,
collected_responses={},
expected_response_count=0,
pending_requests={},
# The sub-workflow is a single shared instance. If it still has pending request_info events
# from an unfinished request/response cycle, a new input advances its shared state and can
# interfere with that cycle - a response arriving later may apply to a sub-workflow that has
# moved on. We allow it (the sub-workflow may be stateless) but warn so the risk is visible.
pending_requests = await self.workflow._runner_context.get_pending_request_info_events() # pyright: ignore[reportPrivateUsage]
if pending_requests:
logger.warning(
f"WorkflowExecutor {self.id} received a new input message while its sub-workflow "
f"({self.workflow.id}) still has {len(pending_requests)} pending request(s) from an "
f"unfinished request/response cycle. The sub-workflow is a single shared instance, so the "
f"new input advances shared state and can interfere with the in-flight cycle. Ensure the "
f"sub-workflow is stateless, or complete the pending cycle before sending new input."
)
logger.debug(f"WorkflowExecutor {self.id} starting sub-workflow {self.workflow.id}")
# Get kwargs from parent workflow's State to propagate to subworkflow
parent_kwargs: dict[str, Any] = ctx.get_state(WORKFLOW_RUN_KWARGS_KEY, {})
# Extract invocation kwargs recognised by Workflow.run()
# The state stores resolved format (with __global__ wrapper for global kwargs).
# Unwrap __global__ before passing to the subworkflow so it gets re-resolved
# against the subworkflow's own executor IDs.
fi_kwargs: dict[str, Any] | None = None
ci_kwargs: dict[str, Any] | None = None
for key in ("function_invocation_kwargs", "client_kwargs"):
resolved = parent_kwargs.get(key)
if isinstance(resolved, dict):
# Unwrap global sentinel; pass per-executor dicts as-is
unwrapped: dict[str, Any] = resolved.get(GLOBAL_KWARGS_KEY, resolved) # type: ignore
if key == "function_invocation_kwargs":
fi_kwargs = unwrapped # type: ignore
else:
ci_kwargs = unwrapped # type: ignore
# Run the sub-workflow and collect all events, passing parent kwargs
result = await self.workflow.run(
input_data,
function_invocation_kwargs=fi_kwargs, # type: ignore
client_kwargs=ci_kwargs, # type: ignore
)
self._execution_contexts[execution_id] = execution_context
logger.debug(f"WorkflowExecutor {self.id} starting sub-workflow {self.workflow.id} execution {execution_id}")
logger.debug(f"WorkflowExecutor {self.id} sub-workflow {self.workflow.id} completed with {len(result)} events")
try:
# Get kwargs from parent workflow's State to propagate to subworkflow
parent_kwargs: dict[str, Any] = ctx.get_state(WORKFLOW_RUN_KWARGS_KEY, {})
# Extract invocation kwargs recognised by Workflow.run()
# The state stores resolved format (with __global__ wrapper for global kwargs).
# Unwrap __global__ before passing to the subworkflow so it gets re-resolved
# against the subworkflow's own executor IDs.
fi_kwargs: dict[str, Any] | None = None
ci_kwargs: dict[str, Any] | None = None
for key in ("function_invocation_kwargs", "client_kwargs"):
resolved = parent_kwargs.get(key)
if isinstance(resolved, dict):
# Unwrap global sentinel; pass per-executor dicts as-is
unwrapped: dict[str, Any] = resolved.get(GLOBAL_KWARGS_KEY, resolved) # type: ignore
if key == "function_invocation_kwargs":
fi_kwargs = unwrapped # type: ignore
else:
ci_kwargs = unwrapped # type: ignore
# Run the sub-workflow and collect all events, passing parent kwargs
result = await self.workflow.run(
input_data,
function_invocation_kwargs=fi_kwargs, # type: ignore
client_kwargs=ci_kwargs, # type: ignore
)
logger.debug(
f"WorkflowExecutor {self.id} sub-workflow {self.workflow.id} "
f"execution {execution_id} completed with {len(result)} events"
)
# Process the workflow result using shared logic
await self._process_workflow_result(result, execution_context, ctx)
finally:
# Clean up execution context if it's completed (no pending requests)
if execution_id in self._execution_contexts:
exec_ctx = self._execution_contexts[execution_id]
if not exec_ctx.pending_requests:
del self._execution_contexts[execution_id]
# Process the workflow result using shared logic
await self._process_workflow_result(result, ctx)
@handler
async def handle_message_wrapped_request_response(
@@ -433,8 +411,8 @@ class WorkflowExecutor(Executor):
) -> None:
"""Handle response from parent for a forwarded request.
This handler accumulates responses and only resumes the sub-workflow
when all expected responses have been received for that execution.
Forwards the response to the sub-workflow, which resumes and validates it against its
own pending requests.
Args:
response: The response to a previous request.
@@ -474,61 +452,44 @@ class WorkflowExecutor(Executor):
async def on_checkpoint_save(self) -> dict[str, Any]:
"""Get the current state of the WorkflowExecutor for checkpointing purposes."""
return {
"execution_contexts": {
execution_id: execution_context for execution_id, execution_context in self._execution_contexts.items()
},
"request_to_execution": dict(self._request_to_execution),
# The sub-workflow's own checkpoint carries everything needed to resume: shared state,
# executor snapshots, in-flight messages, and pending request_info events. The
# WorkflowExecutor keeps no separate request/response bookkeeping of its own. The
# sub-workflow is quiescent here: it ran to idle within this parent superstep before
# the parent checkpoints.
"sub_workflow_checkpoint": await self.workflow._runner.build_checkpoint(), # pyright: ignore[reportPrivateUsage]
}
@override
async def on_checkpoint_restore(self, state: dict[str, Any]) -> None:
"""Restore the WorkflowExecutor state from a checkpoint snapshot."""
# Validate the state contains the right keys
if "execution_contexts" not in state:
raise KeyError("Missing 'execution_contexts' in WorkflowExecutor state.")
if "request_to_execution" not in state:
raise KeyError("Missing 'request_to_execution' in WorkflowExecutor state.")
# The storage backend fully materializes the checkpoint on load, checkpointed data arrives as live objects.
sub_workflow_checkpoint = state.get("sub_workflow_checkpoint")
if sub_workflow_checkpoint is not None:
await self.workflow._runner.restore_checkpoint(sub_workflow_checkpoint) # pyright: ignore[reportPrivateUsage]
return
# Validate the execution contexts stored in the state have the right keys and values
execution_contexts: dict[str, ExecutionContext] | None = None
try:
execution_contexts = {
key: decode_checkpoint_value(value) for key, value in state["execution_contexts"].items()
}
except Exception as ex:
raise RuntimeError("Failed to deserialize execution context.") from ex
if not all(
isinstance(key, str) and isinstance(value, ExecutionContext) for key, value in execution_contexts.items()
):
raise ValueError("Execution contexts must have 'str' as key and 'ExecutionContext' as value.")
if not all(key == value.execution_id for key, value in execution_contexts.items()):
raise ValueError("Execution contexts must have matching keys and IDs.")
# Validate the request_to_execution map contain the right data
request_to_execution = state["request_to_execution"]
if not all(isinstance(key, str) and isinstance(value, str) for key, value in request_to_execution.items()):
raise ValueError("Request to execution map must have 'str' as key and 'str' as value.")
if not all(value in execution_contexts for value in request_to_execution.values()):
raise ValueError(
"'request_to_execution` contains unknown execution ID that is not part of the execution contexts."
)
self._execution_contexts = execution_contexts
self._request_to_execution = request_to_execution
# Add the `request_info_event`s back to the sub workflow.
# This is only a temporary solution to rehydrate the sub workflow with the requests.
# The proper way would be to rehydrate the workflow from a checkpoint on a Workflow
# API instead of the '_runner_context' object that should be hidden. And the sub workflow
# should be rehydrated from a checkpoint object instead of from a subset of the state.
# TODO(@taochen): Issue #1614 - how to handle the case when the parent workflow has checkpointing
# set up but not the sub workflow?
request_info_events = [
request_info_event
for execution_context in self._execution_contexts.values()
for request_info_event in execution_context.pending_requests.values()
]
# Backward-compatibility fallback for checkpoints written before the sub-workflow checkpoint
# was embedded. Those stored per-execution bookkeeping; recover only the pending
# request_info events so the sub-workflow re-emits its pending requests. The sub-workflow's
# deeper executor/shared state cannot be restored from these older checkpoints.
legacy_execution_contexts = state.get("execution_contexts")
if not legacy_execution_contexts:
return
request_info_events: list[WorkflowEvent[Any]] = []
for execution_context in legacy_execution_contexts.values():
if isinstance(execution_context, ExecutionContext):
request_info_events.extend(execution_context.pending_requests.values())
if execution_context.collected_responses:
logger.warning(
"WorkflowExecutor %s restored legacy checkpoint with collected responses for "
"execution_id %s. The sub-workflow is the single source of truth for its pending "
"requests, so these responses will be ignored. Resume instead from a checkpoint created "
"prior to any responses being collected if legacy request/response state must be "
"preserved. Legacy execution contexts for sub-workflows will be removed in a future release.",
self.id,
execution_context.execution_id,
)
await asyncio.gather(*[
self.workflow._runner_context.add_request_info_event(event) # pyright: ignore[reportPrivateUsage]
for event in request_info_events
@@ -537,7 +498,6 @@ class WorkflowExecutor(Executor):
async def _process_workflow_result(
self,
result: WorkflowRunResult,
execution_context: ExecutionContext,
ctx: WorkflowContext[Any],
) -> None:
"""Process the result from a workflow execution.
@@ -547,7 +507,6 @@ class WorkflowExecutor(Executor):
Args:
result: The workflow execution result.
execution_context: The execution context for this sub-workflow run.
ctx: The workflow context.
"""
# Collect all events from the workflow
@@ -586,10 +545,6 @@ class WorkflowExecutor(Executor):
for event in request_info_events:
request_id = event.request_id
response_type = event.response_type
# Track the pending request in execution context
execution_context.pending_requests[request_id] = event
# Map request to execution for response routing
self._request_to_execution[request_id] = execution_context.execution_id
if self._propagate_request:
# In a workflow where the parent workflow does not handle the request, the request
# should be propagated via the `request_info` mechanism to an external source. And
@@ -600,9 +555,6 @@ class WorkflowExecutor(Executor):
# request and handle it directly, a message should be sent.
await ctx.send_message(SubWorkflowRequestMessage(source_event=event, executor_id=self.id))
# Update expected response count for this execution
execution_context.expected_response_count = len(request_info_events)
# Handle final state
if workflow_run_state == WorkflowRunState.FAILED:
# Find the failed event (type='failed').
@@ -621,26 +573,18 @@ class WorkflowExecutor(Executor):
await ctx.add_event(error_event)
elif workflow_run_state == WorkflowRunState.IDLE:
# Sub-workflow is idle - nothing more to do now
logger.debug(
f"Sub-workflow {self.workflow.id} is idle with {len(self._execution_contexts)} active executions"
)
logger.debug(f"Sub-workflow {self.workflow.id} is idle")
elif workflow_run_state == WorkflowRunState.CANCELLED:
# Sub-workflow was cancelled - treat as completion
logger.debug(
f"Sub-workflow {self.workflow.id} was cancelled with {len(self._execution_contexts)} active executions"
)
logger.debug(f"Sub-workflow {self.workflow.id} was cancelled")
elif workflow_run_state == WorkflowRunState.IN_PROGRESS_PENDING_REQUESTS:
# Sub-workflow is still running with pending requests
logger.debug(
f"Sub-workflow {self.workflow.id} is still in progress with {len(request_info_events)} "
f"pending requests with {len(self._execution_contexts)} active executions"
f"Sub-workflow {self.workflow.id} is still in progress with {len(request_info_events)} pending requests"
)
elif workflow_run_state == WorkflowRunState.IDLE_WITH_PENDING_REQUESTS:
# Sub-workflow is idle but has pending requests
logger.debug(
f"Sub-workflow {self.workflow.id} is idle with pending requests: "
f"{len(request_info_events)} with {len(self._execution_contexts)} active executions"
)
logger.debug(f"Sub-workflow {self.workflow.id} is idle with pending requests: {len(request_info_events)}")
else:
raise RuntimeError(f"Unexpected workflow run state: {workflow_run_state}")
@@ -650,48 +594,17 @@ class WorkflowExecutor(Executor):
response: Any,
ctx: WorkflowContext[Any],
) -> None:
execution_id = self._request_to_execution.get(request_id)
if not execution_id or execution_id not in self._execution_contexts:
# The sub-workflow is the source of truth for what it is awaiting. Validate the response
# against its pending requests and ignore anything unknown or already handled.
pending_requests = await self.workflow._runner_context.get_pending_request_info_events() # pyright: ignore[reportPrivateUsage]
if request_id not in pending_requests:
logger.warning(
f"WorkflowExecutor {self.id} received response for unknown request_id: {request_id}. "
"This response will be ignored."
f"WorkflowExecutor {self.id} received a response for an unknown or already-handled "
f"request_id: {request_id}. This response will be ignored."
)
return
execution_context = self._execution_contexts[execution_id]
# Check if we have this pending request in the execution context
if request_id not in execution_context.pending_requests:
logger.warning(
f"WorkflowExecutor {self.id} received response for unknown request_id: "
f"{request_id} in execution {execution_id}, ignoring"
)
return
# Remove the request from pending list and request mapping
execution_context.pending_requests.pop(request_id, None)
self._request_to_execution.pop(request_id, None)
# Accumulate the response in this execution's context
execution_context.collected_responses[request_id] = response
# Check if we have all expected responses for this execution
if len(execution_context.collected_responses) < execution_context.expected_response_count:
logger.debug(
f"WorkflowExecutor {self.id} execution {execution_id} waiting for more responses: "
f"{len(execution_context.collected_responses)}/{execution_context.expected_response_count} received"
)
return # Wait for more responses
# Send all collected responses to the sub-workflow
responses_to_send = dict(execution_context.collected_responses)
execution_context.collected_responses.clear() # Clear for next batch
try:
# Resume the sub-workflow with all collected responses
result = await self.workflow.run(responses=responses_to_send)
# Process the workflow result using shared logic
await self._process_workflow_result(result, execution_context, ctx)
finally:
# Clean up execution context if it's completed (no pending requests)
if not execution_context.pending_requests:
del self._execution_contexts[execution_id]
# Forward the response to the sub-workflow, which resumes and validates it against its own
# pending requests, then process whatever the sub-workflow produces.
result = await self.workflow.run(responses={request_id: response})
await self._process_workflow_result(result, ctx)
@@ -257,6 +257,43 @@ class TestHITL:
assert outputs == ["Final: Looks great!"]
assert result2.get_final_state() == WorkflowRunState.IDLE
async def test_fresh_message_while_pending_requests_warns(self, caplog: pytest.LogCaptureFixture) -> None:
"""A fresh message while request_info events are pending is allowed but logs a warning."""
@workflow
async def review_wf(doc: str, ctx: RunContext) -> str:
feedback = await ctx.request_info({"draft": doc}, response_type=str, request_id="req1")
return f"Final: {feedback}"
result1 = await review_wf.run("my doc")
assert result1.get_final_state() == WorkflowRunState.IDLE_WITH_PENDING_REQUESTS
# Starting fresh input while a request is pending does not abandon it, but advances
# workflow state so a later response may apply to a moved-on workflow -> warn (but proceed).
with caplog.at_level(logging.WARNING):
await review_wf.run("another doc")
assert "request_info event(s) are still pending" in caplog.text
assert "a fresh message" in caplog.text
async def test_responses_while_pending_requests_does_not_warn(self, caplog: pytest.LogCaptureFixture) -> None:
"""Delivering responses is the normal completion path and must not warn."""
@workflow
async def review_wf(doc: str, ctx: RunContext) -> str:
feedback = await ctx.request_info({"draft": doc}, response_type=str, request_id="req1")
return f"Final: {feedback}"
result1 = await review_wf.run("my doc")
assert result1.get_final_state() == WorkflowRunState.IDLE_WITH_PENDING_REQUESTS
caplog.clear()
with caplog.at_level(logging.WARNING):
result2 = await review_wf.run(responses={"req1": "Looks great!"})
assert result2.get_final_state() == WorkflowRunState.IDLE
assert "still pending" not in caplog.text
async def test_untyped_ctx_parameter(self):
"""ctx is injected by parameter name even without a RunContext annotation."""
@@ -512,6 +512,98 @@ async def test_runner_reset_iteration_count():
assert runner._iteration == 0 # pyright: ignore[reportPrivateUsage]
async def test_runner_capture_and_restore_checkpoint_object_roundtrip():
"""build_checkpoint() then restore_checkpoint() must roundtrip.
Shared state and executor snapshots are captured into an in-memory ``WorkflowCheckpoint``
and restored from it without any storage backend (the path the parent WorkflowExecutor
uses to checkpoint a nested sub-workflow).
"""
class CounterExecutor(Executor):
def __init__(self, id: str) -> None:
super().__init__(id=id)
self.count = 0
@handler
async def handle(self, message: MockMessage, ctx: WorkflowContext[Any, int]) -> None:
self.count += message.data
async def on_checkpoint_save(self) -> dict[str, Any]:
return {"count": self.count}
async def on_checkpoint_restore(self, state: dict[str, Any]) -> None:
self.count = int(state.get("count", 0))
executor = CounterExecutor(id="counter")
state = State()
ctx = InProcRunnerContext()
runner = Runner([], {executor.id: executor}, state, ctx, "test_name", graph_signature_hash="test_hash")
# Establish some state to capture.
executor.count = 7
state.set("shared_key", "shared_value")
state.commit()
checkpoint = await runner.build_checkpoint()
assert checkpoint.graph_signature_hash == "test_hash"
# Mutate after capture; restoring must roll back to the captured snapshot.
executor.count = 999
state.set("shared_key", "mutated")
state.commit()
await runner.restore_checkpoint(checkpoint)
assert executor.count == 7
assert state.get("shared_key") == "shared_value"
assert runner._resumed_from_checkpoint is True # pyright: ignore[reportPrivateUsage]
async def test_runner_build_checkpoint_includes_in_flight_messages():
"""build_checkpoint() must snapshot in-flight messages non-destructively."""
executor = MockExecutor(id="executor_a")
state = State()
ctx = InProcRunnerContext()
runner = Runner([], {executor.id: executor}, state, ctx, "test_name", graph_signature_hash="test_hash")
await ctx.send_message(WorkflowMessage(data=MockMessage(data=1), source_id="START"))
checkpoint = await runner.build_checkpoint()
# The in-flight message is captured in the snapshot ...
assert list(checkpoint.messages.keys()) == ["START"]
assert len(checkpoint.messages["START"]) == 1
# ... without draining it from the runner (capture is non-destructive).
assert await ctx.has_messages() is True
async def test_runner_build_checkpoint_do_not_advance_previous_checkpoint_id():
"""build_checkpoint() must not advance _previous_checkpoint_id so a later capture chains to it."""
executor = MockExecutor(id="executor_a")
state = State()
ctx = InProcRunnerContext()
runner = Runner([], {executor.id: executor}, state, ctx, "test_name", graph_signature_hash="test_hash")
# Pre-condition: nothing captured yet, so there is no parent to chain back to.
assert runner._previous_checkpoint_id is None # pyright: ignore[reportPrivateUsage]
first = await runner.build_checkpoint()
assert first.previous_checkpoint_id is None
# Capturing advances the tracked checkpoint id to the newly-created checkpoint ...
assert runner._previous_checkpoint_id is None # pyright: ignore[reportPrivateUsage]
async def test_runner_restore_checkpoint_rejects_graph_mismatch():
"""restore_checkpoint() must reject a checkpoint from a different graph."""
runner = Runner([], {}, State(), InProcRunnerContext(), "test_name", graph_signature_hash="hash-a")
foreign = WorkflowCheckpoint(workflow_name="test_name", graph_signature_hash="hash-b")
with pytest.raises(WorkflowCheckpointException, match="Workflow graph has changed"):
await runner.restore_checkpoint(foreign)
class CheckpointingContext(InProcRunnerContext):
"""A context that supports checkpointing for testing."""
@@ -1,9 +1,11 @@
# Copyright (c) Microsoft. All rights reserved.
import logging
from dataclasses import dataclass, field
from typing import Any
from uuid import uuid4
import pytest
from typing_extensions import Never
from agent_framework import (
@@ -15,6 +17,7 @@ from agent_framework import (
WorkflowContext,
WorkflowEvent,
WorkflowExecutor,
WorkflowRunState,
handler,
response_handler,
)
@@ -465,6 +468,62 @@ async def test_concurrent_sub_workflow_execution() -> None:
# (This is implicitly tested by the fact that we got correct results for all emails)
async def test_sub_workflow_warns_on_overlapping_execution(caplog: pytest.LogCaptureFixture) -> None:
"""A new input while a prior sub-workflow execution is awaiting responses logs a warning.
Overlapping executions share one sub-workflow instance and its state, so WorkflowExecutor
allows the new execution but warns that it is only safe when the wrapped workflow is stateless.
"""
class TwoInputParent(Executor):
def __init__(self) -> None:
super().__init__(id="two_input_parent")
self._pending: dict[str, SubWorkflowRequestMessage] = {}
@handler
async def start(self, emails: list[str], ctx: WorkflowContext[EmailValidationRequest]) -> None:
for email in emails:
await ctx.send_message(EmailValidationRequest(email=email))
@handler
async def handle_domain_request(
self,
sub_workflow_request: SubWorkflowRequestMessage,
ctx: WorkflowContext[SubWorkflowResponseMessage],
) -> None:
domain_request = sub_workflow_request.source_event.data
assert isinstance(domain_request, DomainCheckRequest)
self._pending[domain_request.id] = sub_workflow_request
await ctx.request_info(domain_request, bool)
@handler
async def collect(self, result: ValidationResult, ctx: WorkflowContext) -> None: ...
parent = TwoInputParent()
workflow_executor = WorkflowExecutor(create_email_validation_workflow(), "email_workflow")
main_workflow = (
WorkflowBuilder(start_executor=parent)
.add_edge(parent, workflow_executor)
.add_edge(workflow_executor, parent)
.build()
)
with caplog.at_level(logging.WARNING, logger="agent_framework._workflows._workflow_executor"):
result = await main_workflow.run(["a@domain1.com", "b@domain2.com"])
# Two inputs are delivered to the same WorkflowExecutor in one superstep: the second execution
# starts while the sub-workflow still has a pending request, producing exactly one overlap
# warning from the WorkflowExecutor. (The substring is unique to the WorkflowExecutor warning so
# it is not confused with the core Workflow.run pending-request warning.)
assert len(result.get_request_info_events()) == 2
overlap_warnings = [
record
for record in caplog.records
if record.levelno == logging.WARNING and "new input message while its sub-workflow" in record.getMessage()
]
assert len(overlap_warnings) == 1
# region Checkpoint-related message types and executors for sub-workflow tests
@@ -619,6 +678,59 @@ async def test_sub_workflow_checkpoint_restore_no_duplicate_requests() -> None:
assert request_events[0].data.prompt == "Second request"
async def test_sub_workflow_checkpoint_restore_preserves_sub_workflow_state() -> None:
"""Resuming a sub-workflow mid-progress must restore its internal executor state.
Regression guard for the issue where only the WorkflowExecutor's bookkeeping (pending
requests) was checkpointed, so a sub-workflow executor that accumulates state across
multiple request/response cycles (here ``TwoStepSubWorkflowExecutor._responses``) lost
that state on resume. With the sub-workflow's own checkpoint embedded in the parent
checkpoint, the second response now completes the two-step flow instead of triggering a
spurious third request.
"""
storage = InMemoryCheckpointStorage()
# Step 1: run until the first request.
workflow1 = _build_checkpoint_test_workflow(storage)
first_request_id: str | None = None
async for event in workflow1.run("test_value", stream=True):
if event.type == "request_info":
first_request_id = event.request_id
assert first_request_id is not None
# Step 2: answer the first request so the sub-workflow accumulates internal state
# (``_responses == ["first_answer"]``) and emits the second request. This mid-progress
# point is what we checkpoint and resume from - the case the no-duplicate test (which
# checkpoints at the first request, before any state accrues) does not cover.
second_request_id: str | None = None
async for event in workflow1.run(stream=True, responses={first_request_id: "first_answer"}):
if event.type == "request_info":
second_request_id = event.request_id
assert second_request_id is not None
# Resume from the latest checkpoint (captured after the second request was made).
checkpoints = await storage.list_checkpoints(workflow_name=workflow1.name)
checkpoint_id = max(checkpoints, key=lambda cp: cp.iteration_count).checkpoint_id
workflow2 = _build_checkpoint_test_workflow(storage)
resumed_second_request_id: str | None = None
async for event in workflow2.run(checkpoint_id=checkpoint_id, stream=True):
if event.type == "request_info":
resumed_second_request_id = event.request_id
assert resumed_second_request_id is not None
assert resumed_second_request_id == second_request_id
# Step 3: answer the second request. With the sub-workflow's state restored, the two-step
# executor completes instead of emitting a spurious third request. If the internal state
# were lost, the second answer would be treated as a first answer and a third request
# ("Second request") would be emitted.
result = await workflow2.run(responses={resumed_second_request_id: "second_answer"})
assert result.get_request_info_events() == [], (
"Sub-workflow internal state was lost on resume: a spurious extra request was emitted"
)
assert result.get_final_state() == WorkflowRunState.IDLE
async def test_sub_workflow_intermediate_outputs_propagate_to_parent() -> None:
"""A child workflow's intermediate emissions must bubble up through the parent.
@@ -2,6 +2,7 @@
import asyncio
import gc
import logging
import tempfile
from collections.abc import AsyncIterable, Awaitable, Sequence
from dataclasses import dataclass, field
@@ -109,6 +110,38 @@ class MockExecutorRequestApproval(Executor):
await ctx.send_message(NumberMessage(data=data))
async def test_fresh_message_while_pending_advances_state_without_abandoning_requests(
caplog: pytest.LogCaptureFixture,
) -> None:
"""A fresh message while a request is pending is allowed but hazardous.
A fresh ``message`` does NOT abandon the pending request - it can still be answered
later - but the new run advances executor state, so a response for the earlier request
applies to a workflow that has moved on. The run is allowed and a warning is emitted.
"""
executor = MockExecutorRequestApproval(id="approver")
workflow = WorkflowBuilder(start_executor=executor).build()
# Turn 1: request approval for data=1 -> workflow idles with a pending request.
result1 = await workflow.run(NumberMessage(data=1))
assert result1.get_final_state() == WorkflowRunState.IDLE_WITH_PENDING_REQUESTS
original_request_id = result1.get_request_info_events()[0].request_id
# Turn 2: a fresh message for data=2 while the first request is still pending. This is
# allowed but warns, and advances the executor's stored state from 1 to 2.
with caplog.at_level(logging.WARNING):
result2 = await workflow.run(NumberMessage(data=2))
assert "request_info event(s) are still pending" in caplog.text
assert "a fresh message" in caplog.text
assert result2.get_final_state() == WorkflowRunState.IDLE_WITH_PENDING_REQUESTS
# Turn 3: the ORIGINAL request is still answerable, proving the fresh message did not
# abandon it. But because the executor state moved on to 2, the response applies to the
# moved-on state and yields 2, not the original 1.
result3 = await workflow.run(responses={original_request_id: ApprovalMessage(approved=True)})
assert result3.get_outputs() == [2]
async def test_workflow_run_streaming() -> None:
"""Test the workflow run stream."""
executor_a = IncrementExecutor(id="executor_a")
@@ -109,6 +109,17 @@ class CapturingRunnerContext(RunnerContext):
) -> str:
raise NotImplementedError("Checkpointing is not supported in activity context")
async def build_checkpoint(
self,
workflow_name: str,
graph_signature_hash: str,
state: State,
previous_checkpoint_id: str | None,
iteration_count: int,
metadata: dict[str, Any] | None = None,
) -> WorkflowCheckpoint:
raise NotImplementedError("Checkpointing is not supported in activity context")
async def load_checkpoint(self, checkpoint_id: str) -> WorkflowCheckpoint | None:
raise NotImplementedError("Checkpointing is not supported in activity context")