Compare commits

...

6 Commits

Author SHA1 Message Date
Tomu Hirata 1609d45757 fix(policies): default history_window to 10 2026-07-01 12:31:31 +09:00
Tomu Hirata e70032e432 fix(policies): address Polly review on detect_task_switch
Blocking fix (window freeze):
TASK_SWITCH branch now includes state_updates resetting the history to
[new_message] so the new task accumulates context from the switching
message rather than staying pinned to pre-switch context. On ASK the
update applies only if the user approves (engine behavior), which is
documented in the docstring.

Non-blocking fixes:
- min_turns default changed from 2 → 1 so the classifier fires on the
  2nd message (one prior message), matching the "single prior message
  is enough" intent. Docstring updated to describe the behavior
  accurately.
- Add _strip_code_fences() (copied from prompt.py) and apply it before
  json.loads so fenced JSON from providers that ignore structured-output
  still parses instead of silently failing open.
- Add security note in docstring: action="DENY" is not a security
  control because user messages are interpolated into the classifier
  prompt (prompt injection → forced CONTINUATION).
- Add test_context.py: 13 unit tests covering abstain on non-request
  phases, accumulation below min_turns, no-llm_client fail-open,
  CONTINUATION/TASK_SWITCH paths with mock client, code-fence
  robustness, and min_turns=0 boundary.
2026-07-01 12:11:49 +09:00
Tomu Hirata 674b67b8f8 fix(policies): use unpacking instead of list concatenation (RUF005) 2026-07-01 11:27:58 +09:00
Tomu Hirata 7974449464 refactor(policies): remove cap_conversation_depth, keep detect_task_switch only 2026-07-01 11:14:25 +09:00
Tomu Hirata 87cd097c20 feat(policies): add detect_task_switch LLM classifier policy
Adds a second context-management policy to context.py that fires on
request events and uses the server-level LLM to classify each user
message as CONTINUATION or TASK_SWITCH. On a detected switch, it asks
(or denies) with a recommendation to start a fresh session rather than
accumulating stale context from the prior task.

Maintains a sliding history window in session_state so the classifier
has concrete prior-turn evidence, and defaults to ASK (not DENY) to
minimise the impact of false positives.
2026-07-01 11:01:08 +09:00
Tomu Hirata d2c63c7634 feat(policies): add cap_conversation_depth builtin policy
Adds a new context-management policy that fires on llm_request events
and denies (or asks) when conversation depth exceeds a configured
message count. Encourages agents to start fresh sessions for new tasks
rather than accumulating stale context — the goal is fewer tokens
wasted, not just fewer tokens used.
2026-07-01 10:55:40 +09:00
3 changed files with 572 additions and 0 deletions
+1
View File
@@ -44,5 +44,6 @@ BUILTIN_POLICY_MODULES = [
"omnigent.policies.builtins.routing",
"omnigent.policies.builtins.cel",
"omnigent.policies.builtins.prompt",
"omnigent.policies.builtins.context",
"omnigent.inner.nessie.policies",
]
+338
View File
@@ -0,0 +1,338 @@
"""Built-in context-management policies.
Helps agents keep their working context lean. The guiding principle:
the goal is not fewer tokens *used*, but fewer tokens *wasted* —
sprawling context filled with stale tool results from a prior task
degrades quality without adding value.
The recommended response to a denial is to start a fresh session for
the new task rather than compacting or summarising in place.
"""
from __future__ import annotations
import json
import logging
from typing import Any
from omnigent.policies.schema import PolicyCallable, PolicyEvent, PolicyResponse
_log = logging.getLogger(__name__)
# ── detect_task_switch ────────────────────────────────────────────────────────
_TASK_SWITCH_HISTORY_KEY = "_task_switch_history"
_DEFAULT_TASK_SWITCH_PROMPT = """\
You are a conversation-continuity classifier for a coding assistant.
You are given the user's recent messages (the "prior context") and their
latest message. Decide whether the latest message is a continuation of the
same task or the start of a clearly different, unrelated task.
Guidelines:
- CONTINUATION: the latest message follows naturally from prior work — a
follow-up question, a refinement, a related sub-task, or asking about
something mentioned earlier.
- TASK_SWITCH: the latest message starts a completely different topic or
codebase concern with no meaningful connection to what came before.
- When in doubt, prefer CONTINUATION — false positives (blocking a legitimate
continuation) are more harmful than false negatives.
Return strict JSON only:
{"verdict": "CONTINUATION" | "TASK_SWITCH"}
"""
_TASK_SWITCH_SCHEMA: dict[str, Any] = {
"format": {
"type": "json_schema",
"name": "task_switch_verdict",
"strict": True,
"schema": {
"type": "object",
"properties": {
"verdict": {
"type": "string",
"enum": ["CONTINUATION", "TASK_SWITCH"],
},
},
"required": ["verdict"],
"additionalProperties": False,
},
},
}
def _strip_code_fences(text: str) -> str:
"""Strip markdown code fences from LLM output.
Even with structured output, some providers wrap JSON in
triple-backtick fences. This strips the outermost fence
so ``json.loads`` succeeds.
:param text: Raw LLM response text.
:returns: Text with code fences removed.
"""
stripped = text.strip()
if stripped.startswith("```"):
first_newline = stripped.find("\n")
if first_newline != -1:
stripped = stripped[first_newline + 1 :]
if stripped.rstrip().endswith("```"):
stripped = stripped.rstrip()[:-3].rstrip()
return stripped
def _extract_text(response: Any) -> str:
"""Pull plain text out of a PolicyLLMClient response."""
text = getattr(response, "output_text", None)
if isinstance(text, str) and text.strip():
return text.strip()
output = getattr(response, "output", None)
if not isinstance(output, list) or not output:
return ""
content = getattr(output[0], "content", None)
if not isinstance(content, list) or not content:
return ""
return getattr(content[0], "text", "") or ""
def detect_task_switch(
*,
min_turns: int = 1,
history_window: int = 10,
action: str = "ASK",
classification_prompt: str = _DEFAULT_TASK_SWITCH_PROMPT,
) -> PolicyCallable:
"""Factory: detect when the user switches to an unrelated task.
Fires on ``request`` events. Maintains a rolling window of recent
user messages in ``session_state`` and, once ``min_turns`` prior
messages have accumulated, asks the server-level LLM to classify the
latest message as ``CONTINUATION`` or ``TASK_SWITCH``.
On ``TASK_SWITCH`` the policy returns *action* with a message
recommending a fresh session — not compaction or summarisation. The
window is reset to contain only the switching message so the new
task can accumulate its own history from a clean state (``DENY``
path only — the ``ASK`` path cannot write state on decline, so the
window advances only once the user approves and the next request
arrives).
On ``CONTINUATION`` the policy records the new message into state
and abstains, letting the request through.
Requires the server to have an ``llm:`` config block; abstains
(fail-open) when no LLM client is available.
.. note::
``action="DENY"`` is not a security control — user messages are
interpolated into the classifier prompt and a determined user can
craft a message that forces a ``CONTINUATION`` verdict (prompt
injection). Use this policy for context-hygiene guidance, not
for access control.
:param min_turns: Number of prior messages to accumulate before the
classifier starts firing. With the default of ``1`` the
classifier fires on the **second** user message (one prior
message is enough context to detect a switch). Set to ``0`` to
classify from the very first message.
:param history_window: Maximum number of recent user messages kept
in state as prior context for the classifier. Defaults to
``10``. Older messages are dropped as the window slides.
:param action: Response when a task switch is detected.
``"ASK"`` (default) escalates to the user; ``"DENY"`` blocks
the request outright. Defaults to ``"ASK"`` because
false-positive task-switch classifications are more harmful than
false negatives.
:param classification_prompt: System prompt for the classifier LLM
call. Must instruct the model to return
``{"verdict": "CONTINUATION"|"TASK_SWITCH"}``; the schema is
enforced via structured output regardless.
:returns: An async policy callable that fires on ``request`` events.
"""
normalised_action = action.upper() if isinstance(action, str) else "ASK"
if normalised_action not in {"DENY", "ASK"}:
_log.warning(
"detect_task_switch: unknown action %r — defaulting to ASK",
action,
)
normalised_action = "ASK"
async def evaluate(event: PolicyEvent) -> PolicyResponse | None:
"""Classify the new user message and flag task switches.
Reads ``session_state[_TASK_SWITCH_HISTORY_KEY]`` for prior
context and writes the updated window back via
``state_updates``.
:param event: Policy event dict.
:returns: *action* when a task switch is detected; ``None``
(abstain) otherwise.
"""
if event.get("type") != "request":
return None
new_message = event.get("data", "")
if not isinstance(new_message, str) or not new_message.strip():
return None
state = event.get("session_state") or {}
history: list[str] = state.get(_TASK_SWITCH_HISTORY_KEY) or []
# Slide the window: append new message, keep last history_window entries.
updated_history = [*history, new_message[:500]][-history_window:]
# Not enough prior turns yet — record and pass through.
if len(history) < min_turns:
_log.debug(
"detect_task_switch: history_len=%d < min_turns=%d — accumulating",
len(history),
min_turns,
)
return {
"result": "ALLOW",
"state_updates": [
{
"key": _TASK_SWITCH_HISTORY_KEY,
"action": "set",
"value": updated_history,
}
],
}
# ── Classify ────────────────────────────────────────────────────
llm_client = event.get("llm_client")
if llm_client is None:
_log.warning(
"detect_task_switch: no llm_client — server has no llm: config. Abstaining."
)
return None
prior_context = "\n".join(f"- {msg}" for msg in history[-history_window:])
user_prompt = f"Prior messages:\n{prior_context}\n\nLatest message:\n{new_message[:500]}"
try:
response = await llm_client.create(
instructions=classification_prompt,
input=[
{
"role": "user",
"content": [{"type": "input_text", "text": user_prompt}],
}
],
text=_TASK_SWITCH_SCHEMA,
)
raw = _extract_text(response)
if not raw:
return None
raw = _strip_code_fences(raw)
verdict_obj = json.loads(raw)
except Exception: # noqa: BLE001 — fail-open
_log.exception("detect_task_switch: classifier call failed")
return None
verdict = verdict_obj.get("verdict", "") if isinstance(verdict_obj, dict) else ""
if verdict == "TASK_SWITCH":
_log.info("detect_task_switch: TASK_SWITCH detected — action=%s", normalised_action)
# Reset the window to the switching message alone so the new
# task accumulates fresh history from here. state_updates on a
# DENY are applied immediately; state_updates on an ASK are only
# applied if the user approves — on decline the window stays
# pinned, so the next message will be re-classified against the
# pre-switch context (which is the safe / over-prompting direction).
return {
"result": normalised_action,
"reason": (
"This message looks like the start of a new, unrelated task. "
"The current session carries context from prior work that will "
"waste capacity without helping here. "
"Start a fresh session for this task to keep context lean."
),
"state_updates": [
{
"key": _TASK_SWITCH_HISTORY_KEY,
"action": "set",
"value": [new_message[:500]],
}
],
}
if verdict == "CONTINUATION":
# Update history and let the request through.
return {
"result": "ALLOW",
"state_updates": [
{
"key": _TASK_SWITCH_HISTORY_KEY,
"action": "set",
"value": updated_history,
}
],
}
# Unrecognised verdict — fail open.
return None
return evaluate # type: ignore[return-value]
# ── Registry ──────────────────────────────────────────────────────────────────
POLICY_REGISTRY: list[dict[str, Any]] = [
{
"handler": "omnigent.policies.builtins.context.detect_task_switch",
"kind": "factory",
"name": "Detect Task Switch",
"description": (
"Uses the server-level LLM to classify each user message as a "
"continuation of the current task or the start of a new, unrelated "
"one. On a detected task switch, asks (or denies) with a recommendation "
"to start a fresh session. Implements the 'Keep Context Lean' strategy: "
"start fresh sessions when switching tasks rather than accumulating "
"stale context. Requires an llm: config block on the server; "
"abstains (fail-open) when no LLM client is available."
),
"params_schema": {
"type": "object",
"properties": {
"min_turns": {
"type": "integer",
"default": 2,
"description": (
"Number of prior user messages to accumulate before "
"the classifier starts firing. Defaults to 2."
),
},
"history_window": {
"type": "integer",
"default": 4,
"description": (
"Maximum number of recent user messages kept as prior "
"context for the classifier. Older messages are dropped "
"as the window slides. Defaults to 10."
),
},
"action": {
"type": "string",
"enum": ["ASK", "DENY"],
"default": "ASK",
"description": (
"Response when a task switch is detected. "
"ASK escalates to the user (default); "
"DENY blocks the request outright."
),
},
"classification_prompt": {
"type": "string",
"description": (
"System prompt for the classifier. Must instruct the "
'model to return {"verdict": "CONTINUATION"|"TASK_SWITCH"}; '
"the output schema is enforced via structured output."
),
},
},
"required": [],
},
},
]
+233
View File
@@ -0,0 +1,233 @@
"""Unit tests for omnigent.policies.builtins.context."""
from __future__ import annotations
import pytest
from omnigent.policies.builtins.context import (
_TASK_SWITCH_HISTORY_KEY,
_strip_code_fences,
detect_task_switch,
)
# ── helpers ──────────────────────────────────────────────────────────────────
def _event(
message: str,
*,
history: list[str] | None = None,
phase: str = "request",
) -> dict:
return {
"type": phase,
"data": message,
"session_state": {_TASK_SWITCH_HISTORY_KEY: history or []},
}
# ── _strip_code_fences ───────────────────────────────────────────────────────
def test_strip_code_fences_plain_json() -> None:
assert _strip_code_fences('{"verdict":"CONTINUATION"}') == '{"verdict":"CONTINUATION"}'
def test_strip_code_fences_with_fence() -> None:
assert (
_strip_code_fences('```json\n{"verdict":"TASK_SWITCH"}\n```')
== '{"verdict":"TASK_SWITCH"}'
)
def test_strip_code_fences_bare_fence() -> None:
assert _strip_code_fences('```\n{"v":"x"}\n```') == '{"v":"x"}'
# ── non-gated phases abstain ─────────────────────────────────────────────────
@pytest.mark.asyncio
async def test_non_request_phases_abstain() -> None:
"""Only ``request`` events are evaluated; all others abstain."""
policy = detect_task_switch()
for phase in ("tool_call", "tool_result", "response", "llm_request"):
result = await policy(_event("hello", phase=phase))
assert result is None, f"expected None for phase={phase}"
# ── accumulation (below min_turns) ───────────────────────────────────────────
@pytest.mark.asyncio
async def test_first_message_accumulates_no_history() -> None:
"""First message (history empty) → ALLOW and writes message into state."""
policy = detect_task_switch(min_turns=1)
result = await policy(_event("fix the login bug", history=[]))
assert result is not None
assert result["result"] == "ALLOW"
updates = {u["key"]: u["value"] for u in result["state_updates"]}
assert _TASK_SWITCH_HISTORY_KEY in updates
assert "fix the login bug" in updates[_TASK_SWITCH_HISTORY_KEY][0]
@pytest.mark.asyncio
async def test_below_min_turns_accumulates_without_classifying() -> None:
"""With min_turns=2, two messages accumulate before classification fires."""
policy = detect_task_switch(min_turns=2)
# Message 1 — history empty
r1 = await policy(_event("first task", history=[]))
assert r1["result"] == "ALLOW"
# Message 2 — one prior message, still below min_turns=2
r2 = await policy(_event("second message", history=["first task"]))
assert r2["result"] == "ALLOW"
# Both must have stored the new message into state
for r in (r1, r2):
assert any(u["key"] == _TASK_SWITCH_HISTORY_KEY for u in r["state_updates"])
@pytest.mark.asyncio
async def test_empty_message_abstains() -> None:
"""Blank / whitespace-only messages abstain (nothing to classify)."""
policy = detect_task_switch()
assert await policy(_event("")) is None
assert await policy(_event(" ")) is None
# ── no llm_client abstains ───────────────────────────────────────────────────
@pytest.mark.asyncio
async def test_no_llm_client_abstains_after_min_turns() -> None:
"""When min_turns is satisfied but llm_client is absent, fail-open (None)."""
policy = detect_task_switch(min_turns=1)
event = _event("brand new topic", history=["fix the login bug"])
# no llm_client key → abstain
result = await policy(event)
assert result is None
# ── CONTINUATION path (mocked llm_client) ───────────────────────────────────
class _MockLLMClient:
"""Stub PolicyLLMClient that returns a fixed verdict."""
def __init__(self, verdict: str) -> None:
self._verdict = verdict
self.calls: int = 0
async def create(self, **_kwargs: object) -> object:
self.calls += 1
class _Resp:
output_text = f'{{"verdict": "{self._verdict}"}}'
return _Resp()
@pytest.mark.asyncio
async def test_continuation_updates_history_and_allows() -> None:
"""A CONTINUATION verdict writes the new message into history and ALLOWs."""
client = _MockLLMClient("CONTINUATION")
policy = detect_task_switch(min_turns=1)
event = {
"type": "request",
"data": "also fix the logout bug",
"session_state": {_TASK_SWITCH_HISTORY_KEY: ["fix the login bug"]},
"llm_client": client,
}
result = await policy(event)
assert result is not None
assert result["result"] == "ALLOW"
updates = {u["key"]: u["value"] for u in result["state_updates"]}
history = updates[_TASK_SWITCH_HISTORY_KEY]
assert "fix the login bug" in history
assert "also fix the logout bug" in history
assert client.calls == 1
# ── TASK_SWITCH path ─────────────────────────────────────────────────────────
@pytest.mark.asyncio
async def test_task_switch_ask_returns_ask_and_resets_window() -> None:
"""A TASK_SWITCH verdict with action=ASK returns ASK and resets the window."""
client = _MockLLMClient("TASK_SWITCH")
policy = detect_task_switch(min_turns=1, action="ASK")
event = {
"type": "request",
"data": "write me a poem",
"session_state": {_TASK_SWITCH_HISTORY_KEY: ["fix the login bug"]},
"llm_client": client,
}
result = await policy(event)
assert result is not None
assert result["result"] == "ASK"
assert "reason" in result
# Window must be reset to contain only the switching message
updates = {u["key"]: u["value"] for u in result["state_updates"]}
assert updates[_TASK_SWITCH_HISTORY_KEY] == ["write me a poem"]
@pytest.mark.asyncio
async def test_task_switch_deny_returns_deny_and_resets_window() -> None:
"""A TASK_SWITCH verdict with action=DENY returns DENY and resets the window."""
client = _MockLLMClient("TASK_SWITCH")
policy = detect_task_switch(min_turns=1, action="DENY")
event = {
"type": "request",
"data": "write me a poem",
"session_state": {_TASK_SWITCH_HISTORY_KEY: ["fix the login bug"]},
"llm_client": client,
}
result = await policy(event)
assert result["result"] == "DENY"
updates = {u["key"]: u["value"] for u in result["state_updates"]}
assert updates[_TASK_SWITCH_HISTORY_KEY] == ["write me a poem"]
# ── code-fence robustness ────────────────────────────────────────────────────
@pytest.mark.asyncio
async def test_fenced_json_response_is_parsed() -> None:
"""JSON wrapped in code fences is handled (provider-robustness)."""
class _FencedClient:
async def create(self, **_kwargs: object) -> object:
class _R:
output_text = '```json\n{"verdict": "CONTINUATION"}\n```'
return _R()
policy = detect_task_switch(min_turns=1)
event = {
"type": "request",
"data": "follow-up question",
"session_state": {_TASK_SWITCH_HISTORY_KEY: ["prior message"]},
"llm_client": _FencedClient(),
}
result = await policy(event)
assert result is not None
assert result["result"] == "ALLOW"
# ── min_turns boundary ────────────────────────────────────────────────────────
@pytest.mark.asyncio
async def test_min_turns_zero_classifies_from_first_message() -> None:
"""min_turns=0 means classify even the very first message (no accumulation)."""
client = _MockLLMClient("CONTINUATION")
policy = detect_task_switch(min_turns=0)
event = {
"type": "request",
"data": "hello",
"session_state": {_TASK_SWITCH_HISTORY_KEY: []},
"llm_client": client,
}
result = await policy(event)
# With empty history, prior_context is empty but the call still fires
assert client.calls == 1
assert result is not None