Compare commits
6 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 1609d45757 | |||
| e70032e432 | |||
| 674b67b8f8 | |||
| 7974449464 | |||
| 87cd097c20 | |||
| d2c63c7634 |
@@ -44,5 +44,6 @@ BUILTIN_POLICY_MODULES = [
|
||||
"omnigent.policies.builtins.routing",
|
||||
"omnigent.policies.builtins.cel",
|
||||
"omnigent.policies.builtins.prompt",
|
||||
"omnigent.policies.builtins.context",
|
||||
"omnigent.inner.nessie.policies",
|
||||
]
|
||||
|
||||
@@ -0,0 +1,338 @@
|
||||
"""Built-in context-management policies.
|
||||
|
||||
Helps agents keep their working context lean. The guiding principle:
|
||||
the goal is not fewer tokens *used*, but fewer tokens *wasted* —
|
||||
sprawling context filled with stale tool results from a prior task
|
||||
degrades quality without adding value.
|
||||
|
||||
The recommended response to a denial is to start a fresh session for
|
||||
the new task rather than compacting or summarising in place.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from omnigent.policies.schema import PolicyCallable, PolicyEvent, PolicyResponse
|
||||
|
||||
_log = logging.getLogger(__name__)
|
||||
|
||||
# ── detect_task_switch ────────────────────────────────────────────────────────
|
||||
|
||||
_TASK_SWITCH_HISTORY_KEY = "_task_switch_history"
|
||||
|
||||
_DEFAULT_TASK_SWITCH_PROMPT = """\
|
||||
You are a conversation-continuity classifier for a coding assistant.
|
||||
|
||||
You are given the user's recent messages (the "prior context") and their
|
||||
latest message. Decide whether the latest message is a continuation of the
|
||||
same task or the start of a clearly different, unrelated task.
|
||||
|
||||
Guidelines:
|
||||
- CONTINUATION: the latest message follows naturally from prior work — a
|
||||
follow-up question, a refinement, a related sub-task, or asking about
|
||||
something mentioned earlier.
|
||||
- TASK_SWITCH: the latest message starts a completely different topic or
|
||||
codebase concern with no meaningful connection to what came before.
|
||||
- When in doubt, prefer CONTINUATION — false positives (blocking a legitimate
|
||||
continuation) are more harmful than false negatives.
|
||||
|
||||
Return strict JSON only:
|
||||
{"verdict": "CONTINUATION" | "TASK_SWITCH"}
|
||||
"""
|
||||
|
||||
_TASK_SWITCH_SCHEMA: dict[str, Any] = {
|
||||
"format": {
|
||||
"type": "json_schema",
|
||||
"name": "task_switch_verdict",
|
||||
"strict": True,
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"verdict": {
|
||||
"type": "string",
|
||||
"enum": ["CONTINUATION", "TASK_SWITCH"],
|
||||
},
|
||||
},
|
||||
"required": ["verdict"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _strip_code_fences(text: str) -> str:
|
||||
"""Strip markdown code fences from LLM output.
|
||||
|
||||
Even with structured output, some providers wrap JSON in
|
||||
triple-backtick fences. This strips the outermost fence
|
||||
so ``json.loads`` succeeds.
|
||||
|
||||
:param text: Raw LLM response text.
|
||||
:returns: Text with code fences removed.
|
||||
"""
|
||||
stripped = text.strip()
|
||||
if stripped.startswith("```"):
|
||||
first_newline = stripped.find("\n")
|
||||
if first_newline != -1:
|
||||
stripped = stripped[first_newline + 1 :]
|
||||
if stripped.rstrip().endswith("```"):
|
||||
stripped = stripped.rstrip()[:-3].rstrip()
|
||||
return stripped
|
||||
|
||||
|
||||
def _extract_text(response: Any) -> str:
|
||||
"""Pull plain text out of a PolicyLLMClient response."""
|
||||
text = getattr(response, "output_text", None)
|
||||
if isinstance(text, str) and text.strip():
|
||||
return text.strip()
|
||||
output = getattr(response, "output", None)
|
||||
if not isinstance(output, list) or not output:
|
||||
return ""
|
||||
content = getattr(output[0], "content", None)
|
||||
if not isinstance(content, list) or not content:
|
||||
return ""
|
||||
return getattr(content[0], "text", "") or ""
|
||||
|
||||
|
||||
def detect_task_switch(
|
||||
*,
|
||||
min_turns: int = 1,
|
||||
history_window: int = 10,
|
||||
action: str = "ASK",
|
||||
classification_prompt: str = _DEFAULT_TASK_SWITCH_PROMPT,
|
||||
) -> PolicyCallable:
|
||||
"""Factory: detect when the user switches to an unrelated task.
|
||||
|
||||
Fires on ``request`` events. Maintains a rolling window of recent
|
||||
user messages in ``session_state`` and, once ``min_turns`` prior
|
||||
messages have accumulated, asks the server-level LLM to classify the
|
||||
latest message as ``CONTINUATION`` or ``TASK_SWITCH``.
|
||||
|
||||
On ``TASK_SWITCH`` the policy returns *action* with a message
|
||||
recommending a fresh session — not compaction or summarisation. The
|
||||
window is reset to contain only the switching message so the new
|
||||
task can accumulate its own history from a clean state (``DENY``
|
||||
path only — the ``ASK`` path cannot write state on decline, so the
|
||||
window advances only once the user approves and the next request
|
||||
arrives).
|
||||
On ``CONTINUATION`` the policy records the new message into state
|
||||
and abstains, letting the request through.
|
||||
|
||||
Requires the server to have an ``llm:`` config block; abstains
|
||||
(fail-open) when no LLM client is available.
|
||||
|
||||
.. note::
|
||||
``action="DENY"`` is not a security control — user messages are
|
||||
interpolated into the classifier prompt and a determined user can
|
||||
craft a message that forces a ``CONTINUATION`` verdict (prompt
|
||||
injection). Use this policy for context-hygiene guidance, not
|
||||
for access control.
|
||||
|
||||
:param min_turns: Number of prior messages to accumulate before the
|
||||
classifier starts firing. With the default of ``1`` the
|
||||
classifier fires on the **second** user message (one prior
|
||||
message is enough context to detect a switch). Set to ``0`` to
|
||||
classify from the very first message.
|
||||
:param history_window: Maximum number of recent user messages kept
|
||||
in state as prior context for the classifier. Defaults to
|
||||
``10``. Older messages are dropped as the window slides.
|
||||
:param action: Response when a task switch is detected.
|
||||
``"ASK"`` (default) escalates to the user; ``"DENY"`` blocks
|
||||
the request outright. Defaults to ``"ASK"`` because
|
||||
false-positive task-switch classifications are more harmful than
|
||||
false negatives.
|
||||
:param classification_prompt: System prompt for the classifier LLM
|
||||
call. Must instruct the model to return
|
||||
``{"verdict": "CONTINUATION"|"TASK_SWITCH"}``; the schema is
|
||||
enforced via structured output regardless.
|
||||
:returns: An async policy callable that fires on ``request`` events.
|
||||
"""
|
||||
normalised_action = action.upper() if isinstance(action, str) else "ASK"
|
||||
if normalised_action not in {"DENY", "ASK"}:
|
||||
_log.warning(
|
||||
"detect_task_switch: unknown action %r — defaulting to ASK",
|
||||
action,
|
||||
)
|
||||
normalised_action = "ASK"
|
||||
|
||||
async def evaluate(event: PolicyEvent) -> PolicyResponse | None:
|
||||
"""Classify the new user message and flag task switches.
|
||||
|
||||
Reads ``session_state[_TASK_SWITCH_HISTORY_KEY]`` for prior
|
||||
context and writes the updated window back via
|
||||
``state_updates``.
|
||||
|
||||
:param event: Policy event dict.
|
||||
:returns: *action* when a task switch is detected; ``None``
|
||||
(abstain) otherwise.
|
||||
"""
|
||||
if event.get("type") != "request":
|
||||
return None
|
||||
|
||||
new_message = event.get("data", "")
|
||||
if not isinstance(new_message, str) or not new_message.strip():
|
||||
return None
|
||||
|
||||
state = event.get("session_state") or {}
|
||||
history: list[str] = state.get(_TASK_SWITCH_HISTORY_KEY) or []
|
||||
|
||||
# Slide the window: append new message, keep last history_window entries.
|
||||
updated_history = [*history, new_message[:500]][-history_window:]
|
||||
|
||||
# Not enough prior turns yet — record and pass through.
|
||||
if len(history) < min_turns:
|
||||
_log.debug(
|
||||
"detect_task_switch: history_len=%d < min_turns=%d — accumulating",
|
||||
len(history),
|
||||
min_turns,
|
||||
)
|
||||
return {
|
||||
"result": "ALLOW",
|
||||
"state_updates": [
|
||||
{
|
||||
"key": _TASK_SWITCH_HISTORY_KEY,
|
||||
"action": "set",
|
||||
"value": updated_history,
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
# ── Classify ────────────────────────────────────────────────────
|
||||
llm_client = event.get("llm_client")
|
||||
if llm_client is None:
|
||||
_log.warning(
|
||||
"detect_task_switch: no llm_client — server has no llm: config. Abstaining."
|
||||
)
|
||||
return None
|
||||
|
||||
prior_context = "\n".join(f"- {msg}" for msg in history[-history_window:])
|
||||
user_prompt = f"Prior messages:\n{prior_context}\n\nLatest message:\n{new_message[:500]}"
|
||||
|
||||
try:
|
||||
response = await llm_client.create(
|
||||
instructions=classification_prompt,
|
||||
input=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "input_text", "text": user_prompt}],
|
||||
}
|
||||
],
|
||||
text=_TASK_SWITCH_SCHEMA,
|
||||
)
|
||||
raw = _extract_text(response)
|
||||
if not raw:
|
||||
return None
|
||||
raw = _strip_code_fences(raw)
|
||||
verdict_obj = json.loads(raw)
|
||||
except Exception: # noqa: BLE001 — fail-open
|
||||
_log.exception("detect_task_switch: classifier call failed")
|
||||
return None
|
||||
|
||||
verdict = verdict_obj.get("verdict", "") if isinstance(verdict_obj, dict) else ""
|
||||
|
||||
if verdict == "TASK_SWITCH":
|
||||
_log.info("detect_task_switch: TASK_SWITCH detected — action=%s", normalised_action)
|
||||
# Reset the window to the switching message alone so the new
|
||||
# task accumulates fresh history from here. state_updates on a
|
||||
# DENY are applied immediately; state_updates on an ASK are only
|
||||
# applied if the user approves — on decline the window stays
|
||||
# pinned, so the next message will be re-classified against the
|
||||
# pre-switch context (which is the safe / over-prompting direction).
|
||||
return {
|
||||
"result": normalised_action,
|
||||
"reason": (
|
||||
"This message looks like the start of a new, unrelated task. "
|
||||
"The current session carries context from prior work that will "
|
||||
"waste capacity without helping here. "
|
||||
"Start a fresh session for this task to keep context lean."
|
||||
),
|
||||
"state_updates": [
|
||||
{
|
||||
"key": _TASK_SWITCH_HISTORY_KEY,
|
||||
"action": "set",
|
||||
"value": [new_message[:500]],
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
if verdict == "CONTINUATION":
|
||||
# Update history and let the request through.
|
||||
return {
|
||||
"result": "ALLOW",
|
||||
"state_updates": [
|
||||
{
|
||||
"key": _TASK_SWITCH_HISTORY_KEY,
|
||||
"action": "set",
|
||||
"value": updated_history,
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
# Unrecognised verdict — fail open.
|
||||
return None
|
||||
|
||||
return evaluate # type: ignore[return-value]
|
||||
|
||||
|
||||
# ── Registry ──────────────────────────────────────────────────────────────────
|
||||
|
||||
POLICY_REGISTRY: list[dict[str, Any]] = [
|
||||
{
|
||||
"handler": "omnigent.policies.builtins.context.detect_task_switch",
|
||||
"kind": "factory",
|
||||
"name": "Detect Task Switch",
|
||||
"description": (
|
||||
"Uses the server-level LLM to classify each user message as a "
|
||||
"continuation of the current task or the start of a new, unrelated "
|
||||
"one. On a detected task switch, asks (or denies) with a recommendation "
|
||||
"to start a fresh session. Implements the 'Keep Context Lean' strategy: "
|
||||
"start fresh sessions when switching tasks rather than accumulating "
|
||||
"stale context. Requires an llm: config block on the server; "
|
||||
"abstains (fail-open) when no LLM client is available."
|
||||
),
|
||||
"params_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"min_turns": {
|
||||
"type": "integer",
|
||||
"default": 2,
|
||||
"description": (
|
||||
"Number of prior user messages to accumulate before "
|
||||
"the classifier starts firing. Defaults to 2."
|
||||
),
|
||||
},
|
||||
"history_window": {
|
||||
"type": "integer",
|
||||
"default": 4,
|
||||
"description": (
|
||||
"Maximum number of recent user messages kept as prior "
|
||||
"context for the classifier. Older messages are dropped "
|
||||
"as the window slides. Defaults to 10."
|
||||
),
|
||||
},
|
||||
"action": {
|
||||
"type": "string",
|
||||
"enum": ["ASK", "DENY"],
|
||||
"default": "ASK",
|
||||
"description": (
|
||||
"Response when a task switch is detected. "
|
||||
"ASK escalates to the user (default); "
|
||||
"DENY blocks the request outright."
|
||||
),
|
||||
},
|
||||
"classification_prompt": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"System prompt for the classifier. Must instruct the "
|
||||
'model to return {"verdict": "CONTINUATION"|"TASK_SWITCH"}; '
|
||||
"the output schema is enforced via structured output."
|
||||
),
|
||||
},
|
||||
},
|
||||
"required": [],
|
||||
},
|
||||
},
|
||||
]
|
||||
@@ -0,0 +1,233 @@
|
||||
"""Unit tests for omnigent.policies.builtins.context."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from omnigent.policies.builtins.context import (
|
||||
_TASK_SWITCH_HISTORY_KEY,
|
||||
_strip_code_fences,
|
||||
detect_task_switch,
|
||||
)
|
||||
|
||||
# ── helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _event(
|
||||
message: str,
|
||||
*,
|
||||
history: list[str] | None = None,
|
||||
phase: str = "request",
|
||||
) -> dict:
|
||||
return {
|
||||
"type": phase,
|
||||
"data": message,
|
||||
"session_state": {_TASK_SWITCH_HISTORY_KEY: history or []},
|
||||
}
|
||||
|
||||
|
||||
# ── _strip_code_fences ───────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_strip_code_fences_plain_json() -> None:
|
||||
assert _strip_code_fences('{"verdict":"CONTINUATION"}') == '{"verdict":"CONTINUATION"}'
|
||||
|
||||
|
||||
def test_strip_code_fences_with_fence() -> None:
|
||||
assert (
|
||||
_strip_code_fences('```json\n{"verdict":"TASK_SWITCH"}\n```')
|
||||
== '{"verdict":"TASK_SWITCH"}'
|
||||
)
|
||||
|
||||
|
||||
def test_strip_code_fences_bare_fence() -> None:
|
||||
assert _strip_code_fences('```\n{"v":"x"}\n```') == '{"v":"x"}'
|
||||
|
||||
|
||||
# ── non-gated phases abstain ─────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_request_phases_abstain() -> None:
|
||||
"""Only ``request`` events are evaluated; all others abstain."""
|
||||
policy = detect_task_switch()
|
||||
for phase in ("tool_call", "tool_result", "response", "llm_request"):
|
||||
result = await policy(_event("hello", phase=phase))
|
||||
assert result is None, f"expected None for phase={phase}"
|
||||
|
||||
|
||||
# ── accumulation (below min_turns) ───────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_first_message_accumulates_no_history() -> None:
|
||||
"""First message (history empty) → ALLOW and writes message into state."""
|
||||
policy = detect_task_switch(min_turns=1)
|
||||
result = await policy(_event("fix the login bug", history=[]))
|
||||
assert result is not None
|
||||
assert result["result"] == "ALLOW"
|
||||
updates = {u["key"]: u["value"] for u in result["state_updates"]}
|
||||
assert _TASK_SWITCH_HISTORY_KEY in updates
|
||||
assert "fix the login bug" in updates[_TASK_SWITCH_HISTORY_KEY][0]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_below_min_turns_accumulates_without_classifying() -> None:
|
||||
"""With min_turns=2, two messages accumulate before classification fires."""
|
||||
policy = detect_task_switch(min_turns=2)
|
||||
# Message 1 — history empty
|
||||
r1 = await policy(_event("first task", history=[]))
|
||||
assert r1["result"] == "ALLOW"
|
||||
# Message 2 — one prior message, still below min_turns=2
|
||||
r2 = await policy(_event("second message", history=["first task"]))
|
||||
assert r2["result"] == "ALLOW"
|
||||
# Both must have stored the new message into state
|
||||
for r in (r1, r2):
|
||||
assert any(u["key"] == _TASK_SWITCH_HISTORY_KEY for u in r["state_updates"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_message_abstains() -> None:
|
||||
"""Blank / whitespace-only messages abstain (nothing to classify)."""
|
||||
policy = detect_task_switch()
|
||||
assert await policy(_event("")) is None
|
||||
assert await policy(_event(" ")) is None
|
||||
|
||||
|
||||
# ── no llm_client abstains ───────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_llm_client_abstains_after_min_turns() -> None:
|
||||
"""When min_turns is satisfied but llm_client is absent, fail-open (None)."""
|
||||
policy = detect_task_switch(min_turns=1)
|
||||
event = _event("brand new topic", history=["fix the login bug"])
|
||||
# no llm_client key → abstain
|
||||
result = await policy(event)
|
||||
assert result is None
|
||||
|
||||
|
||||
# ── CONTINUATION path (mocked llm_client) ───────────────────────────────────
|
||||
|
||||
|
||||
class _MockLLMClient:
|
||||
"""Stub PolicyLLMClient that returns a fixed verdict."""
|
||||
|
||||
def __init__(self, verdict: str) -> None:
|
||||
self._verdict = verdict
|
||||
self.calls: int = 0
|
||||
|
||||
async def create(self, **_kwargs: object) -> object:
|
||||
self.calls += 1
|
||||
|
||||
class _Resp:
|
||||
output_text = f'{{"verdict": "{self._verdict}"}}'
|
||||
|
||||
return _Resp()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_continuation_updates_history_and_allows() -> None:
|
||||
"""A CONTINUATION verdict writes the new message into history and ALLOWs."""
|
||||
client = _MockLLMClient("CONTINUATION")
|
||||
policy = detect_task_switch(min_turns=1)
|
||||
event = {
|
||||
"type": "request",
|
||||
"data": "also fix the logout bug",
|
||||
"session_state": {_TASK_SWITCH_HISTORY_KEY: ["fix the login bug"]},
|
||||
"llm_client": client,
|
||||
}
|
||||
result = await policy(event)
|
||||
assert result is not None
|
||||
assert result["result"] == "ALLOW"
|
||||
updates = {u["key"]: u["value"] for u in result["state_updates"]}
|
||||
history = updates[_TASK_SWITCH_HISTORY_KEY]
|
||||
assert "fix the login bug" in history
|
||||
assert "also fix the logout bug" in history
|
||||
assert client.calls == 1
|
||||
|
||||
|
||||
# ── TASK_SWITCH path ─────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_task_switch_ask_returns_ask_and_resets_window() -> None:
|
||||
"""A TASK_SWITCH verdict with action=ASK returns ASK and resets the window."""
|
||||
client = _MockLLMClient("TASK_SWITCH")
|
||||
policy = detect_task_switch(min_turns=1, action="ASK")
|
||||
event = {
|
||||
"type": "request",
|
||||
"data": "write me a poem",
|
||||
"session_state": {_TASK_SWITCH_HISTORY_KEY: ["fix the login bug"]},
|
||||
"llm_client": client,
|
||||
}
|
||||
result = await policy(event)
|
||||
assert result is not None
|
||||
assert result["result"] == "ASK"
|
||||
assert "reason" in result
|
||||
# Window must be reset to contain only the switching message
|
||||
updates = {u["key"]: u["value"] for u in result["state_updates"]}
|
||||
assert updates[_TASK_SWITCH_HISTORY_KEY] == ["write me a poem"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_task_switch_deny_returns_deny_and_resets_window() -> None:
|
||||
"""A TASK_SWITCH verdict with action=DENY returns DENY and resets the window."""
|
||||
client = _MockLLMClient("TASK_SWITCH")
|
||||
policy = detect_task_switch(min_turns=1, action="DENY")
|
||||
event = {
|
||||
"type": "request",
|
||||
"data": "write me a poem",
|
||||
"session_state": {_TASK_SWITCH_HISTORY_KEY: ["fix the login bug"]},
|
||||
"llm_client": client,
|
||||
}
|
||||
result = await policy(event)
|
||||
assert result["result"] == "DENY"
|
||||
updates = {u["key"]: u["value"] for u in result["state_updates"]}
|
||||
assert updates[_TASK_SWITCH_HISTORY_KEY] == ["write me a poem"]
|
||||
|
||||
|
||||
# ── code-fence robustness ────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fenced_json_response_is_parsed() -> None:
|
||||
"""JSON wrapped in code fences is handled (provider-robustness)."""
|
||||
|
||||
class _FencedClient:
|
||||
async def create(self, **_kwargs: object) -> object:
|
||||
class _R:
|
||||
output_text = '```json\n{"verdict": "CONTINUATION"}\n```'
|
||||
|
||||
return _R()
|
||||
|
||||
policy = detect_task_switch(min_turns=1)
|
||||
event = {
|
||||
"type": "request",
|
||||
"data": "follow-up question",
|
||||
"session_state": {_TASK_SWITCH_HISTORY_KEY: ["prior message"]},
|
||||
"llm_client": _FencedClient(),
|
||||
}
|
||||
result = await policy(event)
|
||||
assert result is not None
|
||||
assert result["result"] == "ALLOW"
|
||||
|
||||
|
||||
# ── min_turns boundary ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_min_turns_zero_classifies_from_first_message() -> None:
|
||||
"""min_turns=0 means classify even the very first message (no accumulation)."""
|
||||
client = _MockLLMClient("CONTINUATION")
|
||||
policy = detect_task_switch(min_turns=0)
|
||||
event = {
|
||||
"type": "request",
|
||||
"data": "hello",
|
||||
"session_state": {_TASK_SWITCH_HISTORY_KEY: []},
|
||||
"llm_client": client,
|
||||
}
|
||||
result = await policy(event)
|
||||
# With empty history, prior_context is empty but the call still fires
|
||||
assert client.calls == 1
|
||||
assert result is not None
|
||||
Reference in New Issue
Block a user