fix(agent): correct the runtime facts the agent reports — cloud, OS, cross-turn evidence, connected integrations (#4941)
This commit is contained in:
+4
-1
@@ -738,7 +738,10 @@ OPENSRE_MASK_EXTRA_REGEX=
|
||||
|
||||
# Cloud identity for the agent's runtime facts, injected at deploy time.
|
||||
# Read by session metadata; never fetched from the instance metadata service (IMDS).
|
||||
# CLOUD_REGION falls back to AWS_REGION / AWS_DEFAULT_REGION when unset.
|
||||
# Set CLOUD_PROVIDER on silos (e.g. aws). AWS_REGION alone does NOT mean this
|
||||
# process is running in AWS — laptop .env often sets AWS_REGION for tools.
|
||||
# When CLOUD_PROVIDER is set, CLOUD_REGION may fall back to AWS_REGION /
|
||||
# AWS_DEFAULT_REGION.
|
||||
CLOUD_PROVIDER=
|
||||
CLOUD_REGION=
|
||||
|
||||
|
||||
@@ -21,6 +21,7 @@ from config.runtime_metadata.probes import (
|
||||
capability_warning_facts,
|
||||
cloud_facts,
|
||||
disk_memory_facts,
|
||||
host_os_facts,
|
||||
installed_tools,
|
||||
kubeconfig_path,
|
||||
local_tz_name,
|
||||
@@ -44,6 +45,7 @@ def build_runtime_metadata() -> dict[str, Any]:
|
||||
- ``opensre_build`` — ``""`` in released wheels; ``dev, v0.1.YYYY.M.D @ SHA``
|
||||
in a git checkout so the LLM can quote the exact build in local dev.
|
||||
- ``runtime_env`` — ``OPENSRE_ENV`` env var, else the app environment name.
|
||||
- ``os_family`` — host OS name (macOS/Linux/Windows).
|
||||
- ``tz_name`` — local timezone name (rarely changes mid-session).
|
||||
- ``python_version`` — interpreter version from :data:`sys.version_info`.
|
||||
- ``pid`` / ``ppid`` — this process and its parent from :mod:`os`.
|
||||
@@ -52,9 +54,9 @@ def build_runtime_metadata() -> dict[str, Any]:
|
||||
- ``hostname`` — from ``/etc/hostname`` (the pod name in Kubernetes) or
|
||||
:func:`socket.gethostname`, never the ``hostname`` binary.
|
||||
- ``scratchpad_dir`` — the temp directory scripts may write to.
|
||||
- ``cloud_provider`` / ``cloud_region`` — deploy-time env vars
|
||||
(``CLOUD_PROVIDER`` / ``CLOUD_REGION``, AWS var fallback), never the
|
||||
instance metadata service (IMDS).
|
||||
- ``cloud_provider`` / ``cloud_region`` — deploy-time ``CLOUD_PROVIDER`` /
|
||||
``CLOUD_REGION`` only (``AWS_REGION`` alone does not imply AWS), never
|
||||
the instance metadata service (IMDS).
|
||||
|
||||
The exact key set is :data:`STATIC_FACT_KEYS` (contract-tested). Live
|
||||
values that must NOT be cached (current time, uptime, disk, memory) come
|
||||
@@ -66,6 +68,7 @@ def build_runtime_metadata() -> dict[str, Any]:
|
||||
"opensre_version": get_opensre_version(),
|
||||
"opensre_build": detect_build_info(),
|
||||
"runtime_env": env_override or get_environment().value,
|
||||
**host_os_facts(),
|
||||
"tz_name": local_tz_name(),
|
||||
"python_version": python_version_string(),
|
||||
"pid": os.getpid(),
|
||||
|
||||
@@ -17,6 +17,7 @@ STATIC_FACT_KEYS: tuple[str, ...] = (
|
||||
"opensre_version",
|
||||
"opensre_build",
|
||||
"runtime_env",
|
||||
"os_family",
|
||||
"tz_name",
|
||||
"python_version",
|
||||
"pid",
|
||||
@@ -58,6 +59,7 @@ BLOCKED_INTROSPECTION_COMMANDS: tuple[str, ...] = (
|
||||
"date",
|
||||
"uptime",
|
||||
"hostname",
|
||||
"uname",
|
||||
"ls",
|
||||
"df",
|
||||
"free",
|
||||
|
||||
@@ -43,6 +43,19 @@ _LOCALTIME_LINK = Path("/etc/localtime")
|
||||
|
||||
_HOSTNAME_FILE = Path("/etc/hostname")
|
||||
|
||||
# Values of CLOUD_PROVIDER for which AWS_REGION / AWS_DEFAULT_REGION describe
|
||||
# the same deployment. Any other provider keeps its own region source.
|
||||
_AWS_PROVIDER_NAMES: Final[frozenset[str]] = frozenset({"aws", "amazon"})
|
||||
|
||||
# ``sys.platform`` prefixes mapped to the OS name a user would recognise.
|
||||
# Prefix-matched: Linux reports linux/linux2 and BSDs carry a version suffix.
|
||||
_OS_FAMILY_BY_PLATFORM_PREFIX: Final[tuple[tuple[str, str], ...]] = (
|
||||
("darwin", "macOS"),
|
||||
("linux", "Linux"),
|
||||
("win", "Windows"),
|
||||
("freebsd", "FreeBSD"),
|
||||
)
|
||||
|
||||
|
||||
def local_tz_name() -> str:
|
||||
"""Best-effort local timezone name — IANA (``Europe/Berlin``) when possible.
|
||||
@@ -126,25 +139,46 @@ def disk_memory_facts() -> dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
def host_os_facts() -> dict[str, str]:
|
||||
"""Host OS family — ``macOS``, ``Linux``, ``Windows``.
|
||||
|
||||
Always present so "what environment are you running in?" has a true local
|
||||
answer instead of a vacuum the model fills with a cloud guess.
|
||||
|
||||
No version: ``platform.release()`` is the *kernel* release — on macOS the
|
||||
Darwin number (25.5.0), not the macOS version (26.5.2) — so publishing it
|
||||
as the OS release states a false fact in the block that exists to prevent
|
||||
them.
|
||||
|
||||
``sys.platform`` rather than ``platform.system()``: this repo ships its own
|
||||
``platform`` package (see ``config/secrets/os_keyring.py``), so the stdlib
|
||||
name only resolves through a shim.
|
||||
"""
|
||||
identifier = sys.platform
|
||||
for prefix, family in _OS_FAMILY_BY_PLATFORM_PREFIX:
|
||||
if identifier.startswith(prefix):
|
||||
return {"os_family": family}
|
||||
return {"os_family": identifier or "unknown"}
|
||||
|
||||
|
||||
def cloud_facts() -> dict[str, str]:
|
||||
"""Cloud provider/region from deploy-time env vars — no metadata endpoint.
|
||||
|
||||
``CLOUD_PROVIDER`` / ``CLOUD_REGION`` are the canonical injection points
|
||||
(set at deploy time). Region falls back to ``AWS_REGION`` /
|
||||
``AWS_DEFAULT_REGION`` — the same pair the LLM transports already read —
|
||||
and when the region came from an AWS var the provider defaults to ``aws``.
|
||||
Never calls the instance metadata service (IMDS); the sandbox blocks
|
||||
network anyway.
|
||||
(set at deploy time). ``AWS_REGION`` / ``AWS_DEFAULT_REGION`` alone must
|
||||
**not** claim this process is running in AWS — those vars are routine on
|
||||
developer laptops (``.env.example`` ships ``AWS_REGION=us-east-1`` for the
|
||||
AWS integration). They may fill in the region only when the provider is
|
||||
itself AWS: an AWS region under ``CLOUD_PROVIDER=gcp`` would be a wrong
|
||||
location stated as authoritative runtime identity. Never calls the instance
|
||||
metadata service (IMDS).
|
||||
"""
|
||||
provider = (os.environ.get("CLOUD_PROVIDER") or "").strip()
|
||||
region = (os.environ.get("CLOUD_REGION") or "").strip()
|
||||
aws_region = (
|
||||
os.environ.get("AWS_REGION") or os.environ.get("AWS_DEFAULT_REGION") or ""
|
||||
).strip()
|
||||
if not region and aws_region:
|
||||
region = aws_region
|
||||
if not provider:
|
||||
provider = "aws"
|
||||
if not region and provider.lower() in _AWS_PROVIDER_NAMES:
|
||||
region = (
|
||||
os.environ.get("AWS_REGION") or os.environ.get("AWS_DEFAULT_REGION") or ""
|
||||
).strip()
|
||||
return {"cloud_provider": provider, "cloud_region": region}
|
||||
|
||||
|
||||
@@ -282,6 +316,7 @@ __all__ = [
|
||||
"capability_warning_facts",
|
||||
"cloud_facts",
|
||||
"disk_memory_facts",
|
||||
"host_os_facts",
|
||||
"installed_tools",
|
||||
"kubeconfig_path",
|
||||
"local_tz_name",
|
||||
|
||||
@@ -338,6 +338,10 @@ slash_invoke:
|
||||
This should run the wizard for them; do not hand off just to tell the user to
|
||||
type the command. If no service/server is named, use assistant_handoff to ask
|
||||
which one.
|
||||
Do NOT treat a request to *query/read* a named database tool (active
|
||||
connections, status, dashboard) as setup/enable — that is assistant_handoff
|
||||
with ``database_query:<topic>``, even when a first-party MySQL/MariaDB
|
||||
integration exists.
|
||||
Other tools:
|
||||
- llm_set_provider — switch provider ONLY when the user names an EXACT provider
|
||||
target (e.g. "switch to anthropic", "use openai", "set provider to ollama").
|
||||
@@ -532,6 +536,13 @@ service. Requests to list/query Datadog monitors, Grafana logs, Sentry issues,
|
||||
PostHog events, traces, sessions, or similar integration data are data lookups:
|
||||
emit assistant_handoff so the conversational gather loop can use the integration
|
||||
tools. Do not substitute `/integrations show <service>` for those records.
|
||||
It also does NOT apply to querying or reading data from a named database tool
|
||||
(MySQL, MariaDB, Postgres, etc.) — including prompts that cite a tool id such as
|
||||
``mysql-…`` and ask for active connections, status, or a dashboard read. Those
|
||||
are ``database_query:<topic>`` handoffs (see below). Do NOT emit
|
||||
slash_invoke ``/integrations verify|setup <service>`` or ``/mcp connect`` as a
|
||||
stand-in for answering the query; the assistant explains connect/setup after
|
||||
the handoff. Do NOT set session_goal=true on those handoffs.
|
||||
A vendor's own teammate-messaging actions (channel history, thread reads,
|
||||
workspace search, roster, join, reply, task capture, etc.) are NOT this
|
||||
category — use that vendor's action tools instead (see its action-prompt
|
||||
@@ -576,7 +587,10 @@ call rather than relying on plain-text output. Use concise structured content ta
|
||||
when the topic is known — for example docs:datadog_setup, chat:greeting,
|
||||
provider:local_llama_connect for vague local-model connection requests, or
|
||||
database_query:<topic> when the user asks to query/read a named database tool
|
||||
(MySQL, MariaDB, etc.) that is not a first-party setup-wizard target.
|
||||
(MySQL, MariaDB, Postgres, etc.) — including first-party integrations and MCP
|
||||
tool ids. Emit ONLY that assistant_handoff (no slash_invoke setup/verify
|
||||
alongside it). Example: "Use the MySQL tool (ID: mysql-…) to query active
|
||||
connections" → assistant_handoff(content="database_query:mysql_active_connections").
|
||||
Also set these structured assistant_handoff fields when they apply (the harness
|
||||
keys policy off them; it does not scan user prose for intent). Prefer the
|
||||
schema fields over burying tags in content prose:
|
||||
@@ -595,7 +609,8 @@ schema fields over burying tags in content prose:
|
||||
work), and for metric_read count questions. The host session-goal loop keys
|
||||
off this boolean; omitting it drops continuation (except metric_read, which
|
||||
the host treats as attach). Prefer session_goal_items=["…", …] for checklist
|
||||
criteria.
|
||||
criteria. Do NOT set session_goal=true on database_query handoffs — missing
|
||||
DB connectivity is explained in one reply, not a multi-turn goal loop.
|
||||
- session_goal_max_turns=<n> — optional session-goal turn cap for that goal.
|
||||
- session_goal_items=["…", …] — checklist success criteria (one string per
|
||||
item, in order). The host tracks completion via session_goal:done=<index>
|
||||
|
||||
@@ -96,10 +96,24 @@ def _assistant_context_blocks(
|
||||
|
||||
|
||||
def _build_integration_guard(ctx: TurnSnapshot) -> str:
|
||||
"""Render the no-integrations guidance block from the turn snapshot."""
|
||||
if not (ctx.configured_integrations_known and not ctx.configured_integrations):
|
||||
"""Render what is connected, and the no-integrations guidance when empty.
|
||||
|
||||
Naming the connected set lets "X is not connected" be answered with what
|
||||
*is* — the difference between an assertion and a checked result. The data
|
||||
already reaches the gather prompt; the answer path was told only when the
|
||||
set was empty, so a reply could not say what it had looked at.
|
||||
"""
|
||||
if not ctx.configured_integrations_known:
|
||||
return ""
|
||||
|
||||
if ctx.configured_integrations:
|
||||
connected = ", ".join(ctx.configured_integrations)
|
||||
return (
|
||||
f"Integrations connected in this session: {connected}. When the user "
|
||||
"asks about a data source that is not in that list, say which ones "
|
||||
"are connected rather than only that theirs is missing.\n\n"
|
||||
)
|
||||
|
||||
return (
|
||||
"No integrations are configured in this session. You may still help the user "
|
||||
"configure one: explain `/integrations setup <service>` for integrations or "
|
||||
|
||||
@@ -23,11 +23,15 @@ _BLOCKED_COMMANDS = ", ".join(f"`{command}`" for command in BLOCKED_INTROSPECTIO
|
||||
_STATIC_GUIDANCE = (
|
||||
". When the user asks which OpenSRE version is running, reply with the "
|
||||
"full version string above verbatim — including any parenthetical suffix. "
|
||||
"When the user asks for the local timezone name, Python version, process "
|
||||
"When the user asks what environment this process is running in, or for "
|
||||
"the host operating system, local timezone name, Python version, process "
|
||||
"id, parent process id, host/pod name, cloud provider or region, "
|
||||
"kubeconfig path, or which tools are installed, answer from the strings "
|
||||
"above directly, WITHOUT any tool call — these facts are authoritative "
|
||||
"and re-reading them through the sandbox wastes a round-trip. Never run "
|
||||
"and re-reading them through the sandbox wastes a round-trip. Quote the "
|
||||
"host operating system for environment questions; when no cloud provider "
|
||||
"was detected, say so — never invent AWS, GCP, Azure, or a region. Never "
|
||||
"run "
|
||||
f"{_BLOCKED_COMMANDS}, and never probe cloud instance metadata over the "
|
||||
"network. To list files in the scratchpad or another directory, "
|
||||
"use the Python execution sandbox with `pathlib.Path(...).iterdir()` — "
|
||||
@@ -126,6 +130,14 @@ def _tools_line(runtime: Mapping[str, Any]) -> str | None:
|
||||
return f"installed tools on PATH are {', '.join(present)}"
|
||||
|
||||
|
||||
def _host_os_line(runtime: Mapping[str, Any]) -> str | None:
|
||||
"""Host OS — always stated when the key was probed."""
|
||||
if "os_family" not in runtime:
|
||||
return None
|
||||
family = _clean_str(runtime, "os_family")
|
||||
return f"host operating system is {family}" if family else None
|
||||
|
||||
|
||||
def _cloud_line(runtime: Mapping[str, Any]) -> str | None:
|
||||
"""Cloud identity, or an explicit statement that none was detected.
|
||||
|
||||
@@ -187,6 +199,7 @@ def _capability_warnings_line(runtime: Mapping[str, Any]) -> str | None:
|
||||
_STATIC_FACT_PRODUCERS: tuple[FactProducer, ...] = (
|
||||
_version_line,
|
||||
_str_fact("runtime_env", "runtime environment is {}"),
|
||||
_host_os_line,
|
||||
_str_fact("hostname", "host name is {}"),
|
||||
_str_fact("tz_name", "local timezone is {}"),
|
||||
_str_fact("python_version", "Python interpreter version is {}"),
|
||||
|
||||
@@ -17,6 +17,13 @@ def continuation_nudge(goal: SessionGoal) -> str:
|
||||
"""User-visible follow-up message for the next session-goal turn."""
|
||||
reason = goal.last_reason.strip() or derive_session_goal_reason(goal)
|
||||
reason_block = f"Last progress: {reason}\n\n"
|
||||
if goal.findings:
|
||||
established = "\n".join(f" - {item}" for item in goal.findings)
|
||||
reason_block += (
|
||||
"Already established in earlier turns of this goal — treat these as "
|
||||
"done and do not report them as unavailable:\n"
|
||||
f"{established}\n\n"
|
||||
)
|
||||
unfinished = goal.unfinished_items
|
||||
if unfinished:
|
||||
pending = "\n".join(f" - [{index}] {item}" for index, item in unfinished)
|
||||
|
||||
@@ -29,7 +29,7 @@ from collections.abc import Sequence
|
||||
from dataclasses import dataclass, replace
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from core.agent_harness.turns.handoff_tag_parse import find_tag_suffix
|
||||
from core.agent_harness.turns.handoff_tag_parse import find_tag_suffix, handoff_has_tag
|
||||
from platform.common.evidence_compaction import truncate_message
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -106,6 +106,10 @@ class SessionGoalReason:
|
||||
# A reason is one line of the checklist render; a condition is persisted in
|
||||
# full-ish for resume.
|
||||
MAX_GOAL_REASON_CHARS = 240
|
||||
|
||||
# How many earlier turns a continuation is reminded of. Bounded because the
|
||||
# findings ride in every subsequent prompt; the most recent are what matter.
|
||||
MAX_GOAL_FINDINGS = 4
|
||||
MAX_GOAL_CONDITION_CHARS = 400
|
||||
|
||||
# Session-goal turns a goal may run before the host stops on budget.
|
||||
@@ -136,6 +140,11 @@ class SessionGoal:
|
||||
completed: frozenset[int] = frozenset()
|
||||
# Last host/evaluator reason shown in progress paint and continuation nudges.
|
||||
last_reason: str = ""
|
||||
# What earlier turns established, oldest first. Continuations are fresh
|
||||
# ``chat`` calls and history carries prose only, so without this a later
|
||||
# turn sees only its own tools and reads their absence as an absence
|
||||
# overall — reporting completed work as never done.
|
||||
findings: tuple[str, ...] = ()
|
||||
# Wall-clock start for ``/goal`` duration paint (``time.time()``).
|
||||
started_at: float | None = None
|
||||
# Session token totals when the goal was attached — delta is goal spend.
|
||||
@@ -156,6 +165,13 @@ class SessionGoal:
|
||||
def with_completed(self, completed: frozenset[int]) -> SessionGoal:
|
||||
return replace(self, completed=completed)
|
||||
|
||||
def with_finding(self, finding: str) -> SessionGoal:
|
||||
"""Append one turn's answer to what later turns are told."""
|
||||
text = truncate_message(finding.strip(), MAX_GOAL_REASON_CHARS)
|
||||
if not text:
|
||||
return self
|
||||
return replace(self, findings=(*self.findings, text)[-MAX_GOAL_FINDINGS:])
|
||||
|
||||
def with_reason(self, reason: str) -> SessionGoal:
|
||||
text = truncate_message(reason.strip(), MAX_GOAL_REASON_CHARS)
|
||||
return replace(self, last_reason=text)
|
||||
@@ -259,11 +275,18 @@ def session_goal_from_assistant_handoffs(
|
||||
*,
|
||||
condition: str = "",
|
||||
) -> SessionGoal | None:
|
||||
"""Build a :class:`SessionGoal` from typed :class:`AssistantHandoff` fields."""
|
||||
"""Build a :class:`SessionGoal` from typed :class:`AssistantHandoff` fields.
|
||||
|
||||
``database_query:*`` handoffs never attach a host loop — missing DB
|
||||
connectivity is explained in one reply (connect/setup guidance). A planner
|
||||
that still sets ``session_goal=true`` on those handoffs is ignored here.
|
||||
"""
|
||||
# Reuse the tag body parser by projecting fields to clean content tags —
|
||||
# ontology fields are already validated at decode time.
|
||||
projected: list[str] = []
|
||||
for handoff in handoffs:
|
||||
if handoff_has_tag(handoff.content, "database_query"):
|
||||
continue
|
||||
if handoff.session_goal:
|
||||
projected.append("session_goal:continue")
|
||||
if handoff.session_goal_max_turns is not None:
|
||||
@@ -295,7 +318,9 @@ def attach_session_goal_from_handoffs(
|
||||
detected = None
|
||||
if handoffs:
|
||||
detected = session_goal_from_assistant_handoffs(handoffs, condition=condition)
|
||||
if detected is None:
|
||||
if detected is None and not any(
|
||||
handoff_has_tag(content, "database_query") for content in handoff_contents
|
||||
):
|
||||
detected = session_goal_from_handoffs(handoff_contents, condition=condition)
|
||||
if detected is None:
|
||||
return None
|
||||
|
||||
@@ -16,6 +16,7 @@ from core.agent_harness.session_goal.continuation import continuation_nudge
|
||||
from core.agent_harness.session_goal.evaluate import (
|
||||
default_evaluate_session_goal,
|
||||
session_goal_reply_text,
|
||||
turn_has_session_goal_evidence,
|
||||
)
|
||||
from core.agent_harness.session_goal.goal import (
|
||||
SessionGoal,
|
||||
@@ -148,6 +149,13 @@ def _finish_outer_turn(
|
||||
stored = getattr(session, "session_goal", None)
|
||||
if isinstance(stored, SessionGoal):
|
||||
active = stored
|
||||
# After the reload, never before it: ``evaluate_fn`` re-attaches the goal
|
||||
# and taking the session copy would discard the finding. A continuation is
|
||||
# a fresh chat call and history carries prose only, so this is the only way
|
||||
# a later turn learns what earlier ones established.
|
||||
if turn_has_session_goal_evidence(last):
|
||||
active = active.with_finding(session_goal_reply_text(last))
|
||||
attach_session_goal(session, active)
|
||||
# Evaluate return is authoritative — optional reviewers may keep ACTIVE after
|
||||
# structured evaluate briefly attached ACHIEVED on the session.
|
||||
if active.status != next_status:
|
||||
|
||||
@@ -8,6 +8,7 @@ JSON events path — a defect invisible to admission-level tests.
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
from http import HTTPStatus
|
||||
from typing import Any
|
||||
from urllib.parse import urlencode
|
||||
@@ -30,7 +31,16 @@ from gateway.transports.slack.transport.events_api.server import (
|
||||
from gateway.transports.slack.transport.events_api.signature import expected_signature
|
||||
|
||||
_SECRET = "8f742231b10e8888abcd99yyyzzz85a5"
|
||||
_TIMESTAMP = str(int(__import__("time").time()))
|
||||
|
||||
|
||||
def _now_timestamp() -> str:
|
||||
"""Signed at request time, not import time.
|
||||
|
||||
The signature check enforces a five-minute replay window, so a timestamp
|
||||
captured when the module loads is already stale by the time a full suite
|
||||
reaches these tests — they pass alone and fail in CI.
|
||||
"""
|
||||
return str(int(time.time()))
|
||||
|
||||
|
||||
def _settings() -> SlackGatewaySettings:
|
||||
@@ -54,11 +64,12 @@ def _client(submitted: list[Any]) -> TestClient:
|
||||
|
||||
|
||||
def _headers(body: bytes) -> dict[str, str]:
|
||||
timestamp = _now_timestamp()
|
||||
return {
|
||||
SIGNATURE_HEADER: expected_signature(
|
||||
signing_secret=_SECRET, timestamp=_TIMESTAMP, body=body
|
||||
signing_secret=_SECRET, timestamp=timestamp, body=body
|
||||
),
|
||||
TIMESTAMP_HEADER: _TIMESTAMP,
|
||||
TIMESTAMP_HEADER: timestamp,
|
||||
}
|
||||
|
||||
|
||||
@@ -107,7 +118,7 @@ def test_unsigned_request_is_unauthorized_on_both_routes() -> None:
|
||||
# Arrange.
|
||||
submitted: list[Any] = []
|
||||
client = _client(submitted)
|
||||
bad = {SIGNATURE_HEADER: "v0=deadbeef", TIMESTAMP_HEADER: _TIMESTAMP}
|
||||
bad = {SIGNATURE_HEADER: "v0=deadbeef", TIMESTAMP_HEADER: _now_timestamp()}
|
||||
|
||||
# Act / Assert.
|
||||
for path, body in (
|
||||
|
||||
@@ -44,11 +44,25 @@ class ShellTurnAccounting:
|
||||
def finalize(self, result: TurnResult) -> TurnResult:
|
||||
"""Flush the recorder, persist the turn, and stamp the session intent."""
|
||||
self._flush_prompt_recorder(result)
|
||||
if result.llm_run is not None:
|
||||
if result.llm_run is not None and not self._cli_agent_already_recorded():
|
||||
# ActionRenderObserver may already have recorded this text on the
|
||||
# first non-handoff tool_start (e.g. memory_remember alongside
|
||||
# assistant_handoff). Do not append a duplicate history row.
|
||||
self.session.record("cli_agent", self.text)
|
||||
self.session.last_assistant_intent = result.final_intent
|
||||
return result
|
||||
|
||||
def _cli_agent_already_recorded(self) -> bool:
|
||||
history = getattr(self.session, "history", None) or ()
|
||||
if not history:
|
||||
return False
|
||||
last = history[-1]
|
||||
return (
|
||||
isinstance(last, dict)
|
||||
and last.get("type") == "cli_agent"
|
||||
and last.get("text") == self.text
|
||||
)
|
||||
|
||||
def _record_action_analytics(self, action_result: ToolCallingTurnResult) -> None:
|
||||
from platform.analytics.cli import (
|
||||
capture_repl_execution_policy_decision,
|
||||
|
||||
@@ -144,15 +144,29 @@ def test_cloud_facts_read_deploy_time_env_vars(monkeypatch: pytest.MonkeyPatch)
|
||||
assert meta["cloud_region"] == "europe-west3"
|
||||
|
||||
|
||||
def test_cloud_facts_fall_back_to_aws_region_vars(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""AWS deployments usually carry AWS_REGION/AWS_DEFAULT_REGION already —
|
||||
the same pair the LLM transports read. A region from an AWS var implies
|
||||
provider aws unless CLOUD_PROVIDER says otherwise."""
|
||||
def test_cloud_facts_aws_region_alone_does_not_claim_aws(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Laptop ``.env`` often sets AWS_REGION for the AWS integration — that is
|
||||
not evidence this process is running inside AWS."""
|
||||
for var in ("CLOUD_PROVIDER", "CLOUD_REGION", "AWS_REGION", "AWS_DEFAULT_REGION"):
|
||||
monkeypatch.delenv(var, raising=False)
|
||||
monkeypatch.setenv("AWS_DEFAULT_REGION", "eu-central-1")
|
||||
facts = probes_module.cloud_facts()
|
||||
assert facts == {"cloud_provider": "aws", "cloud_region": "eu-central-1"}
|
||||
assert probes_module.cloud_facts() == {"cloud_provider": "", "cloud_region": ""}
|
||||
|
||||
|
||||
def test_cloud_facts_aws_region_fills_region_when_provider_is_set(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Silos set CLOUD_PROVIDER=aws; AWS_REGION may supply the region."""
|
||||
for var in ("CLOUD_PROVIDER", "CLOUD_REGION", "AWS_REGION", "AWS_DEFAULT_REGION"):
|
||||
monkeypatch.delenv(var, raising=False)
|
||||
monkeypatch.setenv("CLOUD_PROVIDER", "aws")
|
||||
monkeypatch.setenv("AWS_DEFAULT_REGION", "eu-central-1")
|
||||
assert probes_module.cloud_facts() == {
|
||||
"cloud_provider": "aws",
|
||||
"cloud_region": "eu-central-1",
|
||||
}
|
||||
|
||||
|
||||
def test_cloud_facts_empty_when_not_deployed_to_cloud(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
@@ -162,6 +176,43 @@ def test_cloud_facts_empty_when_not_deployed_to_cloud(monkeypatch: pytest.Monkey
|
||||
assert probes_module.cloud_facts() == {"cloud_provider": "", "cloud_region": ""}
|
||||
|
||||
|
||||
def test_host_os_facts_are_always_present() -> None:
|
||||
"""Environment questions need a true host OS, not a cloud vacuum."""
|
||||
# Arrange / Act.
|
||||
facts = probes_module.host_os_facts()
|
||||
meta = build_runtime_metadata()
|
||||
|
||||
# Assert.
|
||||
assert facts["os_family"]
|
||||
assert meta["os_family"] == facts["os_family"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("sys_platform", "expected"),
|
||||
[("darwin", "macOS"), ("linux", "Linux"), ("linux2", "Linux"), ("win32", "Windows")],
|
||||
)
|
||||
def test_host_os_family_reports_the_product_not_the_kernel(
|
||||
monkeypatch: pytest.MonkeyPatch, sys_platform: str, expected: str
|
||||
) -> None:
|
||||
"""``Darwin`` is the kernel; a user on a MacBook is running macOS."""
|
||||
# Arrange.
|
||||
monkeypatch.setattr(probes_module.sys, "platform", sys_platform)
|
||||
|
||||
# Act / Assert.
|
||||
assert probes_module.host_os_facts() == {"os_family": expected}
|
||||
|
||||
|
||||
def test_host_os_facts_omit_a_version() -> None:
|
||||
"""``platform.release()`` is the kernel version, not the OS version.
|
||||
|
||||
On macOS it reports Darwin's number (25.5.0) while the OS is 26.5.2, so
|
||||
publishing it as the OS release states a false fact in the block whose
|
||||
whole purpose is preventing them.
|
||||
"""
|
||||
# Arrange / Act / Assert.
|
||||
assert set(probes_module.host_os_facts()) == {"os_family"}
|
||||
|
||||
|
||||
def test_cloud_facts_never_touch_the_network(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Cloud identity must come from env alone — no instance metadata service
|
||||
(IMDS) call, which would hang or fail off-cloud and is blocked in the sandbox."""
|
||||
|
||||
@@ -4,10 +4,11 @@ Each test times its target N times, uses the median (robust to jitter), and
|
||||
asserts a generous upper bound calibrated at ~10-100x the measured Darwin
|
||||
arm64 baseline so slow CI runners don't flake.
|
||||
|
||||
Baseline (Darwin arm64, Python 3.14.3, n=500):
|
||||
- session dict lookup ~0.0001 ms
|
||||
Baseline (Darwin arm64, Python 3.14.3, n=200; includes host OS + workspace
|
||||
identity probes):
|
||||
- session dict lookup ~0.0001 ms
|
||||
- importlib.metadata.version ~0.56 ms
|
||||
- build_runtime_metadata ~0.58 ms
|
||||
- build_runtime_metadata ~1.7 ms (can approach ~6 ms under xdist load)
|
||||
- build_environment_block ~0.0016 ms
|
||||
"""
|
||||
|
||||
@@ -61,16 +62,17 @@ def test_importlib_version_lookup_stays_under_5ms() -> None:
|
||||
assert median_ms < 5.0, f"regression: {median_ms} ms > 5 ms threshold"
|
||||
|
||||
|
||||
def test_build_runtime_metadata_stays_under_5ms() -> None:
|
||||
def test_build_runtime_metadata_stays_under_15ms() -> None:
|
||||
"""The one-time bootstrap cost at session init / /new / /resume.
|
||||
|
||||
Baseline ~0.58 ms; ~10x buffer = 5 ms. If this breaches, someone added an
|
||||
expensive call to build_runtime_metadata — undermines the "cheap to call
|
||||
per session" invariant.
|
||||
Baseline ~1.7 ms; ~10x buffer = 15 ms (covers xdist/suite-load jitter that
|
||||
previously breached a 5 ms cap around ~6 ms). If this breaches, someone
|
||||
added an expensive call to build_runtime_metadata — undermines the
|
||||
"cheap to call per session" invariant.
|
||||
"""
|
||||
median_ms = _time_median_ms(build_runtime_metadata)
|
||||
print(f"\n build_runtime_metadata: {median_ms:.2f} ms")
|
||||
assert median_ms < 5.0, f"regression: {median_ms} ms > 5 ms threshold"
|
||||
assert median_ms < 15.0, f"regression: {median_ms} ms > 15 ms threshold"
|
||||
|
||||
|
||||
def test_environment_block_render_stays_under_1ms() -> None:
|
||||
@@ -117,5 +119,5 @@ def test_baseline_stability(_i: int) -> None:
|
||||
|
||||
assert dict_ms < 0.01, f"dict {dict_ms} ms"
|
||||
assert imp_ms < 5.0, f"importlib {imp_ms} ms"
|
||||
assert build_ms < 5.0, f"build {build_ms} ms"
|
||||
assert build_ms < 15.0, f"build {build_ms} ms"
|
||||
assert env_ms < 1.0, f"env {env_ms} ms"
|
||||
|
||||
@@ -141,10 +141,32 @@ def _resolve_live_llm_configuration(
|
||||
f" provider={settings.provider!r}, env={spec.api_key_env}"
|
||||
)
|
||||
|
||||
from core.llm.factory import reset_llm_clients
|
||||
from core.llm.factory import LLMRole, get_llm, reset_llm_clients
|
||||
|
||||
monkeypatch.setenv("LLM_PROVIDER", settings.provider)
|
||||
reset_llm_clients()
|
||||
# credential_status can look fine while the provider SDK still refuses to
|
||||
# construct a client (empty/placeholder key, wrong env for the active
|
||||
# provider). Probe once here so live tests skip/fail at setup, not mid-call.
|
||||
try:
|
||||
get_llm(LLMRole.AGENT)
|
||||
except Exception as exc:
|
||||
detail = str(exc).lower()
|
||||
if any(
|
||||
marker in detail
|
||||
for marker in (
|
||||
"missing credentials",
|
||||
"invalid_api_key",
|
||||
"incorrect api key",
|
||||
"authenticationerror",
|
||||
"could not resolve credentials",
|
||||
)
|
||||
):
|
||||
_skip_or_fail_live_llm(
|
||||
"Live LLM turn tests require a constructible provider client:"
|
||||
f" provider={settings.provider!r}. {exc}"
|
||||
)
|
||||
raise
|
||||
yield
|
||||
reset_llm_clients()
|
||||
|
||||
|
||||
@@ -540,6 +540,17 @@ def test_database_query_handoff_guidance_block_matches_prefix() -> None:
|
||||
assert build_handoff_guidance_block(("database_query:mariadb_dashboard",)) == block
|
||||
|
||||
|
||||
def test_action_prompt_routes_mysql_query_to_database_query_handoff_not_setup() -> None:
|
||||
"""Oracle 332: query/read MySQL must not become /integrations setup|verify."""
|
||||
prompt = build_action_system_prompt(_ctx())
|
||||
assert "database_query:<topic>" in prompt
|
||||
assert "database_query:mysql_active_connections" in prompt
|
||||
assert "Do NOT set session_goal=true on database_query handoffs" in prompt
|
||||
assert "Do NOT treat a request to *query/read*" in prompt
|
||||
# Setup still documented for explicit configure requests.
|
||||
assert 'args=["setup", "<service>"]' in prompt
|
||||
|
||||
|
||||
def test_incident_description_handoff_guidance_keeps_user_symptoms() -> None:
|
||||
"""Oracle 325: bare incident handoffs must not drop service/error specifics."""
|
||||
block = build_handoff_guidance_block(("incident_description:checkout_502_rate",))
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -86,12 +86,26 @@ _CREDIT_EXHAUSTED_MARKERS = (
|
||||
"billing_hard_limit_reached",
|
||||
)
|
||||
|
||||
# SDK init / auth failures that mean "no usable key", not a planner assertion.
|
||||
_MISSING_CREDENTIAL_MARKERS = (
|
||||
"missing credentials",
|
||||
"invalid_api_key",
|
||||
"incorrect api key",
|
||||
"authenticationerror",
|
||||
"could not resolve credentials",
|
||||
)
|
||||
|
||||
|
||||
def _provider_credit_exhausted_message(text: str) -> bool:
|
||||
normalized = text.lower()
|
||||
return any(marker in normalized for marker in _CREDIT_EXHAUSTED_MARKERS)
|
||||
|
||||
|
||||
def _missing_llm_credentials_message(text: str) -> bool:
|
||||
normalized = text.lower()
|
||||
return any(marker in normalized for marker in _MISSING_CREDENTIAL_MARKERS)
|
||||
|
||||
|
||||
def _skip_or_fail_provider_credit_exhausted(message: str) -> None:
|
||||
skip_or_fail(
|
||||
"Live LLM provider credit/quota is exhausted; cannot verify live turn "
|
||||
@@ -99,6 +113,13 @@ def _skip_or_fail_provider_credit_exhausted(message: str) -> None:
|
||||
)
|
||||
|
||||
|
||||
def _skip_or_fail_missing_llm_credentials(message: str) -> None:
|
||||
skip_or_fail(
|
||||
"Live LLM credentials are missing or unusable; cannot verify live turn "
|
||||
f"scenario behavior. {message}"
|
||||
)
|
||||
|
||||
|
||||
def _slash_content(command: str, args: list[str]) -> str:
|
||||
return " ".join([command, *args]) if args else command
|
||||
|
||||
@@ -680,11 +701,23 @@ def test_live_action_planning(
|
||||
except LLMCreditExhaustedError as exc:
|
||||
_skip_or_fail_provider_credit_exhausted(str(exc))
|
||||
except RuntimeError as exc:
|
||||
if _provider_credit_exhausted_message(str(exc)):
|
||||
_skip_or_fail_provider_credit_exhausted(str(exc))
|
||||
msg = str(exc)
|
||||
if _provider_credit_exhausted_message(msg):
|
||||
_skip_or_fail_provider_credit_exhausted(msg)
|
||||
if _missing_llm_credentials_message(msg):
|
||||
_skip_or_fail_missing_llm_credentials(msg)
|
||||
raise
|
||||
except AssertionError as exc:
|
||||
failures.append(str(exc))
|
||||
except Exception as exc:
|
||||
# OpenAI/Anthropic SDK init errors (e.g. OpenAIError: Missing
|
||||
# credentials) are not AssertionError/RuntimeError subclasses.
|
||||
msg = str(exc)
|
||||
if _provider_credit_exhausted_message(msg):
|
||||
_skip_or_fail_provider_credit_exhausted(msg)
|
||||
if _missing_llm_credentials_message(msg):
|
||||
_skip_or_fail_missing_llm_credentials(msg)
|
||||
raise
|
||||
else:
|
||||
passed_count += 1
|
||||
|
||||
|
||||
@@ -128,6 +128,23 @@ def test_environment_block_states_cloud_absence_when_not_deployed() -> None:
|
||||
assert "cloud region is" not in block
|
||||
|
||||
|
||||
def test_environment_block_renders_host_os_for_environment_questions() -> None:
|
||||
"""macOS/Linux must be quotable so 'what environment' does not invent AWS."""
|
||||
block = _env_block(
|
||||
{
|
||||
"opensre_version": "0.1",
|
||||
"os_family": "macOS",
|
||||
"cloud_provider": "",
|
||||
"cloud_region": "",
|
||||
}
|
||||
)
|
||||
assert "host operating system is macOS;" in block # no version appended
|
||||
assert "what environment this process is running in" in block
|
||||
assert "never invent AWS" in block
|
||||
assert "`uname`" in block
|
||||
assert "no cloud provider or cloud region was detected" in block
|
||||
|
||||
|
||||
def test_environment_block_does_not_coach_arbitrary_reachability_probing() -> None:
|
||||
"""The always-on prompt must not steer the model toward reachability
|
||||
probing. allow_network has no destination allowlist — coaching sockets
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
"""A session-goal continuation turn must know what earlier turns established.
|
||||
|
||||
Observed live: turn 1 fetched weather and news successfully and printed the
|
||||
figures; a later turn ran one slash command, saw no weather tool among *its*
|
||||
results, and answered "current weather and news retrieval is unavailable in
|
||||
this environment". The work had been done and the agent reported that it had
|
||||
not been.
|
||||
|
||||
Each continuation is a fresh ``chat`` call carrying only the nudge, and
|
||||
``record_conversation_turn`` stores assistant prose — never tool payloads. So
|
||||
nothing hands turn N+1 the evidence turn N gathered.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from core.agent_harness.session.session_core import SessionCore
|
||||
from core.agent_harness.session_goal.goal import SessionGoal
|
||||
from core.agent_harness.session_goal.run_until import run_until_session_goal
|
||||
from core.agent_harness.turns.turn_results import ToolCallingTurnResult, TurnResult
|
||||
|
||||
|
||||
def _turn(text: str, *, executed: int = 0, success: int = 0) -> TurnResult:
|
||||
return TurnResult(
|
||||
final_intent="cli_agent_fallback",
|
||||
action_result=ToolCallingTurnResult(
|
||||
planned_count=executed,
|
||||
executed_count=executed,
|
||||
executed_success_count=success,
|
||||
has_unhandled_clause=False,
|
||||
handled=True,
|
||||
),
|
||||
assistant_response_text=text,
|
||||
llm_run=None,
|
||||
)
|
||||
|
||||
|
||||
def test_continuation_turn_is_told_what_earlier_turns_gathered() -> None:
|
||||
"""The second turn's input must carry the first turn's evidence.
|
||||
|
||||
Without it the second turn can only see its own tools, and an absence there
|
||||
reads as an absence overall — which is how a completed retrieval was
|
||||
reported as unavailable.
|
||||
"""
|
||||
# Arrange — turn 1 gathers real evidence, turn 2 gathers nothing.
|
||||
session = SessionCore()
|
||||
prompts: list[str] = []
|
||||
|
||||
def _chat(message: str) -> TurnResult:
|
||||
prompts.append(message)
|
||||
if len(prompts) == 1:
|
||||
return _turn("Antarctica is -32C and Hawaii is 24C.", executed=2, success=2)
|
||||
return _turn("Checked the Slack integration.", executed=1, success=1)
|
||||
|
||||
# Act — two turns of one goal.
|
||||
run_until_session_goal(
|
||||
_chat,
|
||||
session,
|
||||
"weather brief for Antarctica and Hawaii, then send it",
|
||||
goal=SessionGoal(
|
||||
condition="weather brief for Antarctica and Hawaii, then send it",
|
||||
max_outer_turns=3,
|
||||
# Three items: a two-item checklist hits the same-turn completion
|
||||
# shortcut and the loop never reaches a continuation.
|
||||
checklist=("fetch the weather", "correlate news", "send the brief"),
|
||||
),
|
||||
)
|
||||
|
||||
# Assert — turn 2's prompt must reference what turn 1 established.
|
||||
assert len(prompts) >= 2, "the goal loop did not run a continuation turn"
|
||||
continuation = prompts[1]
|
||||
assert "-32C" in continuation or "Antarctica is" in continuation, (
|
||||
"the continuation turn was not given the evidence turn 1 gathered, so it "
|
||||
"can only reason from its own tools:\n" + continuation
|
||||
)
|
||||
@@ -182,6 +182,37 @@ def test_host_owned_achieved_without_tools_completes() -> None:
|
||||
assert session.session_goal.status == SessionGoalStatus.ACHIEVED
|
||||
|
||||
|
||||
def test_database_query_handoff_does_not_attach_session_goal() -> None:
|
||||
"""Oracle 332: planner session_goal on a DB query must not start the host loop."""
|
||||
from core.agent_harness.session_goal.goal import attach_session_goal_from_handoffs
|
||||
from core.agent_harness.turns.assistant_handoff import AssistantHandoff
|
||||
|
||||
session = SessionCore()
|
||||
handoff = AssistantHandoff.from_tool_input(
|
||||
{
|
||||
"content": "database_query:mysql_active_connections",
|
||||
"session_goal": True,
|
||||
}
|
||||
)
|
||||
attached = attach_session_goal_from_handoffs(
|
||||
session,
|
||||
handoff.to_handoff_contents(),
|
||||
condition="Use the MySQL tool to query active connections.",
|
||||
handoffs=(handoff,),
|
||||
)
|
||||
assert attached is None
|
||||
assert getattr(session, "session_goal", None) is None
|
||||
|
||||
# Legacy tags alone (no typed handoffs) also stay one-shot.
|
||||
session2 = SessionCore()
|
||||
attached_legacy = attach_session_goal_from_handoffs(
|
||||
session2,
|
||||
("database_query:mysql_active_connections", "session_goal:continue"),
|
||||
condition="query mysql",
|
||||
)
|
||||
assert attached_legacy is None
|
||||
|
||||
|
||||
def test_handoff_does_not_replace_active_host_owned_goal() -> None:
|
||||
from core.agent_harness.session_goal.goal import attach_session_goal_from_handoffs
|
||||
|
||||
|
||||
@@ -108,3 +108,21 @@ def test_session_goal_content_tag_attaches_when_schema_omitted() -> None:
|
||||
def test_session_goal_achieved_content_tag_is_not_attach() -> None:
|
||||
handoff = AssistantHandoff.from_tool_input({"content": "Done.\nsession_goal:achieved"})
|
||||
assert handoff.session_goal is not True
|
||||
|
||||
|
||||
def test_database_query_handoff_ignores_session_goal_attach() -> None:
|
||||
"""Oracle 332: DB query/connect guidance is one-shot — no host goal loop."""
|
||||
handoff = AssistantHandoff.from_tool_input(
|
||||
{
|
||||
"content": "database_query:mysql_active_connections",
|
||||
"session_goal": True,
|
||||
}
|
||||
)
|
||||
assert handoff.session_goal is True # decode keeps the flag
|
||||
assert (
|
||||
session_goal_from_assistant_handoffs(
|
||||
(handoff,),
|
||||
condition="Use the MySQL tool to query active connections.",
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
@@ -69,6 +69,22 @@ def test_finalize_prefers_conversational_run_over_pending() -> None:
|
||||
assert session.terminal.pop_pending_turn_llm() is None
|
||||
|
||||
|
||||
def test_finalize_does_not_duplicate_early_cli_agent_history() -> None:
|
||||
"""memory_remember + handoff: ActionRenderObserver records once; finalize must not again."""
|
||||
session = Session()
|
||||
prompt = "Use the MySQL tool to query active connections."
|
||||
session.record("cli_agent", prompt)
|
||||
conversational = LlmRunInfo(model="fresh")
|
||||
recorder = _FakeRecorder()
|
||||
accounting = ShellTurnAccounting(session=session, text=prompt, recorder=recorder) # type: ignore[arg-type]
|
||||
|
||||
accounting.finalize(_result(llm_run=conversational))
|
||||
|
||||
cli_rows = [row for row in session.history if row.get("type") == "cli_agent"]
|
||||
assert len(cli_rows) == 1
|
||||
assert cli_rows[0]["text"] == prompt
|
||||
|
||||
|
||||
def test_finalize_sets_structured_error_from_pending_turn_error() -> None:
|
||||
session = Session()
|
||||
session.terminal.set_pending_turn_error("config", "ANTHROPIC_API_KEY not set")
|
||||
|
||||
Reference in New Issue
Block a user