fix(agent): correct the runtime facts the agent reports — cloud, OS, cross-turn evidence, connected integrations (#4941)

This commit is contained in:
Yauhen Bichel
2026-08-12 11:41:40 +01:00
committed by GitHub
parent 8e9dc3b64d
commit a6164a3b1f
23 changed files with 483 additions and 58 deletions
+4 -1
View File
@@ -738,7 +738,10 @@ OPENSRE_MASK_EXTRA_REGEX=
# Cloud identity for the agent's runtime facts, injected at deploy time.
# Read by session metadata; never fetched from the instance metadata service (IMDS).
# CLOUD_REGION falls back to AWS_REGION / AWS_DEFAULT_REGION when unset.
# Set CLOUD_PROVIDER on silos (e.g. aws). AWS_REGION alone does NOT mean this
# process is running in AWS — laptop .env often sets AWS_REGION for tools.
# When CLOUD_PROVIDER is set, CLOUD_REGION may fall back to AWS_REGION /
# AWS_DEFAULT_REGION.
CLOUD_PROVIDER=
CLOUD_REGION=
+6 -3
View File
@@ -21,6 +21,7 @@ from config.runtime_metadata.probes import (
capability_warning_facts,
cloud_facts,
disk_memory_facts,
host_os_facts,
installed_tools,
kubeconfig_path,
local_tz_name,
@@ -44,6 +45,7 @@ def build_runtime_metadata() -> dict[str, Any]:
- ``opensre_build`` — ``""`` in released wheels; ``dev, v0.1.YYYY.M.D @ SHA``
in a git checkout so the LLM can quote the exact build in local dev.
- ``runtime_env`` — ``OPENSRE_ENV`` env var, else the app environment name.
- ``os_family`` — host OS name (macOS/Linux/Windows).
- ``tz_name`` — local timezone name (rarely changes mid-session).
- ``python_version`` — interpreter version from :data:`sys.version_info`.
- ``pid`` / ``ppid`` — this process and its parent from :mod:`os`.
@@ -52,9 +54,9 @@ def build_runtime_metadata() -> dict[str, Any]:
- ``hostname`` — from ``/etc/hostname`` (the pod name in Kubernetes) or
:func:`socket.gethostname`, never the ``hostname`` binary.
- ``scratchpad_dir`` — the temp directory scripts may write to.
- ``cloud_provider`` / ``cloud_region`` — deploy-time env vars
(``CLOUD_PROVIDER`` / ``CLOUD_REGION``, AWS var fallback), never the
instance metadata service (IMDS).
- ``cloud_provider`` / ``cloud_region`` — deploy-time ``CLOUD_PROVIDER`` /
``CLOUD_REGION`` only (``AWS_REGION`` alone does not imply AWS), never
the instance metadata service (IMDS).
The exact key set is :data:`STATIC_FACT_KEYS` (contract-tested). Live
values that must NOT be cached (current time, uptime, disk, memory) come
@@ -66,6 +68,7 @@ def build_runtime_metadata() -> dict[str, Any]:
"opensre_version": get_opensre_version(),
"opensre_build": detect_build_info(),
"runtime_env": env_override or get_environment().value,
**host_os_facts(),
"tz_name": local_tz_name(),
"python_version": python_version_string(),
"pid": os.getpid(),
+2
View File
@@ -17,6 +17,7 @@ STATIC_FACT_KEYS: tuple[str, ...] = (
"opensre_version",
"opensre_build",
"runtime_env",
"os_family",
"tz_name",
"python_version",
"pid",
@@ -58,6 +59,7 @@ BLOCKED_INTROSPECTION_COMMANDS: tuple[str, ...] = (
"date",
"uptime",
"hostname",
"uname",
"ls",
"df",
"free",
+47 -12
View File
@@ -43,6 +43,19 @@ _LOCALTIME_LINK = Path("/etc/localtime")
_HOSTNAME_FILE = Path("/etc/hostname")
# Values of CLOUD_PROVIDER for which AWS_REGION / AWS_DEFAULT_REGION describe
# the same deployment. Any other provider keeps its own region source.
_AWS_PROVIDER_NAMES: Final[frozenset[str]] = frozenset({"aws", "amazon"})
# ``sys.platform`` prefixes mapped to the OS name a user would recognise.
# Prefix-matched: Linux reports linux/linux2 and BSDs carry a version suffix.
_OS_FAMILY_BY_PLATFORM_PREFIX: Final[tuple[tuple[str, str], ...]] = (
("darwin", "macOS"),
("linux", "Linux"),
("win", "Windows"),
("freebsd", "FreeBSD"),
)
def local_tz_name() -> str:
"""Best-effort local timezone name — IANA (``Europe/Berlin``) when possible.
@@ -126,25 +139,46 @@ def disk_memory_facts() -> dict[str, Any]:
}
def host_os_facts() -> dict[str, str]:
"""Host OS family — ``macOS``, ``Linux``, ``Windows``.
Always present so "what environment are you running in?" has a true local
answer instead of a vacuum the model fills with a cloud guess.
No version: ``platform.release()`` is the *kernel* release — on macOS the
Darwin number (25.5.0), not the macOS version (26.5.2) — so publishing it
as the OS release states a false fact in the block that exists to prevent
them.
``sys.platform`` rather than ``platform.system()``: this repo ships its own
``platform`` package (see ``config/secrets/os_keyring.py``), so the stdlib
name only resolves through a shim.
"""
identifier = sys.platform
for prefix, family in _OS_FAMILY_BY_PLATFORM_PREFIX:
if identifier.startswith(prefix):
return {"os_family": family}
return {"os_family": identifier or "unknown"}
def cloud_facts() -> dict[str, str]:
"""Cloud provider/region from deploy-time env vars — no metadata endpoint.
``CLOUD_PROVIDER`` / ``CLOUD_REGION`` are the canonical injection points
(set at deploy time). Region falls back to ``AWS_REGION`` /
``AWS_DEFAULT_REGION`` — the same pair the LLM transports already read —
and when the region came from an AWS var the provider defaults to ``aws``.
Never calls the instance metadata service (IMDS); the sandbox blocks
network anyway.
(set at deploy time). ``AWS_REGION`` / ``AWS_DEFAULT_REGION`` alone must
**not** claim this process is running in AWS — those vars are routine on
developer laptops (``.env.example`` ships ``AWS_REGION=us-east-1`` for the
AWS integration). They may fill in the region only when the provider is
itself AWS: an AWS region under ``CLOUD_PROVIDER=gcp`` would be a wrong
location stated as authoritative runtime identity. Never calls the instance
metadata service (IMDS).
"""
provider = (os.environ.get("CLOUD_PROVIDER") or "").strip()
region = (os.environ.get("CLOUD_REGION") or "").strip()
aws_region = (
os.environ.get("AWS_REGION") or os.environ.get("AWS_DEFAULT_REGION") or ""
).strip()
if not region and aws_region:
region = aws_region
if not provider:
provider = "aws"
if not region and provider.lower() in _AWS_PROVIDER_NAMES:
region = (
os.environ.get("AWS_REGION") or os.environ.get("AWS_DEFAULT_REGION") or ""
).strip()
return {"cloud_provider": provider, "cloud_region": region}
@@ -282,6 +316,7 @@ __all__ = [
"capability_warning_facts",
"cloud_facts",
"disk_memory_facts",
"host_os_facts",
"installed_tools",
"kubeconfig_path",
"local_tz_name",
+17 -2
View File
@@ -338,6 +338,10 @@ slash_invoke:
This should run the wizard for them; do not hand off just to tell the user to
type the command. If no service/server is named, use assistant_handoff to ask
which one.
Do NOT treat a request to *query/read* a named database tool (active
connections, status, dashboard) as setup/enable — that is assistant_handoff
with ``database_query:<topic>``, even when a first-party MySQL/MariaDB
integration exists.
Other tools:
- llm_set_provider — switch provider ONLY when the user names an EXACT provider
target (e.g. "switch to anthropic", "use openai", "set provider to ollama").
@@ -532,6 +536,13 @@ service. Requests to list/query Datadog monitors, Grafana logs, Sentry issues,
PostHog events, traces, sessions, or similar integration data are data lookups:
emit assistant_handoff so the conversational gather loop can use the integration
tools. Do not substitute `/integrations show <service>` for those records.
It also does NOT apply to querying or reading data from a named database tool
(MySQL, MariaDB, Postgres, etc.) — including prompts that cite a tool id such as
``mysql-…`` and ask for active connections, status, or a dashboard read. Those
are ``database_query:<topic>`` handoffs (see below). Do NOT emit
slash_invoke ``/integrations verify|setup <service>`` or ``/mcp connect`` as a
stand-in for answering the query; the assistant explains connect/setup after
the handoff. Do NOT set session_goal=true on those handoffs.
A vendor's own teammate-messaging actions (channel history, thread reads,
workspace search, roster, join, reply, task capture, etc.) are NOT this
category — use that vendor's action tools instead (see its action-prompt
@@ -576,7 +587,10 @@ call rather than relying on plain-text output. Use concise structured content ta
when the topic is known — for example docs:datadog_setup, chat:greeting,
provider:local_llama_connect for vague local-model connection requests, or
database_query:<topic> when the user asks to query/read a named database tool
(MySQL, MariaDB, etc.) that is not a first-party setup-wizard target.
(MySQL, MariaDB, Postgres, etc.) — including first-party integrations and MCP
tool ids. Emit ONLY that assistant_handoff (no slash_invoke setup/verify
alongside it). Example: "Use the MySQL tool (ID: mysql-…) to query active
connections" → assistant_handoff(content="database_query:mysql_active_connections").
Also set these structured assistant_handoff fields when they apply (the harness
keys policy off them; it does not scan user prose for intent). Prefer the
schema fields over burying tags in content prose:
@@ -595,7 +609,8 @@ schema fields over burying tags in content prose:
work), and for metric_read count questions. The host session-goal loop keys
off this boolean; omitting it drops continuation (except metric_read, which
the host treats as attach). Prefer session_goal_items=["", …] for checklist
criteria.
criteria. Do NOT set session_goal=true on database_query handoffs — missing
DB connectivity is explained in one reply, not a multi-turn goal loop.
- session_goal_max_turns=<n> — optional session-goal turn cap for that goal.
- session_goal_items=["", …] — checklist success criteria (one string per
item, in order). The host tracks completion via session_goal:done=<index>
+16 -2
View File
@@ -96,10 +96,24 @@ def _assistant_context_blocks(
def _build_integration_guard(ctx: TurnSnapshot) -> str:
"""Render the no-integrations guidance block from the turn snapshot."""
if not (ctx.configured_integrations_known and not ctx.configured_integrations):
"""Render what is connected, and the no-integrations guidance when empty.
Naming the connected set lets "X is not connected" be answered with what
*is* — the difference between an assertion and a checked result. The data
already reaches the gather prompt; the answer path was told only when the
set was empty, so a reply could not say what it had looked at.
"""
if not ctx.configured_integrations_known:
return ""
if ctx.configured_integrations:
connected = ", ".join(ctx.configured_integrations)
return (
f"Integrations connected in this session: {connected}. When the user "
"asks about a data source that is not in that list, say which ones "
"are connected rather than only that theirs is missing.\n\n"
)
return (
"No integrations are configured in this session. You may still help the user "
"configure one: explain `/integrations setup <service>` for integrations or "
@@ -23,11 +23,15 @@ _BLOCKED_COMMANDS = ", ".join(f"`{command}`" for command in BLOCKED_INTROSPECTIO
_STATIC_GUIDANCE = (
". When the user asks which OpenSRE version is running, reply with the "
"full version string above verbatim — including any parenthetical suffix. "
"When the user asks for the local timezone name, Python version, process "
"When the user asks what environment this process is running in, or for "
"the host operating system, local timezone name, Python version, process "
"id, parent process id, host/pod name, cloud provider or region, "
"kubeconfig path, or which tools are installed, answer from the strings "
"above directly, WITHOUT any tool call — these facts are authoritative "
"and re-reading them through the sandbox wastes a round-trip. Never run "
"and re-reading them through the sandbox wastes a round-trip. Quote the "
"host operating system for environment questions; when no cloud provider "
"was detected, say so — never invent AWS, GCP, Azure, or a region. Never "
"run "
f"{_BLOCKED_COMMANDS}, and never probe cloud instance metadata over the "
"network. To list files in the scratchpad or another directory, "
"use the Python execution sandbox with `pathlib.Path(...).iterdir()` — "
@@ -126,6 +130,14 @@ def _tools_line(runtime: Mapping[str, Any]) -> str | None:
return f"installed tools on PATH are {', '.join(present)}"
def _host_os_line(runtime: Mapping[str, Any]) -> str | None:
"""Host OS — always stated when the key was probed."""
if "os_family" not in runtime:
return None
family = _clean_str(runtime, "os_family")
return f"host operating system is {family}" if family else None
def _cloud_line(runtime: Mapping[str, Any]) -> str | None:
"""Cloud identity, or an explicit statement that none was detected.
@@ -187,6 +199,7 @@ def _capability_warnings_line(runtime: Mapping[str, Any]) -> str | None:
_STATIC_FACT_PRODUCERS: tuple[FactProducer, ...] = (
_version_line,
_str_fact("runtime_env", "runtime environment is {}"),
_host_os_line,
_str_fact("hostname", "host name is {}"),
_str_fact("tz_name", "local timezone is {}"),
_str_fact("python_version", "Python interpreter version is {}"),
@@ -17,6 +17,13 @@ def continuation_nudge(goal: SessionGoal) -> str:
"""User-visible follow-up message for the next session-goal turn."""
reason = goal.last_reason.strip() or derive_session_goal_reason(goal)
reason_block = f"Last progress: {reason}\n\n"
if goal.findings:
established = "\n".join(f" - {item}" for item in goal.findings)
reason_block += (
"Already established in earlier turns of this goal — treat these as "
"done and do not report them as unavailable:\n"
f"{established}\n\n"
)
unfinished = goal.unfinished_items
if unfinished:
pending = "\n".join(f" - [{index}] {item}" for index, item in unfinished)
+28 -3
View File
@@ -29,7 +29,7 @@ from collections.abc import Sequence
from dataclasses import dataclass, replace
from typing import TYPE_CHECKING, Any
from core.agent_harness.turns.handoff_tag_parse import find_tag_suffix
from core.agent_harness.turns.handoff_tag_parse import find_tag_suffix, handoff_has_tag
from platform.common.evidence_compaction import truncate_message
if TYPE_CHECKING:
@@ -106,6 +106,10 @@ class SessionGoalReason:
# A reason is one line of the checklist render; a condition is persisted in
# full-ish for resume.
MAX_GOAL_REASON_CHARS = 240
# How many earlier turns a continuation is reminded of. Bounded because the
# findings ride in every subsequent prompt; the most recent are what matter.
MAX_GOAL_FINDINGS = 4
MAX_GOAL_CONDITION_CHARS = 400
# Session-goal turns a goal may run before the host stops on budget.
@@ -136,6 +140,11 @@ class SessionGoal:
completed: frozenset[int] = frozenset()
# Last host/evaluator reason shown in progress paint and continuation nudges.
last_reason: str = ""
# What earlier turns established, oldest first. Continuations are fresh
# ``chat`` calls and history carries prose only, so without this a later
# turn sees only its own tools and reads their absence as an absence
# overall — reporting completed work as never done.
findings: tuple[str, ...] = ()
# Wall-clock start for ``/goal`` duration paint (``time.time()``).
started_at: float | None = None
# Session token totals when the goal was attached — delta is goal spend.
@@ -156,6 +165,13 @@ class SessionGoal:
def with_completed(self, completed: frozenset[int]) -> SessionGoal:
return replace(self, completed=completed)
def with_finding(self, finding: str) -> SessionGoal:
"""Append one turn's answer to what later turns are told."""
text = truncate_message(finding.strip(), MAX_GOAL_REASON_CHARS)
if not text:
return self
return replace(self, findings=(*self.findings, text)[-MAX_GOAL_FINDINGS:])
def with_reason(self, reason: str) -> SessionGoal:
text = truncate_message(reason.strip(), MAX_GOAL_REASON_CHARS)
return replace(self, last_reason=text)
@@ -259,11 +275,18 @@ def session_goal_from_assistant_handoffs(
*,
condition: str = "",
) -> SessionGoal | None:
"""Build a :class:`SessionGoal` from typed :class:`AssistantHandoff` fields."""
"""Build a :class:`SessionGoal` from typed :class:`AssistantHandoff` fields.
``database_query:*`` handoffs never attach a host loop — missing DB
connectivity is explained in one reply (connect/setup guidance). A planner
that still sets ``session_goal=true`` on those handoffs is ignored here.
"""
# Reuse the tag body parser by projecting fields to clean content tags —
# ontology fields are already validated at decode time.
projected: list[str] = []
for handoff in handoffs:
if handoff_has_tag(handoff.content, "database_query"):
continue
if handoff.session_goal:
projected.append("session_goal:continue")
if handoff.session_goal_max_turns is not None:
@@ -295,7 +318,9 @@ def attach_session_goal_from_handoffs(
detected = None
if handoffs:
detected = session_goal_from_assistant_handoffs(handoffs, condition=condition)
if detected is None:
if detected is None and not any(
handoff_has_tag(content, "database_query") for content in handoff_contents
):
detected = session_goal_from_handoffs(handoff_contents, condition=condition)
if detected is None:
return None
@@ -16,6 +16,7 @@ from core.agent_harness.session_goal.continuation import continuation_nudge
from core.agent_harness.session_goal.evaluate import (
default_evaluate_session_goal,
session_goal_reply_text,
turn_has_session_goal_evidence,
)
from core.agent_harness.session_goal.goal import (
SessionGoal,
@@ -148,6 +149,13 @@ def _finish_outer_turn(
stored = getattr(session, "session_goal", None)
if isinstance(stored, SessionGoal):
active = stored
# After the reload, never before it: ``evaluate_fn`` re-attaches the goal
# and taking the session copy would discard the finding. A continuation is
# a fresh chat call and history carries prose only, so this is the only way
# a later turn learns what earlier ones established.
if turn_has_session_goal_evidence(last):
active = active.with_finding(session_goal_reply_text(last))
attach_session_goal(session, active)
# Evaluate return is authoritative — optional reviewers may keep ACTIVE after
# structured evaluate briefly attached ACHIEVED on the session.
if active.status != next_status:
+15 -4
View File
@@ -8,6 +8,7 @@ JSON events path — a defect invisible to admission-level tests.
from __future__ import annotations
import json
import time
from http import HTTPStatus
from typing import Any
from urllib.parse import urlencode
@@ -30,7 +31,16 @@ from gateway.transports.slack.transport.events_api.server import (
from gateway.transports.slack.transport.events_api.signature import expected_signature
_SECRET = "8f742231b10e8888abcd99yyyzzz85a5"
_TIMESTAMP = str(int(__import__("time").time()))
def _now_timestamp() -> str:
"""Signed at request time, not import time.
The signature check enforces a five-minute replay window, so a timestamp
captured when the module loads is already stale by the time a full suite
reaches these tests — they pass alone and fail in CI.
"""
return str(int(time.time()))
def _settings() -> SlackGatewaySettings:
@@ -54,11 +64,12 @@ def _client(submitted: list[Any]) -> TestClient:
def _headers(body: bytes) -> dict[str, str]:
timestamp = _now_timestamp()
return {
SIGNATURE_HEADER: expected_signature(
signing_secret=_SECRET, timestamp=_TIMESTAMP, body=body
signing_secret=_SECRET, timestamp=timestamp, body=body
),
TIMESTAMP_HEADER: _TIMESTAMP,
TIMESTAMP_HEADER: timestamp,
}
@@ -107,7 +118,7 @@ def test_unsigned_request_is_unauthorized_on_both_routes() -> None:
# Arrange.
submitted: list[Any] = []
client = _client(submitted)
bad = {SIGNATURE_HEADER: "v0=deadbeef", TIMESTAMP_HEADER: _TIMESTAMP}
bad = {SIGNATURE_HEADER: "v0=deadbeef", TIMESTAMP_HEADER: _now_timestamp()}
# Act / Assert.
for path, body in (
@@ -44,11 +44,25 @@ class ShellTurnAccounting:
def finalize(self, result: TurnResult) -> TurnResult:
"""Flush the recorder, persist the turn, and stamp the session intent."""
self._flush_prompt_recorder(result)
if result.llm_run is not None:
if result.llm_run is not None and not self._cli_agent_already_recorded():
# ActionRenderObserver may already have recorded this text on the
# first non-handoff tool_start (e.g. memory_remember alongside
# assistant_handoff). Do not append a duplicate history row.
self.session.record("cli_agent", self.text)
self.session.last_assistant_intent = result.final_intent
return result
def _cli_agent_already_recorded(self) -> bool:
history = getattr(self.session, "history", None) or ()
if not history:
return False
last = history[-1]
return (
isinstance(last, dict)
and last.get("type") == "cli_agent"
and last.get("text") == self.text
)
def _record_action_analytics(self, action_result: ToolCallingTurnResult) -> None:
from platform.analytics.cli import (
capture_repl_execution_policy_decision,
+57 -6
View File
@@ -144,15 +144,29 @@ def test_cloud_facts_read_deploy_time_env_vars(monkeypatch: pytest.MonkeyPatch)
assert meta["cloud_region"] == "europe-west3"
def test_cloud_facts_fall_back_to_aws_region_vars(monkeypatch: pytest.MonkeyPatch) -> None:
"""AWS deployments usually carry AWS_REGION/AWS_DEFAULT_REGION already —
the same pair the LLM transports read. A region from an AWS var implies
provider aws unless CLOUD_PROVIDER says otherwise."""
def test_cloud_facts_aws_region_alone_does_not_claim_aws(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Laptop ``.env`` often sets AWS_REGION for the AWS integration — that is
not evidence this process is running inside AWS."""
for var in ("CLOUD_PROVIDER", "CLOUD_REGION", "AWS_REGION", "AWS_DEFAULT_REGION"):
monkeypatch.delenv(var, raising=False)
monkeypatch.setenv("AWS_DEFAULT_REGION", "eu-central-1")
facts = probes_module.cloud_facts()
assert facts == {"cloud_provider": "aws", "cloud_region": "eu-central-1"}
assert probes_module.cloud_facts() == {"cloud_provider": "", "cloud_region": ""}
def test_cloud_facts_aws_region_fills_region_when_provider_is_set(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Silos set CLOUD_PROVIDER=aws; AWS_REGION may supply the region."""
for var in ("CLOUD_PROVIDER", "CLOUD_REGION", "AWS_REGION", "AWS_DEFAULT_REGION"):
monkeypatch.delenv(var, raising=False)
monkeypatch.setenv("CLOUD_PROVIDER", "aws")
monkeypatch.setenv("AWS_DEFAULT_REGION", "eu-central-1")
assert probes_module.cloud_facts() == {
"cloud_provider": "aws",
"cloud_region": "eu-central-1",
}
def test_cloud_facts_empty_when_not_deployed_to_cloud(monkeypatch: pytest.MonkeyPatch) -> None:
@@ -162,6 +176,43 @@ def test_cloud_facts_empty_when_not_deployed_to_cloud(monkeypatch: pytest.Monkey
assert probes_module.cloud_facts() == {"cloud_provider": "", "cloud_region": ""}
def test_host_os_facts_are_always_present() -> None:
"""Environment questions need a true host OS, not a cloud vacuum."""
# Arrange / Act.
facts = probes_module.host_os_facts()
meta = build_runtime_metadata()
# Assert.
assert facts["os_family"]
assert meta["os_family"] == facts["os_family"]
@pytest.mark.parametrize(
("sys_platform", "expected"),
[("darwin", "macOS"), ("linux", "Linux"), ("linux2", "Linux"), ("win32", "Windows")],
)
def test_host_os_family_reports_the_product_not_the_kernel(
monkeypatch: pytest.MonkeyPatch, sys_platform: str, expected: str
) -> None:
"""``Darwin`` is the kernel; a user on a MacBook is running macOS."""
# Arrange.
monkeypatch.setattr(probes_module.sys, "platform", sys_platform)
# Act / Assert.
assert probes_module.host_os_facts() == {"os_family": expected}
def test_host_os_facts_omit_a_version() -> None:
"""``platform.release()`` is the kernel version, not the OS version.
On macOS it reports Darwin's number (25.5.0) while the OS is 26.5.2, so
publishing it as the OS release states a false fact in the block whose
whole purpose is preventing them.
"""
# Arrange / Act / Assert.
assert set(probes_module.host_os_facts()) == {"os_family"}
def test_cloud_facts_never_touch_the_network(monkeypatch: pytest.MonkeyPatch) -> None:
"""Cloud identity must come from env alone — no instance metadata service
(IMDS) call, which would hang or fail off-cloud and is blocked in the sandbox."""
+11 -9
View File
@@ -4,10 +4,11 @@ Each test times its target N times, uses the median (robust to jitter), and
asserts a generous upper bound calibrated at ~10-100x the measured Darwin
arm64 baseline so slow CI runners don't flake.
Baseline (Darwin arm64, Python 3.14.3, n=500):
- session dict lookup ~0.0001 ms
Baseline (Darwin arm64, Python 3.14.3, n=200; includes host OS + workspace
identity probes):
- session dict lookup ~0.0001 ms
- importlib.metadata.version ~0.56 ms
- build_runtime_metadata ~0.58 ms
- build_runtime_metadata ~1.7 ms (can approach ~6 ms under xdist load)
- build_environment_block ~0.0016 ms
"""
@@ -61,16 +62,17 @@ def test_importlib_version_lookup_stays_under_5ms() -> None:
assert median_ms < 5.0, f"regression: {median_ms} ms > 5 ms threshold"
def test_build_runtime_metadata_stays_under_5ms() -> None:
def test_build_runtime_metadata_stays_under_15ms() -> None:
"""The one-time bootstrap cost at session init / /new / /resume.
Baseline ~0.58 ms; ~10x buffer = 5 ms. If this breaches, someone added an
expensive call to build_runtime_metadata — undermines the "cheap to call
per session" invariant.
Baseline ~1.7 ms; ~10x buffer = 15 ms (covers xdist/suite-load jitter that
previously breached a 5 ms cap around ~6 ms). If this breaches, someone
added an expensive call to build_runtime_metadata — undermines the
"cheap to call per session" invariant.
"""
median_ms = _time_median_ms(build_runtime_metadata)
print(f"\n build_runtime_metadata: {median_ms:.2f} ms")
assert median_ms < 5.0, f"regression: {median_ms} ms > 5 ms threshold"
assert median_ms < 15.0, f"regression: {median_ms} ms > 15 ms threshold"
def test_environment_block_render_stays_under_1ms() -> None:
@@ -117,5 +119,5 @@ def test_baseline_stability(_i: int) -> None:
assert dict_ms < 0.01, f"dict {dict_ms} ms"
assert imp_ms < 5.0, f"importlib {imp_ms} ms"
assert build_ms < 5.0, f"build {build_ms} ms"
assert build_ms < 15.0, f"build {build_ms} ms"
assert env_ms < 1.0, f"env {env_ms} ms"
+23 -1
View File
@@ -141,10 +141,32 @@ def _resolve_live_llm_configuration(
f" provider={settings.provider!r}, env={spec.api_key_env}"
)
from core.llm.factory import reset_llm_clients
from core.llm.factory import LLMRole, get_llm, reset_llm_clients
monkeypatch.setenv("LLM_PROVIDER", settings.provider)
reset_llm_clients()
# credential_status can look fine while the provider SDK still refuses to
# construct a client (empty/placeholder key, wrong env for the active
# provider). Probe once here so live tests skip/fail at setup, not mid-call.
try:
get_llm(LLMRole.AGENT)
except Exception as exc:
detail = str(exc).lower()
if any(
marker in detail
for marker in (
"missing credentials",
"invalid_api_key",
"incorrect api key",
"authenticationerror",
"could not resolve credentials",
)
):
_skip_or_fail_live_llm(
"Live LLM turn tests require a constructible provider client:"
f" provider={settings.provider!r}. {exc}"
)
raise
yield
reset_llm_clients()
@@ -540,6 +540,17 @@ def test_database_query_handoff_guidance_block_matches_prefix() -> None:
assert build_handoff_guidance_block(("database_query:mariadb_dashboard",)) == block
def test_action_prompt_routes_mysql_query_to_database_query_handoff_not_setup() -> None:
"""Oracle 332: query/read MySQL must not become /integrations setup|verify."""
prompt = build_action_system_prompt(_ctx())
assert "database_query:<topic>" in prompt
assert "database_query:mysql_active_connections" in prompt
assert "Do NOT set session_goal=true on database_query handoffs" in prompt
assert "Do NOT treat a request to *query/read*" in prompt
# Setup still documented for explicit configure requests.
assert 'args=["setup", "<service>"]' in prompt
def test_incident_description_handoff_guidance_keeps_user_symptoms() -> None:
"""Oracle 325: bare incident handoffs must not drop service/error specifics."""
block = build_handoff_guidance_block(("incident_description:checkout_502_rate",))
File diff suppressed because one or more lines are too long
+35 -2
View File
@@ -86,12 +86,26 @@ _CREDIT_EXHAUSTED_MARKERS = (
"billing_hard_limit_reached",
)
# SDK init / auth failures that mean "no usable key", not a planner assertion.
_MISSING_CREDENTIAL_MARKERS = (
"missing credentials",
"invalid_api_key",
"incorrect api key",
"authenticationerror",
"could not resolve credentials",
)
def _provider_credit_exhausted_message(text: str) -> bool:
normalized = text.lower()
return any(marker in normalized for marker in _CREDIT_EXHAUSTED_MARKERS)
def _missing_llm_credentials_message(text: str) -> bool:
normalized = text.lower()
return any(marker in normalized for marker in _MISSING_CREDENTIAL_MARKERS)
def _skip_or_fail_provider_credit_exhausted(message: str) -> None:
skip_or_fail(
"Live LLM provider credit/quota is exhausted; cannot verify live turn "
@@ -99,6 +113,13 @@ def _skip_or_fail_provider_credit_exhausted(message: str) -> None:
)
def _skip_or_fail_missing_llm_credentials(message: str) -> None:
skip_or_fail(
"Live LLM credentials are missing or unusable; cannot verify live turn "
f"scenario behavior. {message}"
)
def _slash_content(command: str, args: list[str]) -> str:
return " ".join([command, *args]) if args else command
@@ -680,11 +701,23 @@ def test_live_action_planning(
except LLMCreditExhaustedError as exc:
_skip_or_fail_provider_credit_exhausted(str(exc))
except RuntimeError as exc:
if _provider_credit_exhausted_message(str(exc)):
_skip_or_fail_provider_credit_exhausted(str(exc))
msg = str(exc)
if _provider_credit_exhausted_message(msg):
_skip_or_fail_provider_credit_exhausted(msg)
if _missing_llm_credentials_message(msg):
_skip_or_fail_missing_llm_credentials(msg)
raise
except AssertionError as exc:
failures.append(str(exc))
except Exception as exc:
# OpenAI/Anthropic SDK init errors (e.g. OpenAIError: Missing
# credentials) are not AssertionError/RuntimeError subclasses.
msg = str(exc)
if _provider_credit_exhausted_message(msg):
_skip_or_fail_provider_credit_exhausted(msg)
if _missing_llm_credentials_message(msg):
_skip_or_fail_missing_llm_credentials(msg)
raise
else:
passed_count += 1
@@ -128,6 +128,23 @@ def test_environment_block_states_cloud_absence_when_not_deployed() -> None:
assert "cloud region is" not in block
def test_environment_block_renders_host_os_for_environment_questions() -> None:
"""macOS/Linux must be quotable so 'what environment' does not invent AWS."""
block = _env_block(
{
"opensre_version": "0.1",
"os_family": "macOS",
"cloud_provider": "",
"cloud_region": "",
}
)
assert "host operating system is macOS;" in block # no version appended
assert "what environment this process is running in" in block
assert "never invent AWS" in block
assert "`uname`" in block
assert "no cloud provider or cloud region was detected" in block
def test_environment_block_does_not_coach_arbitrary_reachability_probing() -> None:
"""The always-on prompt must not steer the model toward reachability
probing. allow_network has no destination allowlist — coaching sockets
@@ -0,0 +1,74 @@
"""A session-goal continuation turn must know what earlier turns established.
Observed live: turn 1 fetched weather and news successfully and printed the
figures; a later turn ran one slash command, saw no weather tool among *its*
results, and answered "current weather and news retrieval is unavailable in
this environment". The work had been done and the agent reported that it had
not been.
Each continuation is a fresh ``chat`` call carrying only the nudge, and
``record_conversation_turn`` stores assistant prose never tool payloads. So
nothing hands turn N+1 the evidence turn N gathered.
"""
from __future__ import annotations
from core.agent_harness.session.session_core import SessionCore
from core.agent_harness.session_goal.goal import SessionGoal
from core.agent_harness.session_goal.run_until import run_until_session_goal
from core.agent_harness.turns.turn_results import ToolCallingTurnResult, TurnResult
def _turn(text: str, *, executed: int = 0, success: int = 0) -> TurnResult:
return TurnResult(
final_intent="cli_agent_fallback",
action_result=ToolCallingTurnResult(
planned_count=executed,
executed_count=executed,
executed_success_count=success,
has_unhandled_clause=False,
handled=True,
),
assistant_response_text=text,
llm_run=None,
)
def test_continuation_turn_is_told_what_earlier_turns_gathered() -> None:
"""The second turn's input must carry the first turn's evidence.
Without it the second turn can only see its own tools, and an absence there
reads as an absence overall which is how a completed retrieval was
reported as unavailable.
"""
# Arrange — turn 1 gathers real evidence, turn 2 gathers nothing.
session = SessionCore()
prompts: list[str] = []
def _chat(message: str) -> TurnResult:
prompts.append(message)
if len(prompts) == 1:
return _turn("Antarctica is -32C and Hawaii is 24C.", executed=2, success=2)
return _turn("Checked the Slack integration.", executed=1, success=1)
# Act — two turns of one goal.
run_until_session_goal(
_chat,
session,
"weather brief for Antarctica and Hawaii, then send it",
goal=SessionGoal(
condition="weather brief for Antarctica and Hawaii, then send it",
max_outer_turns=3,
# Three items: a two-item checklist hits the same-turn completion
# shortcut and the loop never reaches a continuation.
checklist=("fetch the weather", "correlate news", "send the brief"),
),
)
# Assert — turn 2's prompt must reference what turn 1 established.
assert len(prompts) >= 2, "the goal loop did not run a continuation turn"
continuation = prompts[1]
assert "-32C" in continuation or "Antarctica is" in continuation, (
"the continuation turn was not given the evidence turn 1 gathered, so it "
"can only reason from its own tools:\n" + continuation
)
@@ -182,6 +182,37 @@ def test_host_owned_achieved_without_tools_completes() -> None:
assert session.session_goal.status == SessionGoalStatus.ACHIEVED
def test_database_query_handoff_does_not_attach_session_goal() -> None:
"""Oracle 332: planner session_goal on a DB query must not start the host loop."""
from core.agent_harness.session_goal.goal import attach_session_goal_from_handoffs
from core.agent_harness.turns.assistant_handoff import AssistantHandoff
session = SessionCore()
handoff = AssistantHandoff.from_tool_input(
{
"content": "database_query:mysql_active_connections",
"session_goal": True,
}
)
attached = attach_session_goal_from_handoffs(
session,
handoff.to_handoff_contents(),
condition="Use the MySQL tool to query active connections.",
handoffs=(handoff,),
)
assert attached is None
assert getattr(session, "session_goal", None) is None
# Legacy tags alone (no typed handoffs) also stay one-shot.
session2 = SessionCore()
attached_legacy = attach_session_goal_from_handoffs(
session2,
("database_query:mysql_active_connections", "session_goal:continue"),
condition="query mysql",
)
assert attached_legacy is None
def test_handoff_does_not_replace_active_host_owned_goal() -> None:
from core.agent_harness.session_goal.goal import attach_session_goal_from_handoffs
@@ -108,3 +108,21 @@ def test_session_goal_content_tag_attaches_when_schema_omitted() -> None:
def test_session_goal_achieved_content_tag_is_not_attach() -> None:
handoff = AssistantHandoff.from_tool_input({"content": "Done.\nsession_goal:achieved"})
assert handoff.session_goal is not True
def test_database_query_handoff_ignores_session_goal_attach() -> None:
"""Oracle 332: DB query/connect guidance is one-shot — no host goal loop."""
handoff = AssistantHandoff.from_tool_input(
{
"content": "database_query:mysql_active_connections",
"session_goal": True,
}
)
assert handoff.session_goal is True # decode keeps the flag
assert (
session_goal_from_assistant_handoffs(
(handoff,),
condition="Use the MySQL tool to query active connections.",
)
is None
)
@@ -69,6 +69,22 @@ def test_finalize_prefers_conversational_run_over_pending() -> None:
assert session.terminal.pop_pending_turn_llm() is None
def test_finalize_does_not_duplicate_early_cli_agent_history() -> None:
"""memory_remember + handoff: ActionRenderObserver records once; finalize must not again."""
session = Session()
prompt = "Use the MySQL tool to query active connections."
session.record("cli_agent", prompt)
conversational = LlmRunInfo(model="fresh")
recorder = _FakeRecorder()
accounting = ShellTurnAccounting(session=session, text=prompt, recorder=recorder) # type: ignore[arg-type]
accounting.finalize(_result(llm_run=conversational))
cli_rows = [row for row in session.history if row.get("type") == "cli_agent"]
assert len(cli_rows) == 1
assert cli_rows[0]["text"] == prompt
def test_finalize_sets_structured_error_from_pending_turn_error() -> None:
session = Session()
session.terminal.set_pending_turn_error("config", "ANTHROPIC_API_KEY not set")