refactor(harness): fork_history + shell-tool capability axes; derive gating (PR 1.8) (#3648)
Final Phase-1 PR of the modular native-harness registry refactor: move registry-parallel enumerations onto HarnessCapabilities. - Add a fork_history axis (ForkHistory enum: none/rebuild/preamble) to HarnessCapabilities, declared per harness in _BUILTIN_CAPABILITIES. Derive the server's two fork-history gating frozensets in _sessions/common.py from it instead of hand-listing. The derivation emits each canonical id plus its reversed native-<key> spelling, because native-claude/native-codex/native-cursor are valid ids canonicalize_harness passes through unchanged and the read sites match on the canonicalized id (guarded by the existing reversed-spelling fork test) — so the derived sets are a superset of the prior literals. - Add optional shell_tool_name / shell_tool_prompt fields carrying the harness bench's shell-tool provocation; delete the bench's hardcoded _NATIVE_TOOL_PROVOCATION table and read the fields off capabilities in native_vendor() (byte-identical (tool_name, prompt) per harness). - Delete the dead _HARNESS_MODULES literal in runtime/harnesses/__init__.py (~120 lines, overwritten unconditionally by harness_modules() next line). - Extend the drift-guard tests in test_harness_capabilities.py. Scope kept tight to the doc's mandate: sets that would need new NativeCodingAgent identity fields (_ANTIGRAVITY_FAMILY_HARNESSES, _PROVIDER_RESOLUTION_HARNESS, *_NATIVE_TERMINAL_ROLE) are left as-is; noted as follow-ups. Co-authored-by: Isaac Signed-off-by: Pat Sukprasert <pattara.sk127@gmail.com>
This commit is contained in:
@@ -441,7 +441,8 @@ Phase 2: 2.1–2.4).
|
||||
| 1.5b-ii Runner launch (3 special arms + turn-path opencode) | landed | #3501 |
|
||||
| 1.5c Runner terminal-ensure (attach path) | landed | #3543 |
|
||||
| 1.6 Runner interrupt/stop (migration; gap-fill deferred) | landed | #3568 |
|
||||
| 1.7 Server seeding loop | in review | (this PR) |
|
||||
| 1.7 Server seeding loop | landed | #3599 |
|
||||
| 1.8 Derive enumerations + fork_history/shell-tool capability axes | in review | (this PR) |
|
||||
|
||||
## Risks and open questions
|
||||
|
||||
|
||||
@@ -76,6 +76,14 @@ class AuthModel(str, Enum):
|
||||
SESSION_SCOPED_CONFIG = "session-scoped-config" # per-session synthesized vendor config
|
||||
|
||||
|
||||
class ForkHistory(str, Enum):
|
||||
"""How a fork (or in-place agent switch) carries prior history into the harness."""
|
||||
|
||||
NONE = "none" # fork launches fresh; no prior turns are carried
|
||||
REBUILD = "rebuild" # rebuild the vendor's resumable session file from copied items
|
||||
PREAMBLE = "preamble" # replay prior turns as a text preamble (server-backed vendors)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class HarnessCapabilities:
|
||||
"""The declared feature set one harness supports.
|
||||
@@ -101,6 +109,16 @@ class HarnessCapabilities:
|
||||
Optional capability fields use ``None`` when the harness makes no claim;
|
||||
the bench reports those declarations as ``UNKNOWN`` rather than assuming
|
||||
the capability is unsupported.
|
||||
:param fork_history: How a fork / in-place agent switch carries prior history
|
||||
into the harness — ``none`` (fresh), ``rebuild`` (rebuild the vendor's
|
||||
resumable session file from copied items), or ``preamble`` (replay prior
|
||||
turns as a text preamble). Drives the server's fork-history gating.
|
||||
:param shell_tool_name: The harness's shell/exec tool name the harness bench
|
||||
provokes to verify tool-calling (e.g. ``"Bash"``, ``"shell"``). ``None``
|
||||
skips the bench's tool/policy probe for this harness.
|
||||
:param shell_tool_prompt: The prompt the bench sends to provoke that tool.
|
||||
Must contain the ``omnigent-bench-ok`` placeholder the probe token-swaps.
|
||||
``None`` skips the probe.
|
||||
"""
|
||||
|
||||
integration_mode: IntegrationMode
|
||||
@@ -116,6 +134,9 @@ class HarnessCapabilities:
|
||||
live_queue: bool | None = None
|
||||
images: bool | None = None
|
||||
compaction: bool | None = None
|
||||
fork_history: ForkHistory = ForkHistory.NONE
|
||||
shell_tool_name: str | None = None
|
||||
shell_tool_prompt: str | None = None
|
||||
|
||||
def as_dict(self) -> dict[str, str | bool | None]:
|
||||
"""Return a JSON-serializable view for the ``/v1/harnesses`` catalog."""
|
||||
@@ -133,4 +154,7 @@ class HarnessCapabilities:
|
||||
"live_queue": self.live_queue,
|
||||
"images": self.images,
|
||||
"compaction": self.compaction,
|
||||
"fork_history": self.fork_history.value,
|
||||
"shell_tool_name": self.shell_tool_name,
|
||||
"shell_tool_prompt": self.shell_tool_prompt,
|
||||
}
|
||||
|
||||
@@ -33,6 +33,7 @@ from omnigent.harness_capabilities import (
|
||||
AuthModel,
|
||||
EffortFamily,
|
||||
Elicitation,
|
||||
ForkHistory,
|
||||
HarnessCapabilities,
|
||||
IntegrationMode,
|
||||
ModelFamily,
|
||||
@@ -305,6 +306,14 @@ _RS = Resume
|
||||
_EF = EffortFamily
|
||||
_MF = ModelFamily
|
||||
_AU = AuthModel
|
||||
_FH = ForkHistory
|
||||
|
||||
# Bench shell-tool provocation prompts (moved off the bench's hardcoded
|
||||
# _NATIVE_TOOL_PROVOCATION table): the generic variant, and a Bash-specific one
|
||||
# for harnesses whose exec tool is literally "Bash". Both keep the
|
||||
# "omnigent-bench-ok" placeholder the bench token-swaps per allow/deny probe.
|
||||
_SHELL_PROMPT = "Use your shell/terminal tool to run this exact command: echo omnigent-bench-ok"
|
||||
_BASH_PROMPT = "Use the Bash tool to run this exact command: echo omnigent-bench-ok"
|
||||
|
||||
# Trailing two bools are (interrupt, streaming). Only the four P0 SDK harnesses
|
||||
# (claude-sdk, codex, pi, openai-agents) have these verified live by the harness
|
||||
@@ -322,6 +331,9 @@ _BUILTIN_CAPABILITIES: dict[str, HarnessCapabilities] = {
|
||||
subagents=True,
|
||||
interrupt=True,
|
||||
streaming=True,
|
||||
fork_history=_FH.REBUILD,
|
||||
shell_tool_name="Bash",
|
||||
shell_tool_prompt=_BASH_PROMPT,
|
||||
),
|
||||
"codex-native": _C(
|
||||
_IM.NATIVE_TUI,
|
||||
@@ -333,6 +345,9 @@ _BUILTIN_CAPABILITIES: dict[str, HarnessCapabilities] = {
|
||||
subagents=True,
|
||||
interrupt=True,
|
||||
streaming=True,
|
||||
fork_history=_FH.REBUILD,
|
||||
shell_tool_name="shell",
|
||||
shell_tool_prompt=_SHELL_PROMPT,
|
||||
),
|
||||
# streaming is declared True unless a live bench run proves a harness does
|
||||
# NOT emit token-level deltas. Only kiro-native is so proven (0 deltas over
|
||||
@@ -350,6 +365,9 @@ _BUILTIN_CAPABILITIES: dict[str, HarnessCapabilities] = {
|
||||
subagents=False,
|
||||
interrupt=True,
|
||||
streaming=True,
|
||||
fork_history=_FH.REBUILD,
|
||||
shell_tool_name="Bash",
|
||||
shell_tool_prompt=_BASH_PROMPT,
|
||||
),
|
||||
# streaming=False is LIVE-VERIFIED: a bench run observed 0 text deltas.
|
||||
"cursor-native": _C(
|
||||
@@ -362,6 +380,9 @@ _BUILTIN_CAPABILITIES: dict[str, HarnessCapabilities] = {
|
||||
subagents=False,
|
||||
interrupt=True,
|
||||
streaming=False,
|
||||
fork_history=_FH.PREAMBLE,
|
||||
# No shell-tool provocation: cursor-native was intentionally absent from
|
||||
# the bench's table (its tool probe is skipped), so leave shell_tool_* None.
|
||||
),
|
||||
# kiro_native_permissions.py: "TUI ACP recorder -> web elicitation".
|
||||
# streaming=False is LIVE-VERIFIED: a full SSE capture recorded 0 text
|
||||
@@ -376,6 +397,9 @@ _BUILTIN_CAPABILITIES: dict[str, HarnessCapabilities] = {
|
||||
subagents=False,
|
||||
interrupt=True,
|
||||
streaming=False,
|
||||
fork_history=_FH.NONE,
|
||||
shell_tool_name="shell",
|
||||
shell_tool_prompt=_SHELL_PROMPT,
|
||||
),
|
||||
"antigravity-native": _C(
|
||||
_IM.NATIVE_TUI,
|
||||
@@ -387,6 +411,9 @@ _BUILTIN_CAPABILITIES: dict[str, HarnessCapabilities] = {
|
||||
subagents=False,
|
||||
interrupt=True,
|
||||
streaming=True,
|
||||
fork_history=_FH.NONE,
|
||||
shell_tool_name="run_command",
|
||||
shell_tool_prompt=_SHELL_PROMPT,
|
||||
),
|
||||
"goose-native": _C(
|
||||
_IM.NATIVE_TUI,
|
||||
@@ -398,6 +425,9 @@ _BUILTIN_CAPABILITIES: dict[str, HarnessCapabilities] = {
|
||||
subagents=False,
|
||||
interrupt=True,
|
||||
streaming=True,
|
||||
fork_history=_FH.NONE,
|
||||
shell_tool_name="developer__shell",
|
||||
shell_tool_prompt=_SHELL_PROMPT,
|
||||
),
|
||||
# streaming=False is LIVE-VERIFIED: a bench run observed 0 text deltas.
|
||||
"qwen-native": _C(
|
||||
@@ -410,6 +440,9 @@ _BUILTIN_CAPABILITIES: dict[str, HarnessCapabilities] = {
|
||||
subagents=False,
|
||||
interrupt=True,
|
||||
streaming=False,
|
||||
fork_history=_FH.REBUILD,
|
||||
shell_tool_name="run_shell_command",
|
||||
shell_tool_prompt=_SHELL_PROMPT,
|
||||
),
|
||||
"kimi-native": _C(
|
||||
_IM.NATIVE_TUI,
|
||||
@@ -421,6 +454,9 @@ _BUILTIN_CAPABILITIES: dict[str, HarnessCapabilities] = {
|
||||
subagents=False,
|
||||
interrupt=True,
|
||||
streaming=True,
|
||||
fork_history=_FH.NONE,
|
||||
shell_tool_name="Bash",
|
||||
shell_tool_prompt=_BASH_PROMPT,
|
||||
),
|
||||
"opencode-native": _C(
|
||||
_IM.NATIVE_SERVER,
|
||||
@@ -432,6 +468,9 @@ _BUILTIN_CAPABILITIES: dict[str, HarnessCapabilities] = {
|
||||
subagents=True,
|
||||
interrupt=True,
|
||||
streaming=True,
|
||||
fork_history=_FH.PREAMBLE,
|
||||
# NATIVE_SERVER, not driven by the bench's native-tui tool probe, so
|
||||
# shell_tool_* stay None.
|
||||
),
|
||||
"hermes-native": _C(
|
||||
_IM.NATIVE_TUI,
|
||||
@@ -443,6 +482,9 @@ _BUILTIN_CAPABILITIES: dict[str, HarnessCapabilities] = {
|
||||
subagents=False,
|
||||
interrupt=True,
|
||||
streaming=True,
|
||||
fork_history=_FH.REBUILD,
|
||||
shell_tool_name="terminal",
|
||||
shell_tool_prompt=_SHELL_PROMPT,
|
||||
),
|
||||
# SDK / subprocess harnesses (run the vendor model directly). The first four
|
||||
# are bench-verified interrupt=streaming=True.
|
||||
|
||||
@@ -26,126 +26,12 @@ from __future__ import annotations
|
||||
|
||||
from omnigent.harness_plugins import harness_modules
|
||||
|
||||
# Harness-name → fully-qualified module path. Each module must
|
||||
# export ``create_app() -> FastAPI``; the runner imports the module,
|
||||
# calls the factory, and serves the result over a Unix socket.
|
||||
#
|
||||
# Populated as per-harness wraps land in Phase 1 step 4. The test
|
||||
# suite injects fixture entries at test time (via direct dict
|
||||
# mutation in conftest fixtures).
|
||||
_HARNESS_MODULES: dict[str, str] = {
|
||||
# Step 4b: claude-sdk harness wrap. See
|
||||
# omnigent/inner/claude_sdk_harness.py.
|
||||
"claude-sdk": "omnigent.inner.claude_sdk_harness",
|
||||
# User-facing alias accepted in specs / Omnigent harness dispatch.
|
||||
"claude": "omnigent.inner.claude_sdk_harness",
|
||||
# Native Claude Code terminal bridge used by ``omnigent claude``.
|
||||
"claude-native": "omnigent.inner.claude_native_harness",
|
||||
# Native Codex TUI terminal bridge used by ``omnigent codex``.
|
||||
"codex-native": "omnigent.inner.codex_native_harness",
|
||||
# Step 4c: codex harness wrap. See
|
||||
# omnigent/inner/codex_harness.py.
|
||||
"codex": "omnigent.inner.codex_harness",
|
||||
# Step 4d: pi harness wrap. See
|
||||
# omnigent/inner/pi_harness.py.
|
||||
"pi": "omnigent.inner.pi_harness",
|
||||
# Native Pi TUI bridge used by ``omnigent pi``.
|
||||
"pi-native": "omnigent.inner.pi_native_harness",
|
||||
# Native Antigravity (agy) TUI terminal bridge used by
|
||||
# ``omnigent antigravity``. The in-process SDK counterpart is the
|
||||
# canonical ``antigravity`` harness registered below.
|
||||
"antigravity-native": "omnigent.inner.antigravity_native_harness",
|
||||
# Step 4e: openai-agents harness wrap. See
|
||||
# omnigent/inner/openai_agents_sdk_harness.py. Registry
|
||||
# key is the Omnigent-side spelling (``openai-agents``,
|
||||
# no ``-sdk`` suffix) to match
|
||||
# ``OmnigentExecutor``'s harness allowlist and the
|
||||
# ``executor.harness`` field used in Omnigent YAML; the
|
||||
# backing Python module retains the ``_sdk`` suffix because
|
||||
# the underlying SDK package is ``openai-agents`` and the
|
||||
# executor class is :class:`OpenAIAgentsSDKExecutor`.
|
||||
"openai-agents": "omnigent.inner.openai_agents_sdk_harness",
|
||||
# cursor harness wrap (Cursor's ``cursor-agent`` CLI, headless). See
|
||||
# omnigent/inner/cursor_harness.py.
|
||||
"cursor": "omnigent.inner.cursor_harness",
|
||||
# Kimi Code CLI harness wrap (Moonshot AI's ``kimi`` CLI, headless). See
|
||||
# omnigent/inner/kimi_harness.py. Drives ``kimi --print --output-format
|
||||
# stream-json`` per turn; resumes via ``--session <uuid>`` captured from
|
||||
# the prior turn's stderr.
|
||||
"kimi": "omnigent.inner.kimi_harness",
|
||||
# User-facing alias matching the upstream product name ("Kimi Code").
|
||||
"kimi-code": "omnigent.inner.kimi_harness",
|
||||
# cursor-native harness wrap. Drives the resident ``cursor-agent`` TUI by
|
||||
# injecting each web-UI turn into its tmux pane and mirroring the transcript
|
||||
# back — a native-CLI harness like claude/codex/pi-native, so it IS in
|
||||
# ``NATIVE_HARNESSES``. See omnigent/inner/cursor_native_harness.py.
|
||||
"cursor-native": "omnigent.inner.cursor_native_harness",
|
||||
# Native Kiro TUI bridge used by ``omnigent kiro``.
|
||||
"kiro-native": "omnigent.inner.kiro_native_harness",
|
||||
# goose-native harness wrap. Drives the resident ``goose session`` TUI by
|
||||
# injecting each web-UI turn into its tmux pane and mirroring the transcript
|
||||
# back from Goose's SQLite session store — a native-CLI harness like
|
||||
# cursor-native, so it IS in ``NATIVE_HARNESSES``. See
|
||||
# omnigent/inner/goose_native_harness.py.
|
||||
"goose-native": "omnigent.inner.goose_native_harness",
|
||||
# qwen-native harness wrap. Drives the resident ``qwen`` TUI by appending
|
||||
# JSONL ``submit`` commands to its ``--input-file`` and mirroring the
|
||||
# transcript back from its ``--json-file`` event stream — a native-CLI
|
||||
# harness like goose-native, so it IS in ``NATIVE_HARNESSES``. The bare
|
||||
# ``qwen`` name stays the ACP-piped harness. See
|
||||
# omnigent/inner/qwen_native_harness.py.
|
||||
"qwen-native": "omnigent.inner.qwen_native_harness",
|
||||
# Native Kimi Code TUI bridge used by ``omnigent kimi``. Drives the resident
|
||||
# ``kimi`` TUI by injecting each web-UI turn into its tmux pane (tmux paste)
|
||||
# — a native-CLI harness like claude/codex/cursor-native, so it IS in
|
||||
# ``NATIVE_HARNESSES``. Distinct from the headless ``kimi`` SDK harness
|
||||
# above. See omnigent/inner/kimi_native_harness.py.
|
||||
"kimi-native": "omnigent.inner.kimi_native_harness",
|
||||
# Google Antigravity SDK harness wrap. See
|
||||
# omnigent/inner/antigravity_harness.py. In-process SDK harness
|
||||
# (``google-antigravity``), like openai-agents — Omnigent spawns no CLI
|
||||
# binary or sandbox subprocess (the SDK itself launches a native
|
||||
# localharness binary; needs glibc >=~2.36). Drives Gemini 3.5 Flash by
|
||||
# default (also Claude / GPT-OSS), with Gemini API-key or Vertex AI auth.
|
||||
"antigravity": "omnigent.inner.antigravity_harness",
|
||||
# Qwen Code harness wrap. See omnigent/inner/qwen_harness.py.
|
||||
# Drives the ``qwen`` CLI in ACP mode (``qwen --acp``) for agent execution.
|
||||
"qwen": "omnigent.inner.qwen_harness",
|
||||
# Headless Goose harness wrap. See omnigent/inner/goose_harness.py.
|
||||
# Drives Block's ``goose`` CLI in ACP mode (``goose acp``) — the chat-first
|
||||
# counterpart to the terminal-first ``goose-native`` TUI harness. Tool
|
||||
# approvals surface as web elicitation cards via session/request_permission.
|
||||
"goose": "omnigent.inner.goose_harness",
|
||||
# Native OpenCode server bridge used by ``omnigent opencode``. The runner
|
||||
# owns ``opencode serve`` + an SSE forwarder and this harness injects each
|
||||
# web-UI turn over loopback HTTP — a native-server harness like
|
||||
# codex-native, so both ``opencode-native`` and its ``native-opencode``
|
||||
# alias are in ``NATIVE_HARNESSES``. See
|
||||
# omnigent/inner/opencode_native_harness.py.
|
||||
"opencode-native": "omnigent.inner.opencode_native_harness",
|
||||
# ``opencode`` is accepted as a friendly alias for the canonical
|
||||
# ``opencode-native`` (there is no separate SDK ``opencode`` harness).
|
||||
"opencode": "omnigent.inner.opencode_native_harness",
|
||||
# GitHub Copilot SDK harness wrap. See omnigent/inner/copilot_harness.py.
|
||||
# In-process SDK harness (``github-copilot-sdk``), like cursor / antigravity:
|
||||
# the SDK bundles the Copilot CLI binary it drives as a backing server, so
|
||||
# Omnigent spawns no separately-installed CLI. Authenticates against GitHub's
|
||||
# Copilot backend with a GitHub token (no Databricks gateway).
|
||||
"copilot": "omnigent.inner.copilot_harness",
|
||||
# Hermes Agent harness wrap. Runs the ``hermes`` CLI as a subprocess
|
||||
# for each turn, managing its own session state via Hermes' SQLite
|
||||
# session store. See omnigent/inner/hermes_harness.py and
|
||||
# omnigent/inner/hermes_executor.py. The ``hermes`` binary must be
|
||||
# on PATH (or set by OMNIGENT_HERMES_PATH; legacy HARNESS_HERMES_PATH honored).
|
||||
"hermes": "omnigent.inner.hermes_harness",
|
||||
# hermes-native harness wrap. Drives the resident ``hermes`` TUI by
|
||||
# injecting each web-UI turn into its tmux pane and mirroring the transcript
|
||||
# back from Hermes' SQLite ``state.db`` session store — a native-CLI harness
|
||||
# like goose-native, so it IS in ``NATIVE_HARNESSES``. The bare ``hermes``
|
||||
# name stays the headless subprocess harness. See
|
||||
# omnigent/inner/hermes_native_harness.py.
|
||||
"hermes-native": "omnigent.inner.hermes_native_harness",
|
||||
}
|
||||
# Harness-name -> fully-qualified module path, sourced from the harness
|
||||
# registry (built-ins + installed community plugins). Each module must
|
||||
# export ``create_app() -> FastAPI``; the runner imports the module, calls
|
||||
# the factory, and serves the result over a Unix socket. The historical
|
||||
# mutable-dict surface is preserved (tests inject fixture entries by dict
|
||||
# mutation), but the contents come from the dynamic registry, not a literal.
|
||||
|
||||
# Keep the historical mutable dict surface while sourcing builtins and
|
||||
# community plugins from the dynamic registry.
|
||||
|
||||
@@ -22,6 +22,7 @@ from omnigent.db.db_models import LABEL_VALUE_MAX_LEN
|
||||
from omnigent.entities.conversation import (
|
||||
ITEM_TYPE_TO_DATA_CLS,
|
||||
)
|
||||
from omnigent.harness_capabilities import ForkHistory
|
||||
from omnigent.harness_plugins import (
|
||||
CLAUDE_NATIVE_CODING_AGENT,
|
||||
CODEX_NATIVE_CODING_AGENT,
|
||||
@@ -29,6 +30,7 @@ from omnigent.harness_plugins import (
|
||||
KIRO_NATIVE_CODING_AGENT,
|
||||
OPENCODE_NATIVE_CODING_AGENT,
|
||||
PI_NATIVE_CODING_AGENT,
|
||||
harness_capabilities,
|
||||
)
|
||||
from omnigent.runner.routing import RunnerRouter
|
||||
from omnigent.server.host_registry import HostRegistry
|
||||
@@ -586,27 +588,34 @@ _STOP_RUNNER_RESULT_TIMEOUT_S = 10.0
|
||||
_COMPACT_LOCKS: weakref.WeakValueDictionary[str, asyncio.Lock] = weakref.WeakValueDictionary()
|
||||
|
||||
|
||||
_FORK_HISTORY_NATIVE_HARNESSES: frozenset[str] = frozenset(
|
||||
{
|
||||
"claude-native",
|
||||
"native-claude",
|
||||
"codex-native",
|
||||
"native-codex",
|
||||
"hermes-native",
|
||||
"native-hermes",
|
||||
"pi-native",
|
||||
# qwen-native rebuilds qwen's on-disk chat recording (+ runtime/meta
|
||||
# sidecars) from the copied items, so a fork carries history into the
|
||||
# qwen TUI (see _build_qwen_fork_recording / write_qwen_session_recording).
|
||||
# Only the canonical id is needed — "native-qwen" is aliased to it.
|
||||
"qwen-native",
|
||||
}
|
||||
)
|
||||
# Derived from the fork_history capability axis (see harness_capabilities). A
|
||||
# harness declaring fork_history=REBUILD rebuilds its resumable session file
|
||||
# from the copied items (e.g. qwen rebuilds its on-disk chat recording via
|
||||
# _build_qwen_fork_recording); PREAMBLE replays prior turns as text
|
||||
# (cursor/opencode, whose conversations are server-backed).
|
||||
#
|
||||
# The read sites match on canonicalize_harness(harness_kind), but several
|
||||
# reversed "native-<x>" spellings (native-claude / native-codex / native-cursor)
|
||||
# are valid harness ids that canonicalize_harness passes through UNCHANGED — so
|
||||
# each canonical id must be listed alongside its reversed spelling, exactly as
|
||||
# the pre-derivation literals did (see test_fork_reversed_native_spelling_carry_gating).
|
||||
def _fork_history_harness_ids(behavior: ForkHistory) -> frozenset[str]:
|
||||
ids: set[str] = set()
|
||||
for harness, caps in harness_capabilities().items():
|
||||
if caps.fork_history is not behavior:
|
||||
continue
|
||||
ids.add(harness)
|
||||
# Add the reversed "native-<key>" spelling for a canonical "<key>-native"
|
||||
# id; it canonicalizes to itself for some harnesses, so membership needs it.
|
||||
if harness.endswith("-native"):
|
||||
ids.add(f"native-{harness[: -len('-native')]}")
|
||||
return frozenset(ids)
|
||||
|
||||
|
||||
_CURSOR_FORK_HISTORY_HARNESSES: frozenset[str] = frozenset(
|
||||
{"cursor-native", "native-cursor", "opencode-native", "native-opencode"}
|
||||
)
|
||||
_FORK_HISTORY_NATIVE_HARNESSES: frozenset[str] = _fork_history_harness_ids(ForkHistory.REBUILD)
|
||||
|
||||
|
||||
_CURSOR_FORK_HISTORY_HARNESSES: frozenset[str] = _fork_history_harness_ids(ForkHistory.PREAMBLE)
|
||||
|
||||
|
||||
_DENY_SENTINEL_PREFIX = "[Denied by policy: "
|
||||
|
||||
@@ -123,8 +123,7 @@ class NativeVendor:
|
||||
Used for documentation and as a non-empty gate: empty means the
|
||||
tool/policy probes cannot run for this vendor and SKIP. The deny gates
|
||||
on the tool_call phase (name-agnostic), so this need not be wire-exact.
|
||||
Not derivable from the capability model, so it is an explicit per-vendor
|
||||
fact (see :data:`_NATIVE_TOOL_PROVOCATION`).
|
||||
Sourced from the harness's ``shell_tool_name`` capability field.
|
||||
:param tool_prompt: A prompt that reliably makes the vendor call its shell
|
||||
tool. Empty when :attr:`tool_name` is.
|
||||
"""
|
||||
@@ -141,23 +140,6 @@ class NativeVendor:
|
||||
# These vendors create external_session_id only after the first message.
|
||||
_LAZY_CHAT_HARNESSES: frozenset[str] = frozenset({"cursor-native"})
|
||||
|
||||
# Missing entries skip tool and policy probes.
|
||||
_SHELL_PROMPT = "Use your shell/terminal tool to run this exact command: echo omnigent-bench-ok"
|
||||
_NATIVE_TOOL_PROVOCATION: dict[str, tuple[str, str]] = {
|
||||
"claude-native": (
|
||||
"Bash",
|
||||
"Use the Bash tool to run this exact command: echo omnigent-bench-ok",
|
||||
),
|
||||
"codex-native": ("shell", _SHELL_PROMPT),
|
||||
"pi-native": ("Bash", "Use the Bash tool to run this exact command: echo omnigent-bench-ok"),
|
||||
"kiro-native": ("shell", _SHELL_PROMPT),
|
||||
"qwen-native": ("run_shell_command", _SHELL_PROMPT),
|
||||
"goose-native": ("developer__shell", _SHELL_PROMPT),
|
||||
"hermes-native": ("terminal", _SHELL_PROMPT),
|
||||
"antigravity-native": ("run_command", _SHELL_PROMPT),
|
||||
"kimi-native": ("Bash", "Use the Bash tool to run this exact command: echo omnigent-bench-ok"),
|
||||
}
|
||||
|
||||
_NATIVE_OMNIGENT_MCP_HARNESSES = frozenset(
|
||||
{
|
||||
"antigravity-native",
|
||||
@@ -194,7 +176,10 @@ def native_vendor(harness: str) -> NativeVendor | None:
|
||||
caps = harness_capabilities().get(harness)
|
||||
if caps is None or caps.integration_mode is not IntegrationMode.NATIVE_TUI:
|
||||
return None
|
||||
tool_name, tool_prompt = _NATIVE_TOOL_PROVOCATION.get(harness, ("", ""))
|
||||
# Shell-tool provocation is declared on the capability record; an unset
|
||||
# (None) name/prompt leaves these empty, which skips the tool/policy probes.
|
||||
tool_name = caps.shell_tool_name or ""
|
||||
tool_prompt = caps.shell_tool_prompt or ""
|
||||
return NativeVendor(
|
||||
harness=harness,
|
||||
agent_name=f"{harness}-ui",
|
||||
|
||||
@@ -15,6 +15,7 @@ from omnigent.harness_capabilities import (
|
||||
AuthModel,
|
||||
EffortFamily,
|
||||
Elicitation,
|
||||
ForkHistory,
|
||||
HarnessCapabilities,
|
||||
IntegrationMode,
|
||||
ModelFamily,
|
||||
@@ -113,6 +114,10 @@ def test_optional_bench_capabilities_default_to_unknown() -> None:
|
||||
assert capability.live_queue is None
|
||||
assert capability.images is None
|
||||
assert capability.compaction is None
|
||||
# New axes default to their unset value: fork_history=none, shell tool None.
|
||||
assert capability.fork_history is ForkHistory.NONE
|
||||
assert capability.shell_tool_name is None
|
||||
assert capability.shell_tool_prompt is None
|
||||
assert capability.as_dict() == {
|
||||
"integration_mode": "sdk-in-process",
|
||||
"elicitation": "none",
|
||||
@@ -127,6 +132,9 @@ def test_optional_bench_capabilities_default_to_unknown() -> None:
|
||||
"live_queue": None,
|
||||
"images": None,
|
||||
"compaction": None,
|
||||
"fork_history": "none",
|
||||
"shell_tool_name": None,
|
||||
"shell_tool_prompt": None,
|
||||
}
|
||||
|
||||
|
||||
@@ -201,3 +209,96 @@ def test_setup_steps_by_spelling_covers_native_and_installable_ids() -> None:
|
||||
# Installable ids that are NOT picker rows still resolve.
|
||||
assert "opencode" in by_spelling
|
||||
assert "qwen" in by_spelling
|
||||
|
||||
|
||||
def test_fork_history_axis_matches_canonical_declarations() -> None:
|
||||
"""fork_history is the source of truth for the server's fork-history gating.
|
||||
|
||||
The two frozensets in ``_sessions/common`` are DERIVED from this axis; the
|
||||
canonical harness id is in the rebuild set iff it declares REBUILD, the
|
||||
preamble set iff PREAMBLE. (The sets also carry reversed ``native-<x>``
|
||||
spellings — asserted separately below.)
|
||||
"""
|
||||
from omnigent.server.routes._sessions.common import (
|
||||
_CURSOR_FORK_HISTORY_HARNESSES,
|
||||
_FORK_HISTORY_NATIVE_HARNESSES,
|
||||
)
|
||||
|
||||
for harness, capability in harness_capabilities().items():
|
||||
fh = capability.fork_history
|
||||
assert (harness in _FORK_HISTORY_NATIVE_HARNESSES) == (fh is ForkHistory.REBUILD), harness
|
||||
assert (harness in _CURSOR_FORK_HISTORY_HARNESSES) == (fh is ForkHistory.PREAMBLE), harness
|
||||
|
||||
|
||||
def test_fork_history_derivation_preserves_prior_membership() -> None:
|
||||
"""The derived sets are a superset of the pre-1.8 hand-maintained literals.
|
||||
|
||||
Behavior-preservation pin: the exact pre-derivation membership (canonical ids
|
||||
PLUS the reversed ``native-<x>`` spellings the literals carried) must still be
|
||||
present, or a fork silently loses history. The derived set may add extra
|
||||
reversed spellings that canonicalize into it (harmless at the read site).
|
||||
"""
|
||||
from omnigent.server.routes._sessions.common import (
|
||||
_CURSOR_FORK_HISTORY_HARNESSES,
|
||||
_FORK_HISTORY_NATIVE_HARNESSES,
|
||||
)
|
||||
|
||||
prior_rebuild = frozenset(
|
||||
{
|
||||
"claude-native",
|
||||
"native-claude",
|
||||
"codex-native",
|
||||
"native-codex",
|
||||
"hermes-native",
|
||||
"native-hermes",
|
||||
"pi-native",
|
||||
"qwen-native",
|
||||
}
|
||||
)
|
||||
prior_preamble = frozenset(
|
||||
{"cursor-native", "native-cursor", "opencode-native", "native-opencode"}
|
||||
)
|
||||
assert prior_rebuild <= _FORK_HISTORY_NATIVE_HARNESSES
|
||||
assert prior_preamble <= _CURSOR_FORK_HISTORY_HARNESSES
|
||||
|
||||
|
||||
def test_reversed_native_spellings_classify_fork_history() -> None:
|
||||
"""Reversed ``native-<x>`` spellings that don't canonicalize are still gated.
|
||||
|
||||
``native-claude`` / ``native-codex`` / ``native-cursor`` are valid harness
|
||||
ids that ``canonicalize_harness`` passes through unchanged (they are NOT
|
||||
registered aliases). The read sites match on the canonicalized id, so the
|
||||
derived sets must contain these literal spellings — this guards the
|
||||
regression where an identically-behaving reversed-spelling agent silently
|
||||
loses fork history. Mirrors test_fork_reversed_native_spelling_carry_gating.
|
||||
"""
|
||||
from omnigent.harness_aliases import canonicalize_harness
|
||||
from omnigent.server.routes._sessions.common import (
|
||||
_CURSOR_FORK_HISTORY_HARNESSES,
|
||||
_FORK_HISTORY_NATIVE_HARNESSES,
|
||||
)
|
||||
|
||||
for spelling in ("native-claude", "native-codex"):
|
||||
assert canonicalize_harness(spelling) in _FORK_HISTORY_NATIVE_HARNESSES, spelling
|
||||
assert canonicalize_harness("native-cursor") in _CURSOR_FORK_HISTORY_HARNESSES
|
||||
|
||||
|
||||
def test_native_tui_harnesses_declare_shell_tool_provocation() -> None:
|
||||
"""Native-TUI harnesses the bench tool-probes declare a shell-tool prompt.
|
||||
|
||||
The bench derives its provocation from these fields (was a hardcoded table);
|
||||
a prompt must carry the ``omnigent-bench-ok`` placeholder the probe
|
||||
token-swaps. cursor-native is intentionally probe-skipped (no shell tool),
|
||||
matching its prior absence from the table.
|
||||
"""
|
||||
caps = harness_capabilities()
|
||||
probe_skipped = {"cursor-native"}
|
||||
for harness, capability in caps.items():
|
||||
if capability.integration_mode is not IntegrationMode.NATIVE_TUI:
|
||||
continue
|
||||
if harness in probe_skipped:
|
||||
assert capability.shell_tool_name is None, harness
|
||||
continue
|
||||
assert capability.shell_tool_name, harness
|
||||
assert capability.shell_tool_prompt, harness
|
||||
assert "omnigent-bench-ok" in capability.shell_tool_prompt, harness
|
||||
|
||||
Reference in New Issue
Block a user