7ddd3fa7eb
Reproducible suite measuring the managed CodeGraph AI offload and the front-load UserPromptSubmit hook (approach 1) vs raw codegraph and no-codegraph, across repo sizes, on time / main-session tokens+cost / CodeGraph-AI tokens+cost / accuracy. All agent arms run claude -p sonnet --effort high; eval-only, nothing shipped. - offload-eval-setup.sh: clone + index 4 memory-probe-verified "not-trained-on" repos (mtkruto/postybirb/shapeshift/trezor — small→large) so the no-codegraph baseline is honest. - offload-eval-3arm.sh / -frontload.sh: one repo, the arms (offload/raw/nocg, frontload). - offload-eval-matrix.sh / -frontload-matrix.sh: drive all 4 tiers. - offload-eval-hook.mjs: the front-load hook (self-locates its engine; CG_FRONTLOAD_DEBUG to log). - offload-eval-metrics.mjs / -judge.mjs (Sonnet) / -summarize.mjs: extract, score, aggregate. - offload-eval-ground-truth.json: source-verified canonical flows (the judge's reference). - offload-eval.md: usage + the 2026-06 findings (raw = the win; offload least-accurate; front-load solves adoption but exposes explore's dynamic-dispatch gaps). Scripts are path-portable (self-locating $HERE/$ENGINE; AGENT_EVAL_OUT scratch dir). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
85 lines
4.6 KiB
JavaScript
85 lines
4.6 KiB
JavaScript
#!/usr/bin/env node
|
|
// UserPromptSubmit hook — APPROACH 1: additive context-injection.
|
|
// Front-loads codegraph's structural answer for flow/impact/"how/where" prompts so the
|
|
// agent's reflex grep/read has nothing left to find. Strictly additive (never blocks),
|
|
// gated to structural prompts (no cost otherwise), and uses RAW explore (offload disabled)
|
|
// so the injected context is accurate — never the (currently low-fidelity) synthesis.
|
|
//
|
|
// Reads {prompt, cwd} as JSON on stdin; prints the explore result to stdout (which Claude
|
|
// Code injects into the agent's context). Any failure -> silent exit 0 (degradable).
|
|
import { pathToFileURL, fileURLToPath } from 'node:url';
|
|
import { resolve, join, dirname } from 'node:path';
|
|
import { existsSync, readFileSync, appendFileSync } from 'node:fs';
|
|
|
|
// Resolve the engine repo from this script's own location (scripts/agent-eval/ -> ../..),
|
|
// overridable with CG_ENGINE. The hook ships inside the repo, so it finds its own dist.
|
|
const HERE = dirname(fileURLToPath(import.meta.url));
|
|
const ENGINE = process.env.CG_ENGINE || resolve(HERE, '..', '..');
|
|
const BUDGET = Number(process.env.CG_FRONTLOAD_BUDGET || 16000);
|
|
|
|
// Debug log only when CG_FRONTLOAD_DEBUG is set to a file path (the harness points it at a
|
|
// log to count injections); off by default so the shipped hook writes nothing extra.
|
|
const DBG = process.env.CG_FRONTLOAD_DEBUG;
|
|
const dbg = (m) => { if (!DBG) return; try { appendFileSync(DBG, `[${new Date().toISOString()}] ${m}\n`); } catch { /* ignore */ } };
|
|
|
|
let input = {};
|
|
try { input = JSON.parse(readFileSync(0, 'utf8')); } catch (e) { dbg('stdin parse fail: ' + e.message); }
|
|
const prompt = String(input.prompt || '');
|
|
const cwd = String(input.cwd || process.cwd());
|
|
dbg(`invoked: promptLen=${prompt.length} cwd=${cwd}`);
|
|
|
|
// Gate: only structural / flow / impact / where-how questions. Cheap regex; silent no-op
|
|
// otherwise so non-structural prompts ("fix this typo") cost nothing.
|
|
const STRUCTURAL = /\b(how|where|trace|flow|path|reach(es|ed)?|call(s|ed|er|ers|ee)?|depend|impact|affect|wire[ds]?|connect|implement|architect|structure|breaks?|what calls|why does)\b/i;
|
|
if (!prompt || !STRUCTURAL.test(prompt)) { dbg('gate: non-structural, no-op'); process.exit(0); }
|
|
dbg('gate: structural PASS');
|
|
|
|
// Find the index: cwd, then walk up a few levels.
|
|
let root = cwd, found = null;
|
|
for (let i = 0; i < 6 && root; i++) {
|
|
if (existsSync(join(root, '.codegraph'))) { found = root; break; }
|
|
const parent = resolve(root, '..'); if (parent === root) break; root = parent;
|
|
}
|
|
if (!found) { dbg(`no .codegraph found from cwd=${cwd}`); process.exit(0); }
|
|
dbg(`found index at ${found}`);
|
|
|
|
try {
|
|
process.env.CODEGRAPH_OFFLOAD_DISABLE = '1'; // raw, accurate — never the unfixed offload
|
|
process.env.CODEGRAPH_TELEMETRY = '0'; process.env.DO_NOT_TRACK = '1';
|
|
const load = async (rel) => import(pathToFileURL(resolve(ENGINE, rel)).href);
|
|
const idx = await load('dist/index.js');
|
|
const tools = await load('dist/mcp/tools.js');
|
|
const CodeGraph = idx.default?.default ?? idx.default ?? idx.CodeGraph;
|
|
const ToolHandler = tools.ToolHandler ?? tools.default?.ToolHandler;
|
|
if (typeof CodeGraph?.openSync !== 'function' || typeof ToolHandler !== 'function') process.exit(0);
|
|
|
|
// Retry once on a transient busy/locked index (the hook's openSync can race a
|
|
// freshly-warming daemon on the first prompt of a session).
|
|
let text = '';
|
|
for (let attempt = 1; attempt <= 2; attempt++) {
|
|
try {
|
|
const cg = CodeGraph.openSync(found);
|
|
const h = new ToolHandler(cg);
|
|
const res = await h.execute('codegraph_explore', { query: prompt });
|
|
text = res?.content?.[0]?.text ?? '';
|
|
try { cg.close?.(); } catch { /* ignore */ }
|
|
dbg(`explore attempt ${attempt} returned ${text.length} chars`);
|
|
break;
|
|
} catch (e) {
|
|
dbg(`explore attempt ${attempt} failed: ${e?.message || e}`);
|
|
if (attempt === 2) throw e;
|
|
await new Promise((r) => setTimeout(r, 800));
|
|
}
|
|
}
|
|
if (!text.trim()) { dbg('empty explore result, no-op'); process.exit(0); }
|
|
if (text.length > BUDGET) text = text.slice(0, BUDGET) + '\n…[front-load truncated to budget]';
|
|
|
|
process.stdout.write(
|
|
`## CodeGraph structural context (auto-retrieved for this question)\n` +
|
|
`The code graph was queried for your question; the relevant symbols, source, and call flow are below. ` +
|
|
`Treat the quoted source as already read. If you need more, call codegraph_explore with specific symbol names rather than grepping or reading files.\n\n` +
|
|
text + '\n'
|
|
);
|
|
dbg(`INJECTED ${text.length} chars`);
|
|
} catch (e) { dbg('ERROR: ' + (e?.stack || e?.message || e)); process.exit(0); } // degradable
|