colbymchenry--codegraph/scripts/agent-eval/probe-allocation.mjs

#!/usr/bin/env node
/**
 * Deterministic per-file budget-share probe for `codegraph_explore` (CG-6).
 *
 * `probe-explore.mjs` prints what explore returned. This prints how the response
 * was DIVIDED — which files won the byte envelope and in what proportion — and
 * checks that division against a declared expectation. It is the regression gate
 * for GitHub issue #1500 / epic CG-1: an architecture question that doesn't name
 * the exact use-case must concentrate the budget on the code that answers it, not
 * on a generated CRUD layer (or an eval script) that merely name-collides.
 *
 * The numbers come from the CG-4 diagnostic (`CODEGRAPH_EXPLORE_DEBUG`), read back
 * from a JSONL sidecar, so the probe measures the shipping allocator rather than
 * re-deriving shares from the markdown.
 *
 * Fixtures are declared in `allocation-fixtures.json`. A `kind: "fixture"` entry is
 * hermetic — the fixture tree is copied to a fresh temp dir and indexed per run, so
 * two runs on one build give identical numbers. A `kind: "self"` entry reads this
 * repo's live index and therefore moves as the repo changes; its assertions are
 * relative for that reason.
 *
 * Usage (needs a current `npm run build`):
 *   node scripts/agent-eval/probe-allocation.mjs                 # every fixture
 *   node scripts/agent-eval/probe-allocation.mjs payroll-go      # one fixture
 *   node scripts/agent-eval/probe-allocation.mjs --json          # machine-readable
 *   node scripts/agent-eval/probe-allocation.mjs --keep          # keep the temp index
 *
 * Exit code: 0 if every assertion holds, 1 if any fails, 2 on a setup error.
 * BOTH FIXTURES ARE EXPECTED TO FAIL until CG-10/CG-12 land — that failure is the
 * documented bug. Use --expect-fail to invert the exit code while it is the state
 * of the world (0 = still broken, 1 = fixed, go flip the gate).
 */
import { cpSync, mkdtempSync, mkdirSync, readFileSync, rmSync, existsSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { dirname, join, resolve } from 'node:path';
import { fileURLToPath, pathToFileURL } from 'node:url';

const HERE = dirname(fileURLToPath(import.meta.url));
const REPO_ROOT = resolve(HERE, '../..');
const SPEC_PATH = join(HERE, 'allocation-fixtures.json');

const argv = process.argv.slice(2);
const flags = new Set(argv.filter((a) => a.startsWith('--')));
const wanted = argv.filter((a) => !a.startsWith('--'));
const asJson = flags.has('--json');
const keepTemp = flags.has('--keep');
const expectFail = flags.has('--expect-fail');

const say = (line = '') => { if (!asJson) console.log(line); };
const pct = (f) => `${(f * 100).toFixed(1)}%`;
const num = (n) => Math.round(n).toLocaleString('en-US');

/** Load the built dist — the probe measures the shipping allocator, not src. */
async function loadDist() {
  const distIndex = join(REPO_ROOT, 'dist/index.js');
  if (!existsSync(distIndex)) {
    console.error('dist/ not built — run `npm run build` first.');
    process.exit(2);
  }
  const idx = await import(pathToFileURL(distIndex).href);
  const tools = await import(pathToFileURL(join(REPO_ROOT, 'dist/mcp/tools.js')).href);
  // esModuleInterop: dynamic import of CJS yields { default: module.exports, ...named }
  const CodeGraph = idx.default?.default ?? idx.default ?? idx.CodeGraph;
  const ToolHandler = tools.ToolHandler ?? tools.default?.ToolHandler;
  if (typeof CodeGraph?.openSync !== 'function' || typeof ToolHandler !== 'function') {
    console.error('could not resolve CodeGraph/ToolHandler from dist/');
    process.exit(2);
  }
  return { CodeGraph, ToolHandler };
}

/** `internal/gen/**` → /^internal\/gen\/.*$/ . Supports `**`, `*` and literals. */
function globToRegExp(glob) {
  // Park `**` on a sentinel no path can contain, so the `*` pass cannot eat it.
  // Written as an escape, not a literal byte — a raw NUL makes git treat this
  // whole script as binary, which costs every future diff of it.
  const DOUBLE_STAR = '\u0000';
  const escaped = glob.replace(/[.+^${}()|[\]\\]/g, '\\$&');
  const body = escaped
    .replace(/\*\*/g, DOUBLE_STAR)
    .replace(/\*/g, '[^/]*')
    .replaceAll(DOUBLE_STAR, '.*');
  return new RegExp(`^${body}$`);
}

const groupOf = (path, groups) => {
  for (const [name, globs] of Object.entries(groups)) {
    if (globs.some((g) => globToRegExp(g).test(path))) return name;
  }
  return 'other';
};

/**
 * Run one explore call with the diagnostic pointed at a sidecar, and return the
 * report plus the response text.
 */
async function runExplore({ CodeGraph, ToolHandler }, repoPath, query, sidecar) {
  const cg = CodeGraph.openSync(repoPath);
  const prior = process.env.CODEGRAPH_EXPLORE_DEBUG;
  process.env.CODEGRAPH_EXPLORE_DEBUG = sidecar;
  try {
    const res = await new ToolHandler(cg).execute('codegraph_explore', { query });
    const text = res.content?.[0]?.text ?? '';
    const lines = readFileSync(sidecar, 'utf-8').trim().split('\n').filter(Boolean);
    if (lines.length === 0) throw new Error('diagnostic produced no report');
    return { report: JSON.parse(lines[lines.length - 1]), text };
  } finally {
    if (prior === undefined) delete process.env.CODEGRAPH_EXPLORE_DEBUG;
    else process.env.CODEGRAPH_EXPLORE_DEBUG = prior;
    try { cg.close?.(); } catch {}
  }
}

/** Copy a fixture tree to a fresh temp dir and index it — hermetic per run. */
async function materializeFixture({ CodeGraph }, fixturePath) {
  const src = resolve(REPO_ROOT, fixturePath);
  if (!existsSync(src)) throw new Error(`fixture tree not found: ${src}`);
  const dir = mkdtempSync(join(tmpdir(), 'cg-alloc-'));
  cpSync(src, dir, { recursive: true });
  // A stray index inside the checked-in tree would be copied in and reused.
  rmSync(join(dir, '.codegraph'), { recursive: true, force: true });
  const cg = CodeGraph.initSync(dir);
  await cg.indexAll();
  cg.close?.();
  return dir;
}

/** Evaluate one fixture's assertions against its report. Returns check rows. */
function evaluate(fixture, report, text) {
  const { groups, assert: want } = fixture;
  const delivered = new Map();
  const allocated = new Map();
  for (const f of report.files) {
    const g = groupOf(f.path, groups);
    delivered.set(g, (delivered.get(g) ?? 0) + f.share);
    allocated.set(g, (allocated.get(g) ?? 0) + f.allocatedShare);
  }
  const share = (g) => delivered.get(g) ?? 0;
  const top = report.files
    .filter((f) => f.finalChars > 0)
    .sort((a, b) => b.finalChars - a.finalChars)[0];

  const checks = [];
  const add = (name, pass, detail) => checks.push({ name, pass, detail });

  if (want.answerShareAtLeast !== undefined) {
    add(
      `answer group takes >= ${pct(want.answerShareAtLeast)} of the envelope`,
      share('answer') >= want.answerShareAtLeast,
      `answer ${pct(share('answer'))} delivered (${pct(allocated.get('answer') ?? 0)} allocated)`,
    );
  }
  // Same question against the SOURCE the response delivered rather than the
  // whole envelope (CG-26). The envelope-denominated gate above moves whenever
  // the response's prose does — the epilogue surviving instead of being
  // discarded costs it a point, and every additional admitted file that gets
  // paid dilutes it further — so it cannot tell "the answer was starved" from
  // "everything else was also delivered". Allocation is about source bytes;
  // measure it in source bytes.
  if (want.answerShareOfSourceAtLeast !== undefined) {
    const sourceBy = new Map();
    let totalSource = 0;
    for (const f of report.files) {
      const g = groupOf(f.path, groups);
      sourceBy.set(g, (sourceBy.get(g) ?? 0) + f.finalChars);
      totalSource += f.finalChars;
    }
    const answerSource = totalSource > 0 ? (sourceBy.get('answer') ?? 0) / totalSource : 0;
    add(
      `answer group takes >= ${pct(want.answerShareOfSourceAtLeast)} of DELIVERED SOURCE`,
      answerSource >= want.answerShareOfSourceAtLeast,
      `answer ${num(sourceBy.get('answer') ?? 0)} of ${num(totalSource)} source chars (${pct(answerSource)})`,
    );
  }
  if (want.incidentalShareAtMost !== undefined) {
    add(
      `incidental group takes <= ${pct(want.incidentalShareAtMost)} of the envelope`,
      share('incidental') <= want.incidentalShareAtMost,
      `incidental ${pct(share('incidental'))} delivered (${pct(allocated.get('incidental') ?? 0)} allocated)`,
    );
  }
  if (want.topFileGroup) {
    const actual = top ? groupOf(top.path, groups) : '(nothing delivered)';
    add(
      `largest delivered file is in "${want.topFileGroup}"`,
      actual === want.topFileGroup,
      top ? `${top.path} (${pct(top.share)}, group "${actual}")` : 'no file delivered any source',
    );
  }
  for (const path of want.mustDeliverBytes ?? []) {
    const rec = report.files.find((f) => f.path === path);
    add(
      `${path} delivers source`,
      !!rec && rec.finalChars > 0,
      rec
        ? `${num(rec.finalChars)} delivered of ${num(rec.emittedChars)} allocated` +
          (rec.finalChars === 0 && rec.emittedChars > 0 ? ' — hard ceiling dropped the whole section' : '') +
          (rec.emittedChars === 0 ? ` — never rendered (${rec.skipped ?? 'not reached'}, rank #${rec.rank})` : '')
        : 'not among the ranked candidates',
    );
  }
  // Reservation-vs-delivered, per file (CG-36). The share gates above ask which
  // files won the envelope; this asks whether a file that WON its share then
  // actually spent it. A file can rank #1, be reserved the largest slice, and
  // still deliver a quarter of it because the cluster carrying the answer was
  // dropped whole instead of shrunk — and the share gates read that as a pass,
  // since the unspent bytes carry forward and the envelope stays full.
  for (const [path, floor] of Object.entries(want.spendShareAtLeast ?? {})) {
    const rec = report.files.find((f) => f.path === path);
    const spent = rec && rec.allowance ? rec.finalChars / rec.allowance : 0;
    add(
      `${path} spends >= ${pct(floor)} of its reservation`,
      !!rec && rec.allowance > 0 && spent >= floor,
      rec
        ? `${num(rec.finalChars)} delivered of a ${num(rec.allowance ?? 0)} reservation (${pct(spent)})`
        : 'not among the ranked candidates',
    );
  }
  for (const needle of want.mustContain ?? []) {
    add(`response contains "${needle}"`, text.includes(needle), text.includes(needle) ? 'present' : 'absent');
  }
  return { checks, delivered, allocated, top };
}

function printReport(fixture, report, evaluated) {
  const { checks, delivered, allocated } = evaluated;
  const env = report.envelope;
  say('');
  say(`── ${fixture.id} — ${fixture.title}`);
  say(`   query   "${report.query}"`);
  say(`   project ${report.projectRoot} · ${num(report.indexedFileCount)} files indexed`);
  say(
    `   envelope ${num(env.chars)} delivered · ${num(env.allocatedChars)} allocated` +
    ` of ${num(report.budget.maxOutputChars)} budget (hard ceiling ${num(report.budget.hardCeiling)})` +
    `${env.overBudget ? ' [over budget]' : ''}${env.truncated ? ' [TRUNCATED]' : ''}`,
  );
  say('');
  say('   group        alloc%  deliv%');
  for (const g of ['answer', 'incidental', 'other']) {
    if (!delivered.has(g) && !allocated.has(g)) continue;
    say(`   ${g.padEnd(12)} ${pct(allocated.get(g) ?? 0).padStart(6)}  ${pct(delivered.get(g) ?? 0).padStart(6)}`);
  }
  say('');
  say('    #  alloc%  deliv%    bytes  score    graph  hits  gen  render     file');
  for (const f of report.files.filter((f) => f.emittedChars > 0 || f.finalChars > 0)) {
    say(
      '   ' + String(f.rank).padStart(2) + '  ' +
      pct(f.allocatedShare).padStart(6) + '  ' +
      pct(f.share).padStart(6) + '  ' +
      num(f.emittedChars).padStart(7) + '  ' +
      String(f.score).padStart(5) + '  ' +
      f.graphScore.toFixed(5).padStart(7) + '  ' +
      String(f.termHits).padStart(4) + '  ' +
      (f.generated ? ' ✓ ' : '   ') + '  ' +
      ((f.render ?? '-') + (f.clipped ? '*' : '')).padEnd(9) + '  ' +
      f.path,
    );
  }
  say('');
  for (const c of checks) say(`   ${c.pass ? 'PASS' : 'FAIL'}  ${c.name}\n         ${c.detail}`);
}

async function main() {
  const spec = JSON.parse(readFileSync(SPEC_PATH, 'utf-8'));
  const fixtures = spec.fixtures.filter((f) => wanted.length === 0 || wanted.includes(f.id));
  if (fixtures.length === 0) {
    console.error(`no fixture matched ${JSON.stringify(wanted)}; known: ${spec.fixtures.map((f) => f.id).join(', ')}`);
    process.exit(2);
  }

  const dist = await loadDist();
  const sidecarDir = mkdtempSync(join(tmpdir(), 'cg-alloc-diag-'));
  const results = [];
  const temps = [];

  for (const fixture of fixtures) {
    let repoPath;
    if (fixture.kind === 'fixture') {
      repoPath = await materializeFixture(dist, fixture.path);
      temps.push(repoPath);
    } else {
      repoPath = resolve(REPO_ROOT, fixture.path);
      if (!existsSync(join(repoPath, '.codegraph'))) {
        console.error(`${fixture.id}: ${repoPath} has no .codegraph index — run \`codegraph init\` there first.`);
        process.exit(2);
      }
    }

    const sidecar = join(sidecarDir, `${fixture.id}.jsonl`);
    mkdirSync(dirname(sidecar), { recursive: true });
    const { report, text } = await runExplore(dist, repoPath, fixture.query, sidecar);
    const evaluated = evaluate(fixture, report, text);
    printReport(fixture, report, evaluated);
    results.push({
      id: fixture.id,
      kind: fixture.kind,
      query: fixture.query,
      passed: evaluated.checks.every((c) => c.pass),
      checks: evaluated.checks,
      shares: {
        delivered: Object.fromEntries(evaluated.delivered),
        allocated: Object.fromEntries(evaluated.allocated),
      },
      envelope: report.envelope,
      files: report.files.filter((f) => f.emittedChars > 0 || f.finalChars > 0),
    });
  }

  if (!keepTemp) {
    for (const dir of temps) rmSync(dir, { recursive: true, force: true });
    rmSync(sidecarDir, { recursive: true, force: true });
  } else {
    say('');
    say(`   kept: ${[...temps, sidecarDir].join(' ')}`);
  }

  const allPassed = results.every((r) => r.passed);
  if (asJson) {
    console.log(JSON.stringify({ passed: allPassed, fixtures: results }, null, 2));
  } else {
    say('');
    for (const r of results) say(`${r.passed ? 'PASS' : 'FAIL'}  ${r.id}`);
    if (!allPassed) {
      say('');
      say('Failures here are the DOCUMENTED #1500 bug — the budget goes to files that merely');
      say('name-collide with the query. They become the pass gate once CG-10/CG-12 land.');
    }
  }
  process.exit(expectFail ? (allPassed ? 1 : 0) : (allPassed ? 0 : 1));
}

main().catch((err) => {
  console.error(err?.stack ?? String(err));
  process.exit(2);
});