fix(bench): harden deterministic scale reports
This commit is contained in:
KumamuKuma
@@ -48,6 +48,15 @@ The benchmark tests compile that schema and validate normal, empty, degraded,
|
||||
and partial failed reports against it. The runner does not perform runtime
|
||||
schema validation.
|
||||
|
||||
The JSON and Markdown files share a generated `pairId`. Writers for the same
|
||||
pair are serialized with an exclusive lock, and a caught synchronous delivery
|
||||
failure attempts to restore both previous files. This is not a crash-atomic
|
||||
filesystem transaction: process termination, power loss, or a machine crash
|
||||
between renames can still leave a missing or mismatched pair. Consumers should
|
||||
compare the `pairId` in both files and reject a mismatch. Recovery failures are
|
||||
reported only as bounded counters; report errors do not expose filesystem
|
||||
paths.
|
||||
|
||||
The default concurrency is 5. Override it with `--concurrency 1` through
|
||||
`--concurrency 32`. Use the same concurrency when comparing runs.
|
||||
|
||||
@@ -63,6 +72,12 @@ Published warning summaries are bounded. Each entry records the stage, the
|
||||
total warning `count`, up to five sanitized `messages`, and a `truncated` flag
|
||||
that reports omitted or shortened detail.
|
||||
|
||||
Structure diagnostics are bounded across the whole run, rather than once per
|
||||
batch. The report retains deterministic samples and aggregate success, failure,
|
||||
and skip counts, but discards raw per-worker stdout and stderr after those
|
||||
summaries are formed. Structural coverage counts successful structure analyses;
|
||||
failed analyses remain failures instead of being reported as skipped work.
|
||||
|
||||
`--keep-artifacts` preserves intermediate files and prints their location.
|
||||
Those private files contain absolute paths and detailed structural data; do not
|
||||
publish them without reviewing and sanitizing them.
|
||||
@@ -75,7 +90,8 @@ For results that another contributor can reproduce:
|
||||
repository. The report captures both commits when the directories are Git
|
||||
worktrees.
|
||||
2. Use clean worktrees where possible. The report records `dirty: true` when
|
||||
tracked or untracked changes are present.
|
||||
tracked or untracked changes are present, and the Markdown report displays a
|
||||
visible warning when either the tool or subject worktree is dirty.
|
||||
3. Keep the operating system, Node.js version, machine, concurrency, and
|
||||
`.understandignore` rules constant between compared runs.
|
||||
4. Run at least three times and retain every report. Treat the first run as a
|
||||
@@ -138,7 +154,7 @@ explicit.
|
||||
| Exit code | Meaning | Report behavior |
|
||||
| ---: | --- | --- |
|
||||
| `0` | Completed with status `ok` or `degraded` | JSON and Markdown are written |
|
||||
| `1` | A deterministic stage or integrity check failed | Partial reports are written when the output location is writable |
|
||||
| `1` | A deterministic stage, integrity check, or temporary-artifact cleanup failed | Partial reports are written when the output location is writable; cleanup failures appear in `secondaryErrors` without replacing the primary stage error |
|
||||
| `2` | Invalid CLI usage | No report is written |
|
||||
|
||||
`degraded` means the deterministic pipeline completed but reported warnings or
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
"required": [
|
||||
"schemaUrl",
|
||||
"schemaVersion",
|
||||
"pairId",
|
||||
"status",
|
||||
"mode",
|
||||
"run",
|
||||
@@ -20,6 +21,7 @@
|
||||
"determinism",
|
||||
"llm",
|
||||
"warnings",
|
||||
"secondaryErrors",
|
||||
"error"
|
||||
],
|
||||
"properties": {
|
||||
@@ -27,6 +29,10 @@
|
||||
"const": "https://raw.githubusercontent.com/Egonex-AI/Understand-Anything/main/docs/benchmarks/large-repo-report-1.0.0.schema.json"
|
||||
},
|
||||
"schemaVersion": { "const": "1.0.0" },
|
||||
"pairId": {
|
||||
"type": "string",
|
||||
"pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$"
|
||||
},
|
||||
"status": { "enum": ["ok", "degraded", "failed"] },
|
||||
"mode": { "const": "deterministic" },
|
||||
"run": { "$ref": "#/$defs/run" },
|
||||
@@ -58,6 +64,11 @@
|
||||
"type": "array",
|
||||
"items": { "$ref": "#/$defs/warning" }
|
||||
},
|
||||
"secondaryErrors": {
|
||||
"type": "array",
|
||||
"maxItems": 5,
|
||||
"items": { "$ref": "#/$defs/secondaryError" }
|
||||
},
|
||||
"error": { "type": ["string", "null"] }
|
||||
},
|
||||
"allOf": [
|
||||
@@ -93,6 +104,7 @@
|
||||
},
|
||||
"integrity": { "$ref": "#/$defs/successIntegrity" },
|
||||
"determinism": { "$ref": "#/$defs/determinism" },
|
||||
"secondaryErrors": { "type": "array", "maxItems": 0 },
|
||||
"error": { "type": "null" }
|
||||
}
|
||||
}
|
||||
@@ -129,6 +141,34 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"if": {
|
||||
"properties": { "status": { "const": "degraded" } },
|
||||
"required": ["status"]
|
||||
},
|
||||
"then": {
|
||||
"anyOf": [
|
||||
{
|
||||
"properties": {
|
||||
"warnings": { "type": "array", "minItems": 1 }
|
||||
},
|
||||
"required": ["warnings"]
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"integrity": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"filesSkipped": { "type": "integer", "minimum": 1 }
|
||||
},
|
||||
"required": ["filesSkipped"]
|
||||
}
|
||||
},
|
||||
"required": ["integrity"]
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"if": {
|
||||
"properties": { "status": { "const": "failed" } },
|
||||
@@ -438,7 +478,9 @@
|
||||
"warningCount",
|
||||
"warningMessages",
|
||||
"warningMessagesTruncated",
|
||||
"outputBytes"
|
||||
"outputBytes",
|
||||
"failureSamples",
|
||||
"failureSamplesTruncated"
|
||||
],
|
||||
"properties": {
|
||||
"status": { "$ref": "#/$defs/status" },
|
||||
@@ -449,11 +491,22 @@
|
||||
"warningCount": { "$ref": "#/$defs/count" },
|
||||
"warningMessages": { "$ref": "#/$defs/warningMessages" },
|
||||
"warningMessagesTruncated": { "type": "boolean" },
|
||||
"failureSamples": {
|
||||
"type": "array",
|
||||
"maxItems": 5,
|
||||
"items": { "$ref": "#/$defs/failureSample" }
|
||||
},
|
||||
"failureSamplesTruncated": { "type": "boolean" },
|
||||
"outputBytes": { "$ref": "#/$defs/count" },
|
||||
"batchesSucceeded": { "$ref": "#/$defs/count" },
|
||||
"batchesFailed": { "$ref": "#/$defs/count" },
|
||||
"filesAnalyzed": { "$ref": "#/$defs/count" },
|
||||
"filesSkipped": { "$ref": "#/$defs/count" },
|
||||
"structureSucceeded": { "$ref": "#/$defs/count" },
|
||||
"structureFailed": { "$ref": "#/$defs/count" },
|
||||
"callGraphSucceeded": { "$ref": "#/$defs/count" },
|
||||
"callGraphFailed": { "$ref": "#/$defs/count" },
|
||||
"callGraphSkipped": { "$ref": "#/$defs/count" },
|
||||
"entities": { "$ref": "#/$defs/entities" },
|
||||
"batchDurationMs": { "$ref": "#/$defs/distribution" }
|
||||
},
|
||||
@@ -469,6 +522,11 @@
|
||||
"batchesFailed",
|
||||
"filesAnalyzed",
|
||||
"filesSkipped",
|
||||
"structureSucceeded",
|
||||
"structureFailed",
|
||||
"callGraphSucceeded",
|
||||
"callGraphFailed",
|
||||
"callGraphSkipped",
|
||||
"entities",
|
||||
"batchDurationMs"
|
||||
]
|
||||
@@ -496,6 +554,8 @@
|
||||
"unexpectedBatchFiles",
|
||||
"missingImportTargets",
|
||||
"structureCoverage",
|
||||
"structureFailures",
|
||||
"callGraphFailures",
|
||||
"filesSkipped",
|
||||
"failedBatches",
|
||||
"missingStructurePaths",
|
||||
@@ -510,6 +570,8 @@
|
||||
"unexpectedBatchFiles": { "$ref": "#/$defs/count" },
|
||||
"missingImportTargets": { "$ref": "#/$defs/count" },
|
||||
"structureCoverage": { "type": "number", "minimum": 0, "maximum": 1 },
|
||||
"structureFailures": { "$ref": "#/$defs/count" },
|
||||
"callGraphFailures": { "$ref": "#/$defs/count" },
|
||||
"filesSkipped": { "$ref": "#/$defs/count" },
|
||||
"failedBatches": { "$ref": "#/$defs/count" },
|
||||
"missingStructurePaths": { "$ref": "#/$defs/count" },
|
||||
@@ -527,6 +589,8 @@
|
||||
"unexpectedBatchFiles": { "const": 0 },
|
||||
"missingImportTargets": { "const": 0 },
|
||||
"structureCoverage": { "const": 1 },
|
||||
"structureFailures": { "const": 0 },
|
||||
"callGraphFailures": { "const": 0 },
|
||||
"failedBatches": { "const": 0 },
|
||||
"missingStructurePaths": { "const": 0 },
|
||||
"duplicateStructurePaths": { "const": 0 },
|
||||
@@ -561,7 +625,7 @@
|
||||
"required": ["stage", "count", "messages", "truncated"],
|
||||
"properties": {
|
||||
"stage": { "enum": ["scan", "imports", "batching", "structure"] },
|
||||
"count": { "$ref": "#/$defs/count" },
|
||||
"count": { "type": "integer", "minimum": 1 },
|
||||
"messages": { "$ref": "#/$defs/warningMessages" },
|
||||
"truncated": { "type": "boolean" }
|
||||
}
|
||||
@@ -570,6 +634,24 @@
|
||||
"type": "array",
|
||||
"maxItems": 5,
|
||||
"items": { "type": "string" }
|
||||
},
|
||||
"failureSample": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["batchIndex", "message"],
|
||||
"properties": {
|
||||
"batchIndex": { "$ref": "#/$defs/count" },
|
||||
"message": { "type": "string", "minLength": 1, "maxLength": 4096 }
|
||||
}
|
||||
},
|
||||
"secondaryError": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["stage", "message"],
|
||||
"properties": {
|
||||
"stage": { "const": "cleanup" },
|
||||
"message": { "const": "Unable to remove temporary benchmark artifacts" }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,16 +16,30 @@ const resolvedScript = resolve(scriptPath);
|
||||
const startedAt = performance.now();
|
||||
const usageBefore = process.resourceUsage();
|
||||
let failure = null;
|
||||
const originalExit = process.exit;
|
||||
|
||||
class ImportedHelperExitError extends Error {
|
||||
constructor(code) {
|
||||
super(`Imported benchmark helper requested process exit ${code}`);
|
||||
this.name = 'ImportedHelperExitError';
|
||||
this.code = code;
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
// The deterministic helpers use process.argv to identify their CLI entry
|
||||
// point. Recreate the argv shape they receive when launched directly, then
|
||||
// import them in this process so resourceUsage() measures the real stage.
|
||||
process.argv = [process.execPath, resolvedScript, ...scriptArgs];
|
||||
process.exit = (code = 0) => {
|
||||
throw new ImportedHelperExitError(code);
|
||||
};
|
||||
await import(pathToFileURL(resolvedScript).href);
|
||||
} catch (error) {
|
||||
failure = error instanceof Error ? error : new Error(String(error));
|
||||
process.stderr.write(`${failure.stack ?? failure.message}\n`);
|
||||
} finally {
|
||||
process.exit = originalExit;
|
||||
}
|
||||
|
||||
const usageAfter = process.resourceUsage();
|
||||
@@ -37,5 +51,6 @@ const metrics = {
|
||||
systemCpuTimeMicros: usageAfter.systemCPUTime - usageBefore.systemCPUTime,
|
||||
};
|
||||
|
||||
process.stderr.write(`${STAGE_METRICS_PREFIX}${JSON.stringify(metrics)}\n`);
|
||||
// Start the marker on its own line even when a helper left stderr unterminated.
|
||||
process.stderr.write(`\n${STAGE_METRICS_PREFIX}${JSON.stringify(metrics)}\n`);
|
||||
if (failure) process.exitCode = 1;
|
||||
|
||||
@@ -65,6 +65,7 @@ export const GIT_METADATA_MAX_BUFFER = 64 * 1024;
|
||||
// from each stream while continuing to drain and inspect all child output.
|
||||
export const STAGE_OUTPUT_MAX_BYTES = 128 * 1024;
|
||||
export const WARNING_SAMPLE_LIMIT = 5;
|
||||
export const STRUCTURE_DIAGNOSTIC_SAMPLE_MAX_BYTES = 4 * 1024;
|
||||
const ENTITY_FIELDS = [
|
||||
'functions',
|
||||
'classes',
|
||||
@@ -93,10 +94,18 @@ export class BenchmarkStageError extends Error {
|
||||
}
|
||||
|
||||
export class BenchmarkReportWriteError extends Error {
|
||||
constructor(artifactRoot = null) {
|
||||
constructor(artifactRoot = null, recovery = {}) {
|
||||
super('Unable to write benchmark report files');
|
||||
this.name = 'BenchmarkReportWriteError';
|
||||
this.artifactRoot = artifactRoot;
|
||||
this.recovery = {
|
||||
lockAcquisitionFailed: recovery.lockAcquisitionFailed ?? false,
|
||||
rollbackRemoveFailures: recovery.rollbackRemoveFailures ?? 0,
|
||||
restoreFailures: recovery.restoreFailures ?? 0,
|
||||
tempCleanupFailures: recovery.tempCleanupFailures ?? 0,
|
||||
backupCleanupFailures: recovery.backupCleanupFailures ?? 0,
|
||||
lockReleaseFailures: recovery.lockReleaseFailures ?? 0,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -236,6 +245,9 @@ export function parseArgs(argv, cwd = process.cwd()) {
|
||||
|
||||
if (help) return { help: true };
|
||||
if (!repoValue) throw new CliUsageError('A repository path is required');
|
||||
if (outputValue === null || outputValue.trim() === '') {
|
||||
throw new CliUsageError('--output is required and must be non-empty');
|
||||
}
|
||||
if (!Number.isInteger(concurrency) || concurrency < 1 || concurrency > 32) {
|
||||
throw new CliUsageError('--concurrency must be an integer between 1 and 32');
|
||||
}
|
||||
@@ -248,10 +260,7 @@ export function parseArgs(argv, cwd = process.cwd()) {
|
||||
throw new CliUsageError(`Repository path is not a directory: ${repoValue}`);
|
||||
}
|
||||
|
||||
const outputPath = resolve(
|
||||
cwd,
|
||||
outputValue ?? join('benchmark-results', 'large-repo-report.json'),
|
||||
);
|
||||
const outputPath = resolve(cwd, outputValue);
|
||||
const markdownPath = resolve(
|
||||
cwd,
|
||||
outputPath.toLowerCase().endsWith('.json')
|
||||
@@ -278,11 +287,11 @@ export function parseArgs(argv, cwd = process.cwd()) {
|
||||
|
||||
export function helpText() {
|
||||
return `Usage:
|
||||
node scripts/benchmark-large-repo.mjs <repo-path> [options]
|
||||
node scripts/benchmark-large-repo.mjs --repo <repo-path> [options]
|
||||
node scripts/benchmark-large-repo.mjs <repo-path> --output <path> [options]
|
||||
node scripts/benchmark-large-repo.mjs --repo <repo-path> --output <path> [options]
|
||||
|
||||
Options:
|
||||
-o, --output <path> JSON report path
|
||||
-o, --output <path> JSON report path (required)
|
||||
--label <name> Public label for the subject repository
|
||||
--concurrency <1-32> Structural extraction workers (default: 5)
|
||||
--keep-artifacts Preserve temporary deterministic outputs
|
||||
@@ -397,7 +406,13 @@ function addRootAliases(aliases, root, replacement) {
|
||||
|
||||
function hasPathBoundary(text, index) {
|
||||
if (index >= text.length) return true;
|
||||
return text[index] === '/' || text[index] === '\\';
|
||||
return (
|
||||
text[index] === '/' ||
|
||||
text[index] === '\\' ||
|
||||
text[index] === '\n' ||
|
||||
text[index] === '\r' ||
|
||||
text[index] === '\t'
|
||||
);
|
||||
}
|
||||
|
||||
function replacePathAlias(text, alias, replacement, caseInsensitive) {
|
||||
@@ -675,6 +690,99 @@ function readJson(path) {
|
||||
return JSON.parse(readFileSync(path, 'utf-8'));
|
||||
}
|
||||
|
||||
function isRecord(value) {
|
||||
return value !== null && typeof value === 'object' && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function isNonNegativeInteger(value) {
|
||||
return Number.isInteger(value) && value >= 0;
|
||||
}
|
||||
|
||||
function isStringCountMap(value) {
|
||||
return (
|
||||
isRecord(value) && Object.values(value).every(isNonNegativeInteger)
|
||||
);
|
||||
}
|
||||
|
||||
function validateScanArtifact(scan) {
|
||||
if (
|
||||
!isRecord(scan) ||
|
||||
scan.scriptCompleted !== true ||
|
||||
!Array.isArray(scan.files) ||
|
||||
!isNonNegativeInteger(scan.totalFiles) ||
|
||||
scan.totalFiles !== scan.files.length ||
|
||||
!isNonNegativeInteger(scan.filteredByIgnore) ||
|
||||
!isRecord(scan.stats) ||
|
||||
!isNonNegativeInteger(scan.stats.filesScanned) ||
|
||||
scan.stats.filesScanned !== scan.totalFiles ||
|
||||
!isStringCountMap(scan.stats.byCategory) ||
|
||||
!isStringCountMap(scan.stats.byLanguage) ||
|
||||
typeof scan.contentDigest !== 'string' ||
|
||||
!/^[a-f0-9]{64}$/.test(scan.contentDigest) ||
|
||||
scan.files.some(
|
||||
(file) =>
|
||||
!isRecord(file) ||
|
||||
typeof file.path !== 'string' ||
|
||||
file.path.length === 0 ||
|
||||
!isNonNegativeInteger(file.sizeLines),
|
||||
)
|
||||
) {
|
||||
throw new Error('Scan artifact does not match the required shape');
|
||||
}
|
||||
}
|
||||
|
||||
function validateImportArtifact(imports) {
|
||||
if (
|
||||
!isRecord(imports) ||
|
||||
imports.scriptCompleted !== true ||
|
||||
!isRecord(imports.importMap) ||
|
||||
!isRecord(imports.stats) ||
|
||||
!isNonNegativeInteger(imports.stats.filesScanned) ||
|
||||
!isNonNegativeInteger(imports.stats.filesWithImports) ||
|
||||
!isNonNegativeInteger(imports.stats.totalEdges) ||
|
||||
Object.values(imports.importMap).some(
|
||||
(targets) =>
|
||||
!Array.isArray(targets) ||
|
||||
targets.some((target) => typeof target !== 'string'),
|
||||
)
|
||||
) {
|
||||
throw new Error('Import artifact does not match the required shape');
|
||||
}
|
||||
}
|
||||
|
||||
function validateBatchArtifact(batches) {
|
||||
if (
|
||||
!isRecord(batches) ||
|
||||
batches.schemaVersion !== 1 ||
|
||||
typeof batches.algorithm !== 'string' ||
|
||||
batches.algorithm.length === 0 ||
|
||||
!Array.isArray(batches.batches) ||
|
||||
!isNonNegativeInteger(batches.totalBatches) ||
|
||||
batches.totalBatches !== batches.batches.length ||
|
||||
batches.batches.some(
|
||||
(batch) =>
|
||||
!isRecord(batch) ||
|
||||
!isNonNegativeInteger(batch.batchIndex) ||
|
||||
!Array.isArray(batch.files) ||
|
||||
!isRecord(batch.batchImportData) ||
|
||||
!isRecord(batch.neighborMap),
|
||||
)
|
||||
) {
|
||||
throw new Error('Batch artifact does not match the required shape');
|
||||
}
|
||||
}
|
||||
|
||||
function markArtifactFailure(stage, stageName, error, redactionRoots) {
|
||||
const rawMessage = error instanceof Error ? error.message : String(error);
|
||||
const safeMessage = sanitizeBounded(
|
||||
`${stageName} stage produced an invalid artifact: ${rawMessage}`,
|
||||
redactionRoots,
|
||||
);
|
||||
stage.status = 'failed';
|
||||
stage.stderr = safeMessage.text;
|
||||
stage.stderrTruncated ||= safeMessage.truncated;
|
||||
}
|
||||
|
||||
function fileSizeOrZero(path) {
|
||||
try {
|
||||
return statSync(path).size;
|
||||
@@ -696,27 +804,58 @@ function preflightReportTargets(paths, fileSystem) {
|
||||
}
|
||||
}
|
||||
|
||||
function bestEffortRemove(path, fileSystem) {
|
||||
function tryRemove(path, fileSystem) {
|
||||
try {
|
||||
fileSystem.rmSync(path, { force: true });
|
||||
return true;
|
||||
} catch {
|
||||
// Transaction cleanup must never mask the primary delivery failure.
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function rollbackReportEntry(entry, fileSystem) {
|
||||
function rollbackReportEntry(entry, fileSystem, recovery) {
|
||||
if (entry.backupMoved) {
|
||||
bestEffortRemove(entry.targetPath, fileSystem);
|
||||
if (!tryRemove(entry.targetPath, fileSystem)) {
|
||||
recovery.rollbackRemoveFailures += 1;
|
||||
}
|
||||
try {
|
||||
fileSystem.renameSync(entry.backupPath, entry.targetPath);
|
||||
} catch {
|
||||
// Leave an unrestored backup in place rather than delete original data.
|
||||
recovery.restoreFailures += 1;
|
||||
}
|
||||
} else if (!entry.hadOriginal && entry.installAttempted) {
|
||||
bestEffortRemove(entry.targetPath, fileSystem);
|
||||
if (!tryRemove(entry.targetPath, fileSystem)) {
|
||||
recovery.rollbackRemoveFailures += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function reportPairLockPath(outputPath, markdownPath) {
|
||||
const outputDirectory = canonicalizePhysicalPath(dirname(outputPath));
|
||||
const markdownDirectory = canonicalizePhysicalPath(dirname(markdownPath));
|
||||
const normalizePairPath = (pathValue) =>
|
||||
process.platform === 'win32' ? pathValue.toLowerCase() : pathValue;
|
||||
if (
|
||||
normalizePairPath(outputDirectory) !==
|
||||
normalizePairPath(markdownDirectory)
|
||||
) {
|
||||
throw new Error('Benchmark report files must share a directory');
|
||||
}
|
||||
const normalizedOutputPath = normalizePairPath(
|
||||
canonicalizePhysicalPath(outputPath),
|
||||
);
|
||||
const normalizedMarkdownPath = normalizePairPath(
|
||||
canonicalizePhysicalPath(markdownPath),
|
||||
);
|
||||
const pairKey = createHash('sha256')
|
||||
.update(normalizedOutputPath)
|
||||
.update('\0')
|
||||
.update(normalizedMarkdownPath)
|
||||
.digest('hex')
|
||||
.slice(0, 24);
|
||||
return join(resolve(dirname(outputPath)), `.ua-report-pair-${pairKey}.lock`);
|
||||
}
|
||||
|
||||
function stageReportEntry(entry, fileSystem) {
|
||||
let descriptor;
|
||||
let stageError = null;
|
||||
@@ -752,6 +891,17 @@ export function deliverBenchmarkReports(reportFiles, operations = {}) {
|
||||
writeFileSync: operations.writeFileSync ?? writeFileSync,
|
||||
};
|
||||
const transactionId = randomUUID();
|
||||
const recovery = {
|
||||
lockAcquisitionFailed: false,
|
||||
rollbackRemoveFailures: 0,
|
||||
restoreFailures: 0,
|
||||
tempCleanupFailures: 0,
|
||||
backupCleanupFailures: 0,
|
||||
lockReleaseFailures: 0,
|
||||
};
|
||||
let deliveryFailed = false;
|
||||
let lockOwned = false;
|
||||
let lockPath = null;
|
||||
const entries = [
|
||||
[reportFiles.outputPath, reportFiles.jsonContents],
|
||||
[reportFiles.markdownPath, reportFiles.markdownContents],
|
||||
@@ -776,6 +926,19 @@ export function deliverBenchmarkReports(reportFiles, operations = {}) {
|
||||
for (const entry of entries) {
|
||||
fileSystem.mkdirSync(dirname(entry.targetPath), { recursive: true });
|
||||
}
|
||||
lockPath = reportPairLockPath(
|
||||
reportFiles.outputPath,
|
||||
reportFiles.markdownPath,
|
||||
);
|
||||
let lockDescriptor;
|
||||
try {
|
||||
lockDescriptor = fileSystem.openSync(lockPath, 'wx');
|
||||
lockOwned = true;
|
||||
fileSystem.closeSync(lockDescriptor);
|
||||
} catch (error) {
|
||||
if (!lockOwned) recovery.lockAcquisitionFailed = true;
|
||||
throw error;
|
||||
}
|
||||
preflightReportTargets(
|
||||
entries.map((entry) => entry.targetPath),
|
||||
fileSystem,
|
||||
@@ -802,20 +965,36 @@ export function deliverBenchmarkReports(reportFiles, operations = {}) {
|
||||
}
|
||||
for (const entry of entries) {
|
||||
if (entry.backupMoved) {
|
||||
bestEffortRemove(entry.backupPath, fileSystem);
|
||||
if (!tryRemove(entry.backupPath, fileSystem)) {
|
||||
recovery.backupCleanupFailures += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
deliveryFailed = true;
|
||||
for (const entry of [...entries].reverse()) {
|
||||
rollbackReportEntry(entry, fileSystem);
|
||||
rollbackReportEntry(entry, fileSystem, recovery);
|
||||
}
|
||||
throw new BenchmarkReportWriteError();
|
||||
} finally {
|
||||
for (const entry of entries) {
|
||||
if (entry.tempCreated) {
|
||||
bestEffortRemove(entry.tempPath, fileSystem);
|
||||
if (!tryRemove(entry.tempPath, fileSystem)) {
|
||||
recovery.tempCleanupFailures += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (lockOwned && !tryRemove(lockPath, fileSystem)) {
|
||||
recovery.lockReleaseFailures += 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (
|
||||
deliveryFailed ||
|
||||
recovery.tempCleanupFailures > 0 ||
|
||||
recovery.backupCleanupFailures > 0 ||
|
||||
recovery.lockReleaseFailures > 0
|
||||
) {
|
||||
throw new BenchmarkReportWriteError(null, recovery);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -863,9 +1042,12 @@ export function renderMarkdownReport(report) {
|
||||
'| Metric | Value |',
|
||||
'| --- | --- |',
|
||||
metricRow('Status', report.status),
|
||||
metricRow('Pair ID', report.pairId),
|
||||
metricRow('Subject', report.subject.label),
|
||||
metricRow('Subject commit', report.subject.commit),
|
||||
metricRow('Subject dirty', report.subject.dirty),
|
||||
metricRow('Tool commit', report.tool.commit),
|
||||
metricRow('Tool dirty', report.tool.dirty),
|
||||
metricRow('Tool version', report.tool.packageVersion),
|
||||
metricRow('Started (UTC)', report.run.startedAt),
|
||||
metricRow('Total duration', duration(report.run.durationMs)),
|
||||
@@ -917,12 +1099,29 @@ export function renderMarkdownReport(report) {
|
||||
metricRow('Memory (bytes)', report.environment.totalMemoryBytes),
|
||||
];
|
||||
|
||||
if (report.subject.dirty === true || report.tool.dirty === true) {
|
||||
lines.push(
|
||||
'',
|
||||
'> **Warning:** The subject or tool worktree was dirty; commit hashes alone do not reproduce this run.',
|
||||
);
|
||||
}
|
||||
|
||||
if (report.warnings.length > 0) {
|
||||
lines.push('', '## Warnings', '');
|
||||
for (const warning of report.warnings) {
|
||||
lines.push(`- ${markdownValue(warning.stage)}: ${warning.count}`);
|
||||
}
|
||||
}
|
||||
if ((report.secondaryErrors?.length ?? 0) > 0) {
|
||||
lines.push('', '## Secondary errors', '');
|
||||
for (const secondaryError of report.secondaryErrors) {
|
||||
lines.push(
|
||||
`- ${markdownValue(secondaryError.stage)}: ${markdownValue(
|
||||
secondaryError.message,
|
||||
)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
if (report.error) {
|
||||
lines.push('', '## Error', '', markdownValue(report.error));
|
||||
}
|
||||
@@ -1033,11 +1232,21 @@ export function aggregateStructureSummaries(structureSummaries) {
|
||||
const aggregate = {
|
||||
filesAnalyzed: 0,
|
||||
filesSkipped: 0,
|
||||
structureSucceeded: 0,
|
||||
structureFailed: 0,
|
||||
callGraphSucceeded: 0,
|
||||
callGraphFailed: 0,
|
||||
callGraphSkipped: 0,
|
||||
entities: emptyEntityCounts(),
|
||||
};
|
||||
for (const summary of structureSummaries) {
|
||||
aggregate.filesAnalyzed += summary.filesAnalyzed;
|
||||
aggregate.filesSkipped += summary.filesSkipped;
|
||||
aggregate.structureSucceeded += summary.structureSucceeded;
|
||||
aggregate.structureFailed += summary.structureFailed;
|
||||
aggregate.callGraphSucceeded += summary.callGraphSucceeded;
|
||||
aggregate.callGraphFailed += summary.callGraphFailed;
|
||||
aggregate.callGraphSkipped += summary.callGraphSkipped;
|
||||
for (const field of ENTITY_FIELDS) {
|
||||
aggregate.entities[field] += summary.entities[field];
|
||||
}
|
||||
@@ -1051,6 +1260,24 @@ export function summarizeStructureOutput(batch, output) {
|
||||
const skippedPaths = Array.isArray(output?.filesSkipped)
|
||||
? output.filesSkipped
|
||||
: [];
|
||||
const outcomeCounts = output?.analysisOutcomes;
|
||||
const hasOutcomeCounts = outcomeCounts !== undefined;
|
||||
const isCount = (value) => Number.isInteger(value) && value >= 0;
|
||||
const structureSucceeded = hasOutcomeCounts
|
||||
? outcomeCounts?.structure?.succeeded
|
||||
: results.length;
|
||||
const structureFailed = hasOutcomeCounts
|
||||
? outcomeCounts?.structure?.failed
|
||||
: 0;
|
||||
const callGraphSucceeded = hasOutcomeCounts
|
||||
? outcomeCounts?.callGraph?.succeeded
|
||||
: 0;
|
||||
const callGraphFailed = hasOutcomeCounts
|
||||
? outcomeCounts?.callGraph?.failed
|
||||
: 0;
|
||||
const callGraphSkipped = hasOutcomeCounts
|
||||
? outcomeCounts?.callGraph?.skipped
|
||||
: results.length;
|
||||
let malformed =
|
||||
!output ||
|
||||
typeof output !== 'object' ||
|
||||
@@ -1059,7 +1286,14 @@ export function summarizeStructureOutput(batch, output) {
|
||||
!Array.isArray(output.results) ||
|
||||
!Array.isArray(output.filesSkipped) ||
|
||||
!Number.isInteger(output.filesAnalyzed) ||
|
||||
output.filesAnalyzed !== results.length;
|
||||
output.filesAnalyzed !== results.length ||
|
||||
!isCount(structureSucceeded) ||
|
||||
!isCount(structureFailed) ||
|
||||
!isCount(callGraphSucceeded) ||
|
||||
!isCount(callGraphFailed) ||
|
||||
!isCount(callGraphSkipped) ||
|
||||
structureSucceeded + structureFailed !== results.length ||
|
||||
callGraphSucceeded + callGraphFailed + callGraphSkipped !== results.length;
|
||||
|
||||
const pathCounts = new Map();
|
||||
const entities = emptyEntityCounts();
|
||||
@@ -1101,6 +1335,8 @@ export function summarizeStructureOutput(batch, output) {
|
||||
digest: canonicalSha256(output),
|
||||
complete:
|
||||
!malformed &&
|
||||
structureFailed === 0 &&
|
||||
callGraphFailed === 0 &&
|
||||
missingStructurePaths === 0 &&
|
||||
duplicateStructurePaths === 0 &&
|
||||
unexpectedStructurePaths === 0,
|
||||
@@ -1109,6 +1345,11 @@ export function summarizeStructureOutput(batch, output) {
|
||||
accountedExpectedPaths,
|
||||
filesAnalyzed: results.length,
|
||||
filesSkipped: skippedPaths.length,
|
||||
structureSucceeded: isCount(structureSucceeded) ? structureSucceeded : 0,
|
||||
structureFailed: isCount(structureFailed) ? structureFailed : 0,
|
||||
callGraphSucceeded: isCount(callGraphSucceeded) ? callGraphSucceeded : 0,
|
||||
callGraphFailed: isCount(callGraphFailed) ? callGraphFailed : 0,
|
||||
callGraphSkipped: isCount(callGraphSkipped) ? callGraphSkipped : 0,
|
||||
missingStructurePaths,
|
||||
duplicateStructurePaths,
|
||||
unexpectedStructurePaths,
|
||||
@@ -1203,8 +1444,16 @@ export function buildBenchmarkIntegrity(
|
||||
}
|
||||
}
|
||||
|
||||
const accountedExpectedPaths = structureSummaries.reduce(
|
||||
(sum, summary) => sum + summary.accountedExpectedPaths,
|
||||
const structureSucceeded = structureSummaries.reduce(
|
||||
(sum, summary) => sum + summary.structureSucceeded,
|
||||
0,
|
||||
);
|
||||
const structureFailures = structureSummaries.reduce(
|
||||
(sum, summary) => sum + summary.structureFailed,
|
||||
0,
|
||||
);
|
||||
const callGraphFailures = structureSummaries.reduce(
|
||||
(sum, summary) => sum + summary.callGraphFailed,
|
||||
0,
|
||||
);
|
||||
const filesSkipped = structureSummaries.reduce(
|
||||
@@ -1236,9 +1485,14 @@ export function buildBenchmarkIntegrity(
|
||||
unexpectedBatchFiles: unexpectedBatchFiles.length,
|
||||
missingImportTargets,
|
||||
structureCoverage:
|
||||
scan.totalFiles === 0
|
||||
structureSucceeded + structureFailures === 0
|
||||
? 1
|
||||
: Math.round((accountedExpectedPaths / scan.totalFiles) * 10000) / 10000,
|
||||
: Math.round(
|
||||
(structureSucceeded / (structureSucceeded + structureFailures)) *
|
||||
10000,
|
||||
) / 10000,
|
||||
structureFailures,
|
||||
callGraphFailures,
|
||||
filesSkipped,
|
||||
failedBatches,
|
||||
missingStructurePaths,
|
||||
@@ -1253,6 +1507,8 @@ export function hasFailedIntegrity(integrity) {
|
||||
!integrity.allScannedFilesBatched ||
|
||||
integrity.missingImportTargets > 0 ||
|
||||
integrity.structureCoverage !== 1 ||
|
||||
integrity.structureFailures > 0 ||
|
||||
integrity.callGraphFailures > 0 ||
|
||||
integrity.failedBatches > 0 ||
|
||||
integrity.missingStructurePaths > 0 ||
|
||||
integrity.duplicateStructurePaths > 0 ||
|
||||
@@ -1292,6 +1548,112 @@ export function aggregateStageWarnings(stages) {
|
||||
};
|
||||
}
|
||||
|
||||
export function createStructureDiagnosticsAccumulator() {
|
||||
const warningCandidates = [];
|
||||
const failureCandidates = [];
|
||||
let warningCount = 0;
|
||||
let warningMessagesTruncated = false;
|
||||
let failureSamplesTruncated = false;
|
||||
|
||||
function retainCandidate(candidates, candidate) {
|
||||
candidates.push(candidate);
|
||||
candidates.sort(
|
||||
(left, right) =>
|
||||
left.inputIndex - right.inputIndex ||
|
||||
left.sampleIndex - right.sampleIndex,
|
||||
);
|
||||
if (candidates.length > WARNING_SAMPLE_LIMIT) {
|
||||
candidates.pop();
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function retainFailure(inputIndex, batchIndex, message, sampleIndex = 0) {
|
||||
const safeMessage = boundUtf8(
|
||||
`[batch ${batchIndex}] ${message}`,
|
||||
STRUCTURE_DIAGNOSTIC_SAMPLE_MAX_BYTES,
|
||||
);
|
||||
const omitted = retainCandidate(failureCandidates, {
|
||||
inputIndex,
|
||||
sampleIndex,
|
||||
batchIndex,
|
||||
message: safeMessage.text,
|
||||
});
|
||||
failureSamplesTruncated ||=
|
||||
safeMessage.truncated || omitted;
|
||||
}
|
||||
|
||||
return {
|
||||
record(inputIndex, batchIndex, stage) {
|
||||
const stageMessages = stage.warningMessages ?? [];
|
||||
const stageWarningCount = stage.warningCount ?? 0;
|
||||
warningCount += stageWarningCount;
|
||||
for (let sampleIndex = 0; sampleIndex < stageMessages.length; sampleIndex += 1) {
|
||||
const safeMessage = boundUtf8(
|
||||
`[batch ${batchIndex}] ${stageMessages[sampleIndex]}`,
|
||||
STRUCTURE_DIAGNOSTIC_SAMPLE_MAX_BYTES,
|
||||
);
|
||||
const omitted = retainCandidate(warningCandidates, {
|
||||
inputIndex,
|
||||
sampleIndex,
|
||||
message: safeMessage.text,
|
||||
});
|
||||
warningMessagesTruncated ||=
|
||||
safeMessage.truncated || omitted;
|
||||
}
|
||||
warningMessagesTruncated ||=
|
||||
stage.warningMessagesTruncated ||
|
||||
stageWarningCount > stageMessages.length;
|
||||
|
||||
if (stage.status === 'failed') {
|
||||
const message = stage.stderr?.trim()
|
||||
? stage.stderr.trim()
|
||||
: `Structure worker failed with exit code ${stage.exitCode ?? 'unknown'}`;
|
||||
retainFailure(inputIndex, batchIndex, message);
|
||||
}
|
||||
|
||||
return {
|
||||
name: stage.name,
|
||||
status: stage.status,
|
||||
exitCode: stage.exitCode,
|
||||
durationMs: stage.durationMs,
|
||||
peakRssBytes: stage.peakRssBytes,
|
||||
userCpuTimeMicros: stage.userCpuTimeMicros,
|
||||
systemCpuTimeMicros: stage.systemCpuTimeMicros,
|
||||
warningCount: stageWarningCount,
|
||||
warningMessagesTruncated:
|
||||
stage.warningMessagesTruncated ||
|
||||
stageWarningCount > stageMessages.length,
|
||||
stdoutTruncated: stage.stdoutTruncated,
|
||||
stderrTruncated: stage.stderrTruncated,
|
||||
};
|
||||
},
|
||||
recordAnalysisOutcome(inputIndex, batchIndex, summary) {
|
||||
if (summary.structureFailed > 0 || summary.callGraphFailed > 0) {
|
||||
retainFailure(
|
||||
inputIndex,
|
||||
batchIndex,
|
||||
`Analysis outcomes: ${summary.structureFailed} structure failure(s), ${summary.callGraphFailed} call-graph failure(s)`,
|
||||
1,
|
||||
);
|
||||
}
|
||||
},
|
||||
summary() {
|
||||
return {
|
||||
warningCount,
|
||||
warningMessages: warningCandidates.map((candidate) => candidate.message),
|
||||
warningMessagesTruncated,
|
||||
failureSamples: failureCandidates.map(({ batchIndex, message }) => ({
|
||||
batchIndex,
|
||||
message,
|
||||
})),
|
||||
failureSamplesTruncated,
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function warningSummary(stageResults) {
|
||||
return stageResults
|
||||
.filter((stage) => stage.warningCount > 0)
|
||||
@@ -1316,6 +1678,7 @@ export async function runBenchmark(options, hooks = {}) {
|
||||
[artifactRoot, '<artifacts>'],
|
||||
];
|
||||
const onProgress = hooks.onProgress ?? (() => {});
|
||||
const runStage = hooks.runStage ?? runNodeStage;
|
||||
const toolGit = gitMetadata(REPO_ROOT);
|
||||
const subjectGit = gitMetadata(options.repoRoot);
|
||||
const stageResults = [];
|
||||
@@ -1323,6 +1686,7 @@ export async function runBenchmark(options, hooks = {}) {
|
||||
const report = {
|
||||
schemaUrl: REPORT_SCHEMA_URL,
|
||||
schemaVersion: REPORT_SCHEMA_VERSION,
|
||||
pairId: randomUUID(),
|
||||
status: 'failed',
|
||||
mode: 'deterministic',
|
||||
run: {
|
||||
@@ -1356,6 +1720,7 @@ export async function runBenchmark(options, hooks = {}) {
|
||||
costUsd: null,
|
||||
},
|
||||
warnings: [],
|
||||
secondaryErrors: [],
|
||||
error: null,
|
||||
};
|
||||
|
||||
@@ -1363,7 +1728,7 @@ export async function runBenchmark(options, hooks = {}) {
|
||||
try {
|
||||
const scanPath = join(artifactRoot, 'scan-result.json');
|
||||
onProgress('scan');
|
||||
const scanStage = await runNodeStage(
|
||||
const scanStage = await runStage(
|
||||
'scan',
|
||||
SCAN_SCRIPT,
|
||||
[options.repoRoot, scanPath, '--exclude-analysis-data'],
|
||||
@@ -1374,8 +1739,20 @@ export async function runBenchmark(options, hooks = {}) {
|
||||
scanStage,
|
||||
fileSizeOrZero(scanPath),
|
||||
);
|
||||
let scan = null;
|
||||
if (scanStage.status === 'ok') {
|
||||
try {
|
||||
scan = readJson(scanPath);
|
||||
validateScanArtifact(scan);
|
||||
} catch (error) {
|
||||
markArtifactFailure(scanStage, 'scan', error, redactionRoots);
|
||||
report.stages.scan = summarizeStage(
|
||||
scanStage,
|
||||
fileSizeOrZero(scanPath),
|
||||
);
|
||||
}
|
||||
}
|
||||
if (scanStage.status === 'failed') throw new BenchmarkStageError(scanStage);
|
||||
const scan = readJson(scanPath);
|
||||
report.scale = subjectScale(options.repoRoot, scan);
|
||||
report.stages.scan = summarizeStage(scanStage, fileSizeOrZero(scanPath), {
|
||||
files: scan.totalFiles,
|
||||
@@ -1391,7 +1768,7 @@ export async function runBenchmark(options, hooks = {}) {
|
||||
files: scan.files,
|
||||
});
|
||||
onProgress('imports');
|
||||
const importStage = await runNodeStage(
|
||||
const importStage = await runStage(
|
||||
'imports',
|
||||
IMPORT_SCRIPT,
|
||||
[importInputPath, importOutputPath],
|
||||
@@ -1402,8 +1779,20 @@ export async function runBenchmark(options, hooks = {}) {
|
||||
importStage,
|
||||
fileSizeOrZero(importOutputPath),
|
||||
);
|
||||
let imports = null;
|
||||
if (importStage.status === 'ok') {
|
||||
try {
|
||||
imports = readJson(importOutputPath);
|
||||
validateImportArtifact(imports);
|
||||
} catch (error) {
|
||||
markArtifactFailure(importStage, 'imports', error, redactionRoots);
|
||||
report.stages.imports = summarizeStage(
|
||||
importStage,
|
||||
fileSizeOrZero(importOutputPath),
|
||||
);
|
||||
}
|
||||
}
|
||||
if (importStage.status === 'failed') throw new BenchmarkStageError(importStage);
|
||||
const imports = readJson(importOutputPath);
|
||||
report.stages.imports = summarizeStage(
|
||||
importStage,
|
||||
fileSizeOrZero(importOutputPath),
|
||||
@@ -1419,7 +1808,7 @@ export async function runBenchmark(options, hooks = {}) {
|
||||
const enrichedScan = { ...scan, importMap: imports.importMap };
|
||||
writeJson(enrichedScanPath, enrichedScan);
|
||||
onProgress('batching');
|
||||
const batchStage = await runNodeStage(
|
||||
const batchStage = await runStage(
|
||||
'batching',
|
||||
BATCH_SCRIPT,
|
||||
[
|
||||
@@ -1434,8 +1823,20 @@ export async function runBenchmark(options, hooks = {}) {
|
||||
batchStage,
|
||||
fileSizeOrZero(batchesPath),
|
||||
);
|
||||
let batches = null;
|
||||
if (batchStage.status === 'ok') {
|
||||
try {
|
||||
batches = readJson(batchesPath);
|
||||
validateBatchArtifact(batches);
|
||||
} catch (error) {
|
||||
markArtifactFailure(batchStage, 'batching', error, redactionRoots);
|
||||
report.stages.batching = summarizeStage(
|
||||
batchStage,
|
||||
fileSizeOrZero(batchesPath),
|
||||
);
|
||||
}
|
||||
}
|
||||
if (batchStage.status === 'failed') throw new BenchmarkStageError(batchStage);
|
||||
const batches = readJson(batchesPath);
|
||||
const batchSizes = batches.batches.map((batch) => batch.files.length);
|
||||
const estimatedAgentInputBytes = batches.batches.reduce(
|
||||
(sum, batch) =>
|
||||
@@ -1463,10 +1864,11 @@ export async function runBenchmark(options, hooks = {}) {
|
||||
onProgress('structure');
|
||||
const structureStart = performance.now();
|
||||
let completedBatches = 0;
|
||||
const structureDiagnostics = createStructureDiagnosticsAccumulator();
|
||||
const structureRuns = await mapWithConcurrency(
|
||||
batches.batches,
|
||||
options.concurrency,
|
||||
async (batch) => {
|
||||
async (batch, inputIndex) => {
|
||||
const inputPath = join(artifactRoot, `structure-input-${batch.batchIndex}.json`);
|
||||
const outputPath = join(artifactRoot, `structure-output-${batch.batchIndex}.json`);
|
||||
writeJson(inputPath, {
|
||||
@@ -1474,7 +1876,7 @@ export async function runBenchmark(options, hooks = {}) {
|
||||
batchFiles: batch.files,
|
||||
batchImportData: batch.batchImportData,
|
||||
});
|
||||
const stage = await runNodeStage(
|
||||
const stage = await runStage(
|
||||
`structure:${batch.batchIndex}`,
|
||||
STRUCTURE_SCRIPT,
|
||||
[inputPath, outputPath],
|
||||
@@ -1487,6 +1889,11 @@ export async function runBenchmark(options, hooks = {}) {
|
||||
try {
|
||||
const output = readJson(outputPath);
|
||||
summary = summarizeStructureOutput(batch, output);
|
||||
structureDiagnostics.recordAnalysisOutcome(
|
||||
inputIndex,
|
||||
batch.batchIndex,
|
||||
summary,
|
||||
);
|
||||
} catch (error) {
|
||||
stage.status = 'failed';
|
||||
stage.stderr = redactPaths(
|
||||
@@ -1495,8 +1902,13 @@ export async function runBenchmark(options, hooks = {}) {
|
||||
);
|
||||
}
|
||||
}
|
||||
return {
|
||||
const compactStage = structureDiagnostics.record(
|
||||
inputIndex,
|
||||
batch.batchIndex,
|
||||
stage,
|
||||
);
|
||||
return {
|
||||
stage: compactStage,
|
||||
summary,
|
||||
outputBytes: fileSizeOrZero(outputPath),
|
||||
};
|
||||
@@ -1508,13 +1920,17 @@ export async function runBenchmark(options, hooks = {}) {
|
||||
.map((run) => run.summary);
|
||||
const structureAggregate = aggregateStructureSummaries(structureSummaries);
|
||||
const failedBatches = structureRuns.filter(
|
||||
(run) => run.stage.status === 'failed',
|
||||
(run) => run.stage.status === 'failed' || run.summary?.complete === false,
|
||||
).length;
|
||||
const structureDurationMs =
|
||||
Math.round((performance.now() - structureStart) * 100) / 100;
|
||||
const structureWarnings = aggregateStageWarnings(
|
||||
structureRuns.map((run) => run.stage),
|
||||
);
|
||||
const structureDiagnosticsSummary = structureDiagnostics.summary();
|
||||
const structureWarnings = {
|
||||
warningCount: structureDiagnosticsSummary.warningCount,
|
||||
warningMessages: structureDiagnosticsSummary.warningMessages,
|
||||
warningMessagesTruncated:
|
||||
structureDiagnosticsSummary.warningMessagesTruncated,
|
||||
};
|
||||
const structureResources = aggregateStructureResources(
|
||||
structureRuns.map((run) => run.stage),
|
||||
);
|
||||
@@ -1535,11 +1951,19 @@ export async function runBenchmark(options, hooks = {}) {
|
||||
userCpuTimeMicros: structureResources.userCpuTimeMicros,
|
||||
systemCpuTimeMicros: structureResources.systemCpuTimeMicros,
|
||||
...structureWarnings,
|
||||
failureSamples: structureDiagnosticsSummary.failureSamples,
|
||||
failureSamplesTruncated:
|
||||
structureDiagnosticsSummary.failureSamplesTruncated,
|
||||
outputBytes: structureRuns.reduce((sum, run) => sum + run.outputBytes, 0),
|
||||
batchesSucceeded: structureRuns.length - failedBatches,
|
||||
batchesFailed: failedBatches,
|
||||
filesAnalyzed: structureAggregate.filesAnalyzed,
|
||||
filesSkipped: structureAggregate.filesSkipped,
|
||||
structureSucceeded: structureAggregate.structureSucceeded,
|
||||
structureFailed: structureAggregate.structureFailed,
|
||||
callGraphSucceeded: structureAggregate.callGraphSucceeded,
|
||||
callGraphFailed: structureAggregate.callGraphFailed,
|
||||
callGraphSkipped: structureAggregate.callGraphSkipped,
|
||||
entities: structureAggregate.entities,
|
||||
batchDurationMs: distribution(
|
||||
structureRuns.map((run) => run.stage.durationMs),
|
||||
@@ -1596,7 +2020,20 @@ export async function runBenchmark(options, hooks = {}) {
|
||||
}
|
||||
|
||||
if (!options.keepArtifacts) {
|
||||
cleanupBenchmarkArtifacts(artifactRoot);
|
||||
try {
|
||||
const cleanupArtifacts =
|
||||
hooks.cleanupArtifacts ?? cleanupBenchmarkArtifacts;
|
||||
cleanupArtifacts(artifactRoot);
|
||||
} catch {
|
||||
const cleanupMessage = 'Unable to remove temporary benchmark artifacts';
|
||||
report.secondaryErrors.push({
|
||||
stage: 'cleanup',
|
||||
message: cleanupMessage,
|
||||
});
|
||||
report.error ??= cleanupMessage;
|
||||
report.status = 'failed';
|
||||
exitCode = 1;
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
@@ -1606,9 +2043,10 @@ export async function runBenchmark(options, hooks = {}) {
|
||||
jsonContents: `${JSON.stringify(report, null, 2)}\n`,
|
||||
markdownContents: renderMarkdownReport(report),
|
||||
});
|
||||
} catch {
|
||||
} catch (error) {
|
||||
throw new BenchmarkReportWriteError(
|
||||
options.keepArtifacts ? artifactRoot : null,
|
||||
error instanceof BenchmarkReportWriteError ? error.recovery : {},
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@ import {
|
||||
writeFileSync,
|
||||
} from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { dirname, join, resolve } from 'node:path';
|
||||
import { basename, dirname, join, resolve } from 'node:path';
|
||||
import { fileURLToPath, pathToFileURL } from 'node:url';
|
||||
import Ajv2020 from 'ajv/dist/2020.js';
|
||||
import { afterEach, describe, expect, it } from 'vitest';
|
||||
@@ -254,33 +254,25 @@ throw new Error(\`helper failed below \${subject}/failed.txt\`);
|
||||
expect(retained).not.toContain(artifacts);
|
||||
});
|
||||
|
||||
it('parses a final metrics marker without a newline', async () => {
|
||||
it('keeps unterminated helper stderr separate from worker metrics', async () => {
|
||||
const root = mkdtempSync(join(tmpdir(), 'ua benchmark eof-metrics-'));
|
||||
cleanup.push(root);
|
||||
const helperPath = join(root, 'eof-metrics-helper.mjs');
|
||||
const metrics = {
|
||||
peakRssBytes: 123456,
|
||||
userCpuTimeMicros: 234,
|
||||
systemCpuTimeMicros: 345,
|
||||
};
|
||||
writeFileSync(
|
||||
helperPath,
|
||||
`
|
||||
import { writeSync } from 'node:fs';
|
||||
writeSync(2, ${JSON.stringify(
|
||||
`${benchmark.STAGE_METRICS_PREFIX}${JSON.stringify(metrics)}`,
|
||||
)});
|
||||
process.exit(0);
|
||||
writeSync(2, 'ordinary final stderr');
|
||||
`,
|
||||
);
|
||||
|
||||
const stage = await benchmark.runNodeStage('eof-metrics', helperPath, []);
|
||||
|
||||
expect(stage.status).toBe('ok');
|
||||
expect(stage.peakRssBytes).toBe(metrics.peakRssBytes);
|
||||
expect(stage.userCpuTimeMicros).toBe(metrics.userCpuTimeMicros);
|
||||
expect(stage.systemCpuTimeMicros).toBe(metrics.systemCpuTimeMicros);
|
||||
expect(stage.stderr).toBe('');
|
||||
expect(stage.peakRssBytes).toBeGreaterThan(0);
|
||||
expect(stage.userCpuTimeMicros).toBeGreaterThanOrEqual(0);
|
||||
expect(stage.systemCpuTimeMicros).toBeGreaterThanOrEqual(0);
|
||||
expect(stage.stderr).toBe('ordinary final stderr');
|
||||
expect(stage.warningCount).toBe(0);
|
||||
});
|
||||
|
||||
@@ -295,7 +287,6 @@ process.exit(0);
|
||||
`
|
||||
import { writeSync } from 'node:fs';
|
||||
writeSync(2, \`Warning: final \${process.argv[2]}/file.ts\`);
|
||||
process.exit(0);
|
||||
`,
|
||||
);
|
||||
|
||||
@@ -314,7 +305,25 @@ process.exit(0);
|
||||
expect(stage.warningMessagesTruncated).toBe(false);
|
||||
expect(stage.stderr).toBe('Warning: final <subject>/file.ts');
|
||||
expect(stage.stderr.endsWith('\n')).toBe(false);
|
||||
expect(stage.peakRssBytes).toBeNull();
|
||||
expect(stage.peakRssBytes).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('captures resource metrics before failing an imported helper process.exit', async () => {
|
||||
const root = mkdtempSync(join(tmpdir(), 'ua benchmark helper-exit-'));
|
||||
cleanup.push(root);
|
||||
const helperPath = join(root, 'exit-helper.mjs');
|
||||
writeFileSync(
|
||||
helperPath,
|
||||
`process.stderr.write('helper requested exit\\n');\nprocess.exit(7);\n`,
|
||||
);
|
||||
|
||||
const stage = await benchmark.runNodeStage('helper-exit', helperPath, []);
|
||||
|
||||
expect(stage.status).toBe('failed');
|
||||
expect(stage.exitCode).not.toBe(0);
|
||||
expect(stage.peakRssBytes).toBeGreaterThan(0);
|
||||
expect(stage.userCpuTimeMicros).toBeGreaterThanOrEqual(0);
|
||||
expect(stage.systemCpuTimeMicros).toBeGreaterThanOrEqual(0);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -364,6 +373,59 @@ describe('benchmark warning aggregation', () => {
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it('compacts completed structure workers into a globally bounded deterministic sample', () => {
|
||||
expect(benchmark.createStructureDiagnosticsAccumulator).toBeTypeOf(
|
||||
'function',
|
||||
);
|
||||
const diagnostics = benchmark.createStructureDiagnosticsAccumulator();
|
||||
const compactStages = [];
|
||||
for (let inputIndex = 999; inputIndex >= 0; inputIndex -= 1) {
|
||||
compactStages.push(
|
||||
diagnostics.record(inputIndex, inputIndex + 10, {
|
||||
name: `structure:${inputIndex + 10}`,
|
||||
status: inputIndex % 2 === 0 ? 'failed' : 'ok',
|
||||
exitCode: inputIndex % 2 === 0 ? 1 : 0,
|
||||
durationMs: 1,
|
||||
peakRssBytes: 1024,
|
||||
userCpuTimeMicros: 1,
|
||||
systemCpuTimeMicros: 1,
|
||||
warningCount: 1,
|
||||
warningMessages: [`Warning: batch ${inputIndex} ${'w'.repeat(200_000)}`],
|
||||
warningMessagesTruncated: false,
|
||||
stdout: 'o'.repeat(200_000),
|
||||
stderr: `failure ${inputIndex} ${'e'.repeat(200_000)}`,
|
||||
stdoutTruncated: true,
|
||||
stderrTruncated: true,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
expect(
|
||||
compactStages.every(
|
||||
(stage) =>
|
||||
!Object.hasOwn(stage, 'stdout') &&
|
||||
!Object.hasOwn(stage, 'stderr') &&
|
||||
!Object.hasOwn(stage, 'warningMessages'),
|
||||
),
|
||||
).toBe(true);
|
||||
const summary = diagnostics.summary();
|
||||
expect(summary.warningMessages).toHaveLength(benchmark.WARNING_SAMPLE_LIMIT);
|
||||
expect(summary.warningMessages[0]).toContain('batch 0');
|
||||
expect(summary.failureSamples).toHaveLength(benchmark.WARNING_SAMPLE_LIMIT);
|
||||
expect(summary.failureSamples.map((sample) => sample.batchIndex)).toEqual([
|
||||
10,
|
||||
12,
|
||||
14,
|
||||
16,
|
||||
18,
|
||||
]);
|
||||
expect(summary.warningMessagesTruncated).toBe(true);
|
||||
expect(summary.failureSamplesTruncated).toBe(true);
|
||||
expect(Buffer.byteLength(JSON.stringify(summary))).toBeLessThanOrEqual(
|
||||
64 * 1024,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('canonical benchmark digests', () => {
|
||||
@@ -434,6 +496,11 @@ describe('structure output summaries', () => {
|
||||
accountedExpectedPaths: 3,
|
||||
filesAnalyzed: 2,
|
||||
filesSkipped: 1,
|
||||
structureSucceeded: 2,
|
||||
structureFailed: 0,
|
||||
callGraphSucceeded: 0,
|
||||
callGraphFailed: 0,
|
||||
callGraphSkipped: 2,
|
||||
missingStructurePaths: 0,
|
||||
duplicateStructurePaths: 0,
|
||||
unexpectedStructurePaths: 0,
|
||||
@@ -484,6 +551,33 @@ describe('structure output summaries', () => {
|
||||
});
|
||||
});
|
||||
|
||||
it('retains explicit parser outcomes instead of treating every returned path as success', () => {
|
||||
const output = {
|
||||
scriptCompleted: true,
|
||||
filesAnalyzed: 3,
|
||||
filesSkipped: [],
|
||||
analysisOutcomes: {
|
||||
structure: { succeeded: 2, failed: 1 },
|
||||
callGraph: { succeeded: 1, failed: 1, skipped: 1 },
|
||||
},
|
||||
results: [
|
||||
{ path: 'src/a.ts' },
|
||||
{ path: 'src/b.ts' },
|
||||
{ path: 'src/c.ts' },
|
||||
],
|
||||
};
|
||||
|
||||
expect(benchmark.summarizeStructureOutput(batch, output)).toMatchObject({
|
||||
complete: false,
|
||||
malformed: false,
|
||||
structureSucceeded: 2,
|
||||
structureFailed: 1,
|
||||
callGraphSucceeded: 1,
|
||||
callGraphFailed: 1,
|
||||
callGraphSkipped: 1,
|
||||
});
|
||||
});
|
||||
|
||||
it('aggregates file and entity counts from compact summaries', () => {
|
||||
expect(benchmark.aggregateStructureSummaries).toBeTypeOf('function');
|
||||
const first = benchmark.summarizeStructureOutput(batch, {
|
||||
@@ -509,6 +603,11 @@ describe('structure output summaries', () => {
|
||||
expect(benchmark.aggregateStructureSummaries([first, second])).toEqual({
|
||||
filesAnalyzed: 3,
|
||||
filesSkipped: 1,
|
||||
structureSucceeded: 3,
|
||||
structureFailed: 0,
|
||||
callGraphSucceeded: 0,
|
||||
callGraphFailed: 0,
|
||||
callGraphSkipped: 3,
|
||||
entities: {
|
||||
functions: 3,
|
||||
classes: 1,
|
||||
@@ -558,9 +657,10 @@ describe('structure resource aggregation', () => {
|
||||
expect(benchmark.renderMarkdownReport).toBeTypeOf('function');
|
||||
const markdown = benchmark.renderMarkdownReport({
|
||||
schemaVersion: '1.0.0',
|
||||
pairId: '11111111-1111-4111-8111-111111111111',
|
||||
status: 'ok',
|
||||
subject: { label: 'fixture', commit: null },
|
||||
tool: { commit: null, packageVersion: '0.0.0' },
|
||||
subject: { label: 'fixture', commit: null, dirty: true },
|
||||
tool: { commit: null, dirty: true, packageVersion: '0.0.0' },
|
||||
run: { startedAt: '2026-01-01T00:00:00.000Z', durationMs: 12 },
|
||||
configuration: { concurrency: 2 },
|
||||
llm: { invoked: false },
|
||||
@@ -594,6 +694,12 @@ describe('structure resource aggregation', () => {
|
||||
'| Stage | Status | Duration | Peak / max worker RSS (bytes) | User CPU (micros) | System CPU (micros) | Output (bytes) |',
|
||||
);
|
||||
expect(markdown).toContain('| structure | ok | 12 ms | 250 | 50 | 10 | 99 |');
|
||||
expect(markdown).toContain(
|
||||
'| Pair ID | 11111111-1111-4111-8111-111111111111 |',
|
||||
);
|
||||
expect(markdown).toContain('| Tool dirty | true |');
|
||||
expect(markdown).toContain('| Subject dirty | true |');
|
||||
expect(markdown).toMatch(/warning.*dirty/i);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -662,7 +768,7 @@ describe('benchmark integrity aggregation', () => {
|
||||
);
|
||||
|
||||
expect(integrity).toMatchObject({
|
||||
structureCoverage: 0.6667,
|
||||
structureCoverage: 1,
|
||||
missingStructurePaths: 1,
|
||||
duplicateStructurePaths: 1,
|
||||
unexpectedStructurePaths: 1,
|
||||
@@ -671,6 +777,37 @@ describe('benchmark integrity aggregation', () => {
|
||||
expect(benchmark.hasFailedIntegrity(integrity)).toBe(true);
|
||||
});
|
||||
|
||||
it('fails integrity when explicit structure or call-graph outcomes fail', () => {
|
||||
const summary = benchmark.summarizeStructureOutput(batch, {
|
||||
scriptCompleted: true,
|
||||
filesAnalyzed: 3,
|
||||
filesSkipped: [],
|
||||
analysisOutcomes: {
|
||||
structure: { succeeded: 2, failed: 1 },
|
||||
callGraph: { succeeded: 1, failed: 1, skipped: 1 },
|
||||
},
|
||||
results: [
|
||||
{ path: 'src/a.ts' },
|
||||
{ path: 'src/b.ts' },
|
||||
{ path: 'src/c.ts' },
|
||||
],
|
||||
});
|
||||
const integrity = benchmark.buildBenchmarkIntegrity(
|
||||
scan,
|
||||
{},
|
||||
batches,
|
||||
[summary],
|
||||
0,
|
||||
);
|
||||
|
||||
expect(integrity).toMatchObject({
|
||||
structureCoverage: 0.6667,
|
||||
structureFailures: 1,
|
||||
callGraphFailures: 1,
|
||||
});
|
||||
expect(benchmark.hasFailedIntegrity(integrity)).toBe(true);
|
||||
});
|
||||
|
||||
it('rejects malformed output even when the expected path set is empty', () => {
|
||||
const emptyScan = { totalFiles: 0, files: [] };
|
||||
const emptyBatch = { batchIndex: 1, files: [] };
|
||||
@@ -950,20 +1087,39 @@ describe('large repository benchmark CLI', () => {
|
||||
cleanup.push(root);
|
||||
|
||||
const options = parseArgs(
|
||||
[subject, '--concurrency', '3', '--label', 'polyglot-mini'],
|
||||
[
|
||||
subject,
|
||||
'--output',
|
||||
join(root, 'report.json'),
|
||||
'--concurrency',
|
||||
'3',
|
||||
'--label',
|
||||
'polyglot-mini',
|
||||
],
|
||||
root,
|
||||
);
|
||||
expect(options.repoRoot).toBe(subject);
|
||||
expect(options.concurrency).toBe(3);
|
||||
expect(options.label).toBe('polyglot-mini');
|
||||
expect(options.markdownPath).toBe(
|
||||
join(root, 'benchmark-results', 'large-repo-report.md'),
|
||||
);
|
||||
expect(options.markdownPath).toBe(join(root, 'report.md'));
|
||||
expect(() => parseArgs([subject, '--concurrency', '0'], root)).toThrow(
|
||||
CliUsageError,
|
||||
);
|
||||
});
|
||||
|
||||
it('requires a non-empty explicit output path and reports usage exit 2', () => {
|
||||
const { root, subject } = makeSubject();
|
||||
cleanup.push(root);
|
||||
|
||||
expect(() => parseArgs([subject], root)).toThrow(/--output/);
|
||||
expect(() => parseArgs([subject, '--output='], root)).toThrow(/--output/);
|
||||
const result = runCli([subject]);
|
||||
expect(result.status).toBe(2);
|
||||
expect(result.stdout).toBe('');
|
||||
expect(result.stderr).toContain('--output <path>');
|
||||
expect(result.stderr).toMatch(/required/i);
|
||||
});
|
||||
|
||||
it.each(['2.5', '3junk', '1e1', ''])(
|
||||
'rejects malformed concurrency %j in split and equals forms',
|
||||
(rawConcurrency) => {
|
||||
@@ -1089,9 +1245,9 @@ describe('large repository benchmark CLI', () => {
|
||||
warningMessagesTruncated: false,
|
||||
});
|
||||
expect(result.report.stages.scan.durationMs).toBeGreaterThanOrEqual(0);
|
||||
expect(result.report.stages.scan).toHaveProperty('peakRssBytes');
|
||||
expect(result.report.stages.scan).toHaveProperty('userCpuTimeMicros');
|
||||
expect(result.report.stages.scan).toHaveProperty('systemCpuTimeMicros');
|
||||
expect(result.report.stages.scan.peakRssBytes).toBeGreaterThan(0);
|
||||
expect(result.report.stages.scan.userCpuTimeMicros).toBeGreaterThanOrEqual(0);
|
||||
expect(result.report.stages.scan.systemCpuTimeMicros).toBeGreaterThanOrEqual(0);
|
||||
expect(existsSync(outputPath)).toBe(true);
|
||||
expect(existsSync(markdownPath)).toBe(true);
|
||||
const serialized = readFileSync(outputPath, 'utf-8');
|
||||
@@ -1102,6 +1258,183 @@ describe('large repository benchmark CLI', () => {
|
||||
expect(readFileSync(markdownPath, 'utf-8')).not.toContain(subject);
|
||||
});
|
||||
|
||||
it.each([
|
||||
['scan', 'missing'],
|
||||
['imports', 'malformed'],
|
||||
['batching', 'wrong-shape'],
|
||||
])(
|
||||
'marks an exit-zero %s stage failed for a %s artifact and emits schema-valid partial telemetry',
|
||||
async (targetStage, corruption) => {
|
||||
const { root, subject } = makeSubject();
|
||||
cleanup.push(root);
|
||||
const outputPath = join(root, 'reports', `${targetStage}.json`);
|
||||
const markdownPath = join(root, 'reports', `${targetStage}.md`);
|
||||
|
||||
const result = await benchmark.runBenchmark(
|
||||
{
|
||||
repoRoot: subject,
|
||||
outputPath,
|
||||
markdownPath,
|
||||
label: `corrupt-${targetStage}`,
|
||||
concurrency: 1,
|
||||
keepArtifacts: false,
|
||||
},
|
||||
{
|
||||
async runStage(name, scriptPath, args, redactionRoots) {
|
||||
const stage = await benchmark.runNodeStage(
|
||||
name,
|
||||
scriptPath,
|
||||
args,
|
||||
redactionRoots,
|
||||
);
|
||||
if (name === targetStage && stage.status === 'ok') {
|
||||
const artifactPath =
|
||||
name === 'batching'
|
||||
? args.find((arg) => arg.startsWith('--output=')).slice(
|
||||
'--output='.length,
|
||||
)
|
||||
: args[1];
|
||||
if (corruption === 'missing') {
|
||||
rmSync(artifactPath, { force: true });
|
||||
} else if (corruption === 'malformed') {
|
||||
writeFileSync(artifactPath, '{ definitely not JSON', 'utf-8');
|
||||
} else {
|
||||
writeFileSync(artifactPath, '{}\n', 'utf-8');
|
||||
}
|
||||
}
|
||||
return stage;
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
expect(result.exitCode).toBe(1);
|
||||
expect(result.report.status).toBe('failed');
|
||||
expect(result.report.stages[targetStage].status).toBe('failed');
|
||||
expect(result.report.error).toBeTruthy();
|
||||
expectValidReport(result.report);
|
||||
expectValidReport(JSON.parse(readFileSync(outputPath, 'utf-8')));
|
||||
expect(existsSync(markdownPath)).toBe(true);
|
||||
},
|
||||
70_000,
|
||||
);
|
||||
|
||||
it.each([
|
||||
[
|
||||
'string file.sizeLines',
|
||||
(scan, subject) => {
|
||||
scan.files[0].sizeLines = subject;
|
||||
},
|
||||
],
|
||||
[
|
||||
'negative file.sizeLines',
|
||||
(scan) => {
|
||||
scan.files[0].sizeLines = -1;
|
||||
},
|
||||
],
|
||||
[
|
||||
'string filteredByIgnore',
|
||||
(scan, subject) => {
|
||||
scan.filteredByIgnore = subject;
|
||||
},
|
||||
],
|
||||
[
|
||||
'negative filteredByIgnore',
|
||||
(scan) => {
|
||||
scan.filteredByIgnore = -1;
|
||||
},
|
||||
],
|
||||
[
|
||||
'non-record stats',
|
||||
(scan) => {
|
||||
scan.stats = null;
|
||||
},
|
||||
],
|
||||
[
|
||||
'non-record stats.byCategory',
|
||||
(scan) => {
|
||||
scan.stats.byCategory = [];
|
||||
},
|
||||
],
|
||||
[
|
||||
'non-count stats.byCategory value',
|
||||
(scan, subject) => {
|
||||
scan.stats.byCategory = { code: subject };
|
||||
},
|
||||
],
|
||||
[
|
||||
'non-record stats.byLanguage',
|
||||
(scan) => {
|
||||
scan.stats.byLanguage = [];
|
||||
},
|
||||
],
|
||||
[
|
||||
'negative stats.byLanguage value',
|
||||
(scan) => {
|
||||
scan.stats.byLanguage = { TypeScript: -1 };
|
||||
},
|
||||
],
|
||||
[
|
||||
'stats.filesScanned inconsistent with totalFiles',
|
||||
(scan) => {
|
||||
scan.stats.filesScanned = scan.totalFiles + 1;
|
||||
},
|
||||
],
|
||||
])(
|
||||
'rejects an exit-zero scan artifact with %s before copying nested values into the report',
|
||||
async (_description, corruptScan) => {
|
||||
const { root, subject } = makeSubject();
|
||||
cleanup.push(root);
|
||||
const outputPath = join(root, 'reports', 'nested-scan.json');
|
||||
const markdownPath = join(root, 'reports', 'nested-scan.md');
|
||||
const stagesStarted = [];
|
||||
|
||||
const result = await benchmark.runBenchmark(
|
||||
{
|
||||
repoRoot: subject,
|
||||
outputPath,
|
||||
markdownPath,
|
||||
label: 'nested-corrupt-scan',
|
||||
concurrency: 1,
|
||||
keepArtifacts: false,
|
||||
},
|
||||
{
|
||||
async runStage(name, scriptPath, args, redactionRoots) {
|
||||
stagesStarted.push(name);
|
||||
if (name !== 'scan') {
|
||||
throw new Error('nested scan corruption reached a later stage');
|
||||
}
|
||||
const stage = await benchmark.runNodeStage(
|
||||
name,
|
||||
scriptPath,
|
||||
args,
|
||||
redactionRoots,
|
||||
);
|
||||
expect(stage.status).toBe('ok');
|
||||
const artifactPath = args[1];
|
||||
const scan = JSON.parse(readFileSync(artifactPath, 'utf-8'));
|
||||
corruptScan(scan, subject);
|
||||
writeFileSync(artifactPath, `${JSON.stringify(scan, null, 2)}\n`, 'utf-8');
|
||||
return stage;
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
expect(stagesStarted).toEqual(['scan']);
|
||||
expect(result.exitCode).toBe(1);
|
||||
expect(result.report.status).toBe('failed');
|
||||
expect(result.report.scale).toBeNull();
|
||||
expect(result.report.stages.scan.status).toBe('failed');
|
||||
expect(result.report.stages.scan).not.toHaveProperty('files');
|
||||
expectValidReport(result.report);
|
||||
const serialized = readFileSync(outputPath, 'utf-8');
|
||||
const persisted = JSON.parse(serialized);
|
||||
expectValidReport(persisted);
|
||||
expect(serialized).not.toContain(subject);
|
||||
expect(readFileSync(markdownPath, 'utf-8')).not.toContain(subject);
|
||||
},
|
||||
70_000,
|
||||
);
|
||||
|
||||
it('wraps cleanup operation failures without exposing artifact paths', () => {
|
||||
const artifactRoot = join(tmpdir(), 'ua-large-bench-private-cleanup');
|
||||
let cleanupError;
|
||||
@@ -1127,6 +1460,91 @@ describe('large repository benchmark CLI', () => {
|
||||
expect(cleanupError.message).not.toContain('EPERM');
|
||||
});
|
||||
|
||||
it('preserves a primary stage failure, records cleanup as secondary, and still delivers both reports', async () => {
|
||||
const root = mkdtempSync(join(tmpdir(), 'ua stage cleanup failure-'));
|
||||
cleanup.push(root);
|
||||
const subject = join(root, 'not-a-directory.txt');
|
||||
const outputPath = join(root, 'reports', 'failed.json');
|
||||
const markdownPath = join(root, 'reports', 'failed.md');
|
||||
writeFileSync(subject, 'not a directory\n');
|
||||
let artifactRoot;
|
||||
|
||||
const result = await benchmark.runBenchmark(
|
||||
{
|
||||
repoRoot: subject,
|
||||
outputPath,
|
||||
markdownPath,
|
||||
label: 'stage-and-cleanup-failure',
|
||||
concurrency: 1,
|
||||
keepArtifacts: false,
|
||||
},
|
||||
{
|
||||
cleanupArtifacts(path) {
|
||||
artifactRoot = path;
|
||||
throw new benchmark.BenchmarkArtifactCleanupError();
|
||||
},
|
||||
},
|
||||
);
|
||||
if (artifactRoot) cleanup.push(artifactRoot);
|
||||
|
||||
expect(result.exitCode).toBe(1);
|
||||
expect(result.report.status).toBe('failed');
|
||||
expect(result.report.error).not.toBe(
|
||||
'Unable to remove temporary benchmark artifacts',
|
||||
);
|
||||
expect(result.report.secondaryErrors).toEqual([
|
||||
{
|
||||
stage: 'cleanup',
|
||||
message: 'Unable to remove temporary benchmark artifacts',
|
||||
},
|
||||
]);
|
||||
expect(result.artifactRoot).toBeNull();
|
||||
expectValidReport(result.report);
|
||||
expect(JSON.parse(readFileSync(outputPath, 'utf-8')).error).toBe(
|
||||
result.report.error,
|
||||
);
|
||||
const markdown = readFileSync(markdownPath, 'utf-8');
|
||||
expect(markdown).toContain(result.report.error.split(/\r?\n/, 1)[0]);
|
||||
expect(markdown).toContain('Unable to remove temporary benchmark artifacts');
|
||||
expect(JSON.stringify(result.report)).not.toContain(artifactRoot);
|
||||
});
|
||||
|
||||
it('turns an otherwise successful run into exit 1 when artifact cleanup fails', async () => {
|
||||
const { root, subject } = makeSubject();
|
||||
cleanup.push(root);
|
||||
const outputPath = join(root, 'reports', 'cleanup-failed.json');
|
||||
const markdownPath = join(root, 'reports', 'cleanup-failed.md');
|
||||
let artifactRoot;
|
||||
|
||||
const result = await benchmark.runBenchmark(
|
||||
{
|
||||
repoRoot: subject,
|
||||
outputPath,
|
||||
markdownPath,
|
||||
label: 'cleanup-failure',
|
||||
concurrency: 1,
|
||||
keepArtifacts: false,
|
||||
},
|
||||
{
|
||||
cleanupArtifacts(path) {
|
||||
artifactRoot = path;
|
||||
throw new benchmark.BenchmarkArtifactCleanupError();
|
||||
},
|
||||
},
|
||||
);
|
||||
if (artifactRoot) cleanup.push(artifactRoot);
|
||||
|
||||
expect(result.exitCode).toBe(1);
|
||||
expect(result.report.status).toBe('failed');
|
||||
expect(result.report.error).toBe(
|
||||
'Unable to remove temporary benchmark artifacts',
|
||||
);
|
||||
expect(result.report.secondaryErrors).toHaveLength(1);
|
||||
expectValidReport(result.report);
|
||||
expect(existsSync(outputPath)).toBe(true);
|
||||
expect(existsSync(markdownPath)).toBe(true);
|
||||
}, 70_000);
|
||||
|
||||
it('rolls back both reports when the second report commit fails', () => {
|
||||
const root = mkdtempSync(join(tmpdir(), 'ua report transaction-'));
|
||||
cleanup.push(root);
|
||||
@@ -1179,6 +1597,146 @@ describe('large repository benchmark CLI', () => {
|
||||
expect(readdirSync(reportsDirectory).sort()).toEqual(entriesBefore);
|
||||
});
|
||||
|
||||
it('excludes a concurrent writer while the shared pair lock is held', () => {
|
||||
const root = mkdtempSync(join(tmpdir(), 'ua report pair lock-'));
|
||||
cleanup.push(root);
|
||||
const outputPath = join(root, 'result.json');
|
||||
const markdownPath = join(root, 'result.md');
|
||||
expect(benchmark.reportPairLockPath).toBeTypeOf('function');
|
||||
const lockPath = benchmark.reportPairLockPath(outputPath, markdownPath);
|
||||
writeFileSync(lockPath, 'held by another writer\n', { flag: 'wx' });
|
||||
let deliveryError;
|
||||
|
||||
try {
|
||||
benchmark.deliverBenchmarkReports({
|
||||
outputPath,
|
||||
markdownPath,
|
||||
jsonContents: 'new json\n',
|
||||
markdownContents: 'new markdown\n',
|
||||
});
|
||||
} catch (error) {
|
||||
deliveryError = error;
|
||||
}
|
||||
|
||||
expect(deliveryError).toBeInstanceOf(benchmark.BenchmarkReportWriteError);
|
||||
expect(deliveryError.recovery).toMatchObject({
|
||||
lockAcquisitionFailed: true,
|
||||
});
|
||||
expect(existsSync(outputPath)).toBe(false);
|
||||
expect(existsSync(markdownPath)).toBe(false);
|
||||
expect(JSON.stringify(deliveryError.recovery)).not.toContain(root);
|
||||
});
|
||||
|
||||
it.runIf(process.platform === 'win32')(
|
||||
'uses one pair lock for Windows path-case aliases',
|
||||
() => {
|
||||
const root = mkdtempSync(join(tmpdir(), 'ua report case lock-'));
|
||||
cleanup.push(root);
|
||||
const outputPath = join(root, 'result.json');
|
||||
const markdownPath = join(root, 'result.md');
|
||||
const caseAliasRoot = root.toUpperCase();
|
||||
|
||||
expect(
|
||||
basename(
|
||||
benchmark.reportPairLockPath(
|
||||
join(caseAliasRoot, 'RESULT.JSON'),
|
||||
join(caseAliasRoot, 'RESULT.MD'),
|
||||
),
|
||||
),
|
||||
).toBe(
|
||||
basename(benchmark.reportPairLockPath(outputPath, markdownPath)),
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it('surfaces a rollback restore failure as bounded path-free recovery metadata', () => {
|
||||
const root = mkdtempSync(join(tmpdir(), 'ua report rollback recovery-'));
|
||||
cleanup.push(root);
|
||||
const outputPath = join(root, 'result.json');
|
||||
const markdownPath = join(root, 'result.md');
|
||||
writeFileSync(outputPath, 'old json\n');
|
||||
writeFileSync(markdownPath, 'old markdown\n');
|
||||
let deliveryError;
|
||||
|
||||
try {
|
||||
benchmark.deliverBenchmarkReports(
|
||||
{
|
||||
outputPath,
|
||||
markdownPath,
|
||||
jsonContents: 'new json\n',
|
||||
markdownContents: 'new markdown\n',
|
||||
},
|
||||
{
|
||||
renameSync(source, destination) {
|
||||
if (destination === markdownPath && source.endsWith('.tmp')) {
|
||||
throw new Error('injected second install failure');
|
||||
}
|
||||
if (destination === markdownPath && source.endsWith('.backup')) {
|
||||
throw new Error('injected restore failure');
|
||||
}
|
||||
renameSync(source, destination);
|
||||
},
|
||||
},
|
||||
);
|
||||
} catch (error) {
|
||||
deliveryError = error;
|
||||
}
|
||||
|
||||
expect(deliveryError).toBeInstanceOf(benchmark.BenchmarkReportWriteError);
|
||||
expect(deliveryError.recovery).toMatchObject({
|
||||
restoreFailures: 1,
|
||||
});
|
||||
expect(JSON.stringify(deliveryError.recovery)).not.toContain(root);
|
||||
expect(JSON.stringify(deliveryError.recovery)).not.toContain('injected');
|
||||
});
|
||||
|
||||
it('surfaces a rollback target-removal failure as path-free recovery metadata', () => {
|
||||
const root = mkdtempSync(join(tmpdir(), 'ua report rollback removal-'));
|
||||
cleanup.push(root);
|
||||
const outputPath = join(root, 'result.json');
|
||||
const markdownPath = join(root, 'result.md');
|
||||
writeFileSync(outputPath, 'old json\n');
|
||||
writeFileSync(markdownPath, 'old markdown\n');
|
||||
let removalFailed = false;
|
||||
let deliveryError;
|
||||
|
||||
try {
|
||||
benchmark.deliverBenchmarkReports(
|
||||
{
|
||||
outputPath,
|
||||
markdownPath,
|
||||
jsonContents: 'new json\n',
|
||||
markdownContents: 'new markdown\n',
|
||||
},
|
||||
{
|
||||
renameSync(source, destination) {
|
||||
if (destination === markdownPath && source.endsWith('.tmp')) {
|
||||
throw new Error('injected second install failure');
|
||||
}
|
||||
renameSync(source, destination);
|
||||
},
|
||||
rmSync(path, options) {
|
||||
if (path === markdownPath && !removalFailed) {
|
||||
removalFailed = true;
|
||||
throw new Error('injected rollback removal failure');
|
||||
}
|
||||
rmSync(path, options);
|
||||
},
|
||||
},
|
||||
);
|
||||
} catch (error) {
|
||||
deliveryError = error;
|
||||
}
|
||||
|
||||
expect(removalFailed).toBe(true);
|
||||
expect(deliveryError).toBeInstanceOf(benchmark.BenchmarkReportWriteError);
|
||||
expect(deliveryError.recovery).toMatchObject({
|
||||
rollbackRemoveFailures: 1,
|
||||
});
|
||||
expect(JSON.stringify(deliveryError.recovery)).not.toContain(root);
|
||||
expect(JSON.stringify(deliveryError.recovery)).not.toContain('injected');
|
||||
});
|
||||
|
||||
it('removes an owned partial temp when a staging write fails', () => {
|
||||
const root = mkdtempSync(join(tmpdir(), 'ua report partial write-'));
|
||||
cleanup.push(root);
|
||||
@@ -1365,6 +1923,9 @@ describe('large repository benchmark CLI', () => {
|
||||
expectValidReport(report);
|
||||
expect(report.schemaUrl).toBe(schema.$id);
|
||||
expect(report.schemaVersion).toBe('1.0.0');
|
||||
expect(report.pairId).toMatch(
|
||||
/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/,
|
||||
);
|
||||
expect(report.status).toBe('ok');
|
||||
expect(report.mode).toBe('deterministic');
|
||||
expect(report.subject).toEqual({
|
||||
@@ -1389,6 +1950,7 @@ describe('large repository benchmark CLI', () => {
|
||||
expect(report.determinism.outputDigest).toMatch(/^[a-f0-9]{64}$/);
|
||||
expect(JSON.stringify(report)).not.toContain(subject);
|
||||
expect(markdown).toContain('# Large Repository Benchmark Report');
|
||||
expect(markdown).toContain(`| Pair ID | ${report.pairId} |`);
|
||||
expect(markdown).toContain('| Files | 3 |');
|
||||
expect(markdown).toContain('| LLM invoked | No |');
|
||||
expect(markdown).toContain('Peak / max worker RSS (bytes)');
|
||||
@@ -1416,6 +1978,44 @@ describe('large repository benchmark CLI', () => {
|
||||
);
|
||||
}, 70_000);
|
||||
|
||||
it('keeps full Unicode benchmark digests stable across locale environments', () => {
|
||||
const { root, subject } = makeSubject();
|
||||
cleanup.push(root);
|
||||
const firstReportPath = join(root, 'locale-c.json');
|
||||
const secondReportPath = join(root, 'locale-sv.json');
|
||||
for (const name of ['Z', 'a', 'ä']) {
|
||||
writeFileSync(join(subject, 'src', `${name}.ts`), `export const ${
|
||||
name === 'ä' ? 'accented' : name
|
||||
} = 1;\n`);
|
||||
}
|
||||
writeFileSync(
|
||||
join(subject, 'src', 'index.ts'),
|
||||
[
|
||||
'import "./ä";',
|
||||
'import "./Z";',
|
||||
'import "./a";',
|
||||
'export const answer = 42;',
|
||||
'',
|
||||
].join('\n'),
|
||||
);
|
||||
|
||||
const runWithLocale = (locale, reportPath) =>
|
||||
runCli([subject, '--output', reportPath, '--concurrency', '2'], {
|
||||
env: { ...process.env, LANG: locale, LC_ALL: locale },
|
||||
});
|
||||
const firstResult = runWithLocale('C', firstReportPath);
|
||||
const secondResult = runWithLocale('sv_SE.UTF-8', secondReportPath);
|
||||
|
||||
expect(firstResult.status, firstResult.stderr).toBe(0);
|
||||
expect(secondResult.status, secondResult.stderr).toBe(0);
|
||||
const firstReport = JSON.parse(readFileSync(firstReportPath, 'utf-8'));
|
||||
const secondReport = JSON.parse(readFileSync(secondReportPath, 'utf-8'));
|
||||
expectValidReport(firstReport);
|
||||
expectValidReport(secondReport);
|
||||
expect(secondReport.determinism).toEqual(firstReport.determinism);
|
||||
expect(secondReport.stages.imports.edges).toBe(3);
|
||||
}, 70_000);
|
||||
|
||||
it('handles an empty repository as a valid deterministic run', () => {
|
||||
const root = mkdtempSync(join(tmpdir(), 'ua empty benchmark-'));
|
||||
cleanup.push(root);
|
||||
|
||||
@@ -47,6 +47,7 @@ function validReport() {
|
||||
return {
|
||||
schemaUrl: schema.$id,
|
||||
schemaVersion: '1.0.0',
|
||||
pairId: '11111111-1111-4111-8111-111111111111',
|
||||
status: 'ok',
|
||||
mode: 'deterministic',
|
||||
run: {
|
||||
@@ -113,11 +114,18 @@ function validReport() {
|
||||
warningCount: 0,
|
||||
warningMessages: [],
|
||||
warningMessagesTruncated: false,
|
||||
failureSamples: [],
|
||||
failureSamplesTruncated: false,
|
||||
outputBytes: 128,
|
||||
batchesSucceeded: 1,
|
||||
batchesFailed: 0,
|
||||
filesAnalyzed: 1,
|
||||
filesSkipped: 0,
|
||||
structureSucceeded: 1,
|
||||
structureFailed: 0,
|
||||
callGraphSucceeded: 1,
|
||||
callGraphFailed: 0,
|
||||
callGraphSkipped: 0,
|
||||
entities: {
|
||||
functions: 0,
|
||||
classes: 0,
|
||||
@@ -139,6 +147,8 @@ function validReport() {
|
||||
unexpectedBatchFiles: 0,
|
||||
missingImportTargets: 0,
|
||||
structureCoverage: 1,
|
||||
structureFailures: 0,
|
||||
callGraphFailures: 0,
|
||||
filesSkipped: 0,
|
||||
failedBatches: 0,
|
||||
missingStructurePaths: 0,
|
||||
@@ -158,6 +168,7 @@ function validReport() {
|
||||
costUsd: null,
|
||||
},
|
||||
warnings: [],
|
||||
secondaryErrors: [],
|
||||
error: null,
|
||||
};
|
||||
}
|
||||
@@ -169,6 +180,8 @@ describe('large repository report schema 1.0.0', () => {
|
||||
degraded.status = 'degraded';
|
||||
degraded.stages.structure.filesAnalyzed = 0;
|
||||
degraded.stages.structure.filesSkipped = 1;
|
||||
degraded.stages.structure.structureSucceeded = 0;
|
||||
degraded.stages.structure.callGraphSucceeded = 0;
|
||||
degraded.integrity.filesSkipped = 1;
|
||||
|
||||
expectValid(ok);
|
||||
@@ -260,4 +273,21 @@ describe('large repository report schema 1.0.0', () => {
|
||||
report.integrity[field] = value;
|
||||
expectInvalid(report);
|
||||
});
|
||||
|
||||
it('rejects degraded status without a warning or skipped-file reason', () => {
|
||||
const report = validReport();
|
||||
report.status = 'degraded';
|
||||
|
||||
expectInvalid(report);
|
||||
});
|
||||
|
||||
it('rejects warning summaries whose count is zero', () => {
|
||||
const report = validReport();
|
||||
report.status = 'degraded';
|
||||
report.warnings = [
|
||||
{ stage: 'scan', count: 0, messages: [], truncated: false },
|
||||
];
|
||||
|
||||
expectInvalid(report);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -30,14 +30,14 @@ function setupTree(files) {
|
||||
* `extraNodeArgs` is prepended to the node argv before the script path, so
|
||||
* tests can pass `--import` loader hooks to force specific failure modes.
|
||||
*/
|
||||
function runScript(projectRoot, input, extraNodeArgs = []) {
|
||||
function runScript(projectRoot, input, extraNodeArgs = [], env = process.env) {
|
||||
const inputPath = join(projectRoot, 'ua-eim-input.json');
|
||||
const outputPath = join(projectRoot, 'ua-eim-output.json');
|
||||
writeFileSync(inputPath, JSON.stringify(input), 'utf-8');
|
||||
const result = spawnSync(
|
||||
'node',
|
||||
[...extraNodeArgs, SCRIPT, inputPath, outputPath],
|
||||
{ encoding: 'utf-8' },
|
||||
{ encoding: 'utf-8', env },
|
||||
);
|
||||
let output = null;
|
||||
try {
|
||||
@@ -45,7 +45,13 @@ function runScript(projectRoot, input, extraNodeArgs = []) {
|
||||
} catch {
|
||||
/* output missing on hard failure */
|
||||
}
|
||||
return { status: result.status, stdout: result.stdout, stderr: result.stderr, output };
|
||||
return {
|
||||
status: result.status,
|
||||
stdout: result.stdout,
|
||||
stderr: result.stderr,
|
||||
output,
|
||||
outputText: output ? readFileSync(outputPath, 'utf-8') : null,
|
||||
};
|
||||
}
|
||||
|
||||
describe('extract-import-map.mjs — TypeScript / JavaScript resolver', () => {
|
||||
@@ -91,6 +97,44 @@ describe('extract-import-map.mjs — TypeScript / JavaScript resolver', () => {
|
||||
expect(result.output.stats.totalEdges).toBe(2);
|
||||
});
|
||||
|
||||
it('orders Unicode import targets by locale-independent UTF-16 code units', () => {
|
||||
projectRoot = setupTree({
|
||||
'src/index.ts': `import './ä';\nimport './Z';\nimport './a';\n`,
|
||||
'src/ä.ts': 'export const umlaut = true;\n',
|
||||
'src/Z.ts': 'export const upper = true;\n',
|
||||
'src/a.ts': 'export const lower = true;\n',
|
||||
});
|
||||
const input = {
|
||||
projectRoot,
|
||||
files: [
|
||||
{ path: 'src/ä.ts', language: 'typescript', fileCategory: 'code' },
|
||||
{ path: 'src/index.ts', language: 'typescript', fileCategory: 'code' },
|
||||
{ path: 'src/a.ts', language: 'typescript', fileCategory: 'code' },
|
||||
{ path: 'src/Z.ts', language: 'typescript', fileCategory: 'code' },
|
||||
],
|
||||
};
|
||||
|
||||
const cLocale = runScript(projectRoot, input, [], {
|
||||
...process.env,
|
||||
LANG: 'C',
|
||||
LC_ALL: 'C',
|
||||
});
|
||||
const swedishLocale = runScript(projectRoot, input, [], {
|
||||
...process.env,
|
||||
LANG: 'sv_SE.UTF-8',
|
||||
LC_ALL: 'sv_SE.UTF-8',
|
||||
});
|
||||
|
||||
expect(cLocale.status, cLocale.stderr).toBe(0);
|
||||
expect(swedishLocale.status, swedishLocale.stderr).toBe(0);
|
||||
expect(cLocale.output.importMap['src/index.ts']).toEqual([
|
||||
'src/Z.ts',
|
||||
'src/a.ts',
|
||||
'src/ä.ts',
|
||||
]);
|
||||
expect(swedishLocale.outputText).toBe(cLocale.outputText);
|
||||
});
|
||||
|
||||
it('resolves tsconfig paths aliases', () => {
|
||||
projectRoot = setupTree({
|
||||
'tsconfig.json': JSON.stringify({
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
import * as extractStructure from '../../../understand-anything-plugin/skills/understand/extract-structure-result.mjs';
|
||||
|
||||
describe('extract-structure analysis outcomes', () => {
|
||||
it('records final structure and call-graph exceptions after full-analysis fallback', () => {
|
||||
expect(extractStructure.analyzeFileWithOutcomes).toBeTypeOf('function');
|
||||
const registry = {
|
||||
analyzeFileFull() {
|
||||
throw new Error('combined parser failed');
|
||||
},
|
||||
analyzeFile() {
|
||||
throw new Error('structure parser failed');
|
||||
},
|
||||
extractCallGraph() {
|
||||
throw new Error('call graph failed');
|
||||
},
|
||||
};
|
||||
|
||||
expect(
|
||||
extractStructure.analyzeFileWithOutcomes(
|
||||
registry,
|
||||
{ path: 'src/failing.ts', fileCategory: 'code' },
|
||||
'export const value = 1;\n',
|
||||
),
|
||||
).toEqual({
|
||||
analysis: null,
|
||||
callGraph: null,
|
||||
structureOutcome: 'failed',
|
||||
callGraphOutcome: 'failed',
|
||||
});
|
||||
});
|
||||
|
||||
it('does not count missing parser return values as successful analysis', () => {
|
||||
const registry = {
|
||||
analyzeFileFull() {
|
||||
return undefined;
|
||||
},
|
||||
analyzeFile() {
|
||||
return undefined;
|
||||
},
|
||||
extractCallGraph() {
|
||||
return undefined;
|
||||
},
|
||||
};
|
||||
|
||||
expect(
|
||||
extractStructure.analyzeFileWithOutcomes(
|
||||
registry,
|
||||
{ path: 'src/missing.ts', fileCategory: 'code' },
|
||||
'export const value = 1;\n',
|
||||
),
|
||||
).toMatchObject({
|
||||
structureOutcome: 'failed',
|
||||
callGraphOutcome: 'failed',
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -96,6 +96,13 @@ function toPosix(p) {
|
||||
return p.split(/[\\/]/).filter(Boolean).join('/');
|
||||
}
|
||||
|
||||
// ECMAScript relational string comparison is lexicographic over UTF-16 code
|
||||
// units, so path ordering is stable across ICU versions, locales, and hosts.
|
||||
function comparePaths(a, b) {
|
||||
if (a === b) return 0;
|
||||
return a < b ? -1 : 1;
|
||||
}
|
||||
|
||||
/**
|
||||
* Join a directory with a relative segment, normalizing `.`/`..` segments and
|
||||
* returning a forward-slash POSIX path. Anchored at project root (no leading
|
||||
@@ -476,7 +483,7 @@ async function buildResolutionContext(projectRoot, files) {
|
||||
goFilesByDir.get(d).push(p);
|
||||
}
|
||||
for (const arr of goFilesByDir.values()) {
|
||||
arr.sort((a, b) => a.localeCompare(b));
|
||||
arr.sort(comparePaths);
|
||||
}
|
||||
|
||||
// Build per-extension suffix indices for dotted-FQN resolvers (Java,
|
||||
@@ -1022,7 +1029,7 @@ function buildSuffixIndex(files, extPredicate) {
|
||||
}
|
||||
// Deterministic order within each bucket
|
||||
for (const arr of idx.values()) {
|
||||
arr.sort((a, b) => a.localeCompare(b));
|
||||
arr.sort(comparePaths);
|
||||
}
|
||||
return idx;
|
||||
}
|
||||
@@ -1043,7 +1050,7 @@ function buildPackageIndex(files, extPredicate) {
|
||||
}
|
||||
}
|
||||
for (const arr of idx.values()) {
|
||||
arr.sort((a, b) => a.localeCompare(b));
|
||||
arr.sort(comparePaths);
|
||||
}
|
||||
return idx;
|
||||
}
|
||||
@@ -1100,7 +1107,7 @@ function buildSwiftModuleIndex(files, packageTargets) {
|
||||
const idx = new Map();
|
||||
const targetEntries = [...packageTargets.entries()].map(([name, paths]) => [
|
||||
name,
|
||||
[...paths].sort((a, b) => a.localeCompare(b)),
|
||||
[...paths].sort(comparePaths),
|
||||
]);
|
||||
|
||||
for (const f of files) {
|
||||
@@ -1123,7 +1130,7 @@ function buildSwiftModuleIndex(files, packageTargets) {
|
||||
|
||||
const out = new Map();
|
||||
for (const [moduleName, paths] of idx.entries()) {
|
||||
out.set(moduleName, [...paths].sort((a, b) => a.localeCompare(b)));
|
||||
out.set(moduleName, [...paths].sort(comparePaths));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
@@ -1199,7 +1206,7 @@ export function resolveScalaImport(rawImport, specifiers, _file, ctx) {
|
||||
|
||||
if (specs.includes('*')) {
|
||||
for (const m of resolveScalaPackage(rawImport, ctx)) out.add(m);
|
||||
return [...out].sort((a, b) => a.localeCompare(b));
|
||||
return [...out].sort(comparePaths);
|
||||
}
|
||||
|
||||
if (isPlain) {
|
||||
@@ -1208,7 +1215,7 @@ export function resolveScalaImport(rawImport, specifiers, _file, ctx) {
|
||||
const pkg = rawImport.slice(0, -(specs[0].length + 1));
|
||||
for (const m of resolveScalaDottedFqn(`${pkg}.package`, ctx)) out.add(m);
|
||||
}
|
||||
return [...out].sort((a, b) => a.localeCompare(b));
|
||||
return [...out].sort(comparePaths);
|
||||
}
|
||||
|
||||
let unresolvedSelector = false;
|
||||
@@ -1223,7 +1230,7 @@ export function resolveScalaImport(rawImport, specifiers, _file, ctx) {
|
||||
for (const m of resolveScalaDottedFqn(`${rawImport}.package`, ctx)) out.add(m);
|
||||
}
|
||||
|
||||
return [...out].sort((a, b) => a.localeCompare(b));
|
||||
return [...out].sort(comparePaths);
|
||||
}
|
||||
|
||||
function resolveScalaDottedFqn(fqn, ctx) {
|
||||
@@ -1244,7 +1251,7 @@ function compareScalaPackageMembers(a, b) {
|
||||
const aPackage = /\/package\.s(?:cala|c)$/.test(a);
|
||||
const bPackage = /\/package\.s(?:cala|c)$/.test(b);
|
||||
if (dirOf(a) === dirOf(b) && aPackage !== bPackage) return aPackage ? 1 : -1;
|
||||
return a.localeCompare(b);
|
||||
return comparePaths(a, b);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -1913,7 +1920,7 @@ async function main() {
|
||||
resolved = [...resolvedSet].sort((a, b) =>
|
||||
file.language === 'scala'
|
||||
? compareScalaPackageMembers(a, b)
|
||||
: a.localeCompare(b),
|
||||
: comparePaths(a, b),
|
||||
);
|
||||
} catch (err) {
|
||||
process.stderr.write(
|
||||
|
||||
@@ -1,5 +1,63 @@
|
||||
// Pure result mapping for extract-structure.mjs.
|
||||
// Kept separate from the CLI entrypoint so unit tests do not import a shebang script.
|
||||
function mapCallGraph(callGraph) {
|
||||
return callGraph && callGraph.length > 0
|
||||
? callGraph.map(entry => ({
|
||||
caller: entry.caller,
|
||||
callee: entry.callee,
|
||||
lineNumber: entry.lineNumber,
|
||||
}))
|
||||
: null;
|
||||
}
|
||||
|
||||
export function analyzeFileWithOutcomes(registry, file, content) {
|
||||
const wantsCallGraph =
|
||||
file.fileCategory === 'code' || file.fileCategory === 'script';
|
||||
let analysis = null;
|
||||
let callGraph = null;
|
||||
let structureOutcome = 'failed';
|
||||
let callGraphOutcome = wantsCallGraph ? 'failed' : 'skipped';
|
||||
|
||||
let full = null;
|
||||
if (wantsCallGraph && typeof registry.analyzeFileFull === 'function') {
|
||||
try {
|
||||
full = registry.analyzeFileFull(file.path, content);
|
||||
} catch {
|
||||
full = null;
|
||||
}
|
||||
}
|
||||
|
||||
if (full) {
|
||||
analysis = full.structure ?? null;
|
||||
callGraph = mapCallGraph(full.callGraph);
|
||||
structureOutcome = analysis === null ? 'failed' : 'succeeded';
|
||||
callGraphOutcome = Array.isArray(full.callGraph) ? 'succeeded' : 'failed';
|
||||
} else {
|
||||
try {
|
||||
analysis = registry.analyzeFile(file.path, content) ?? null;
|
||||
structureOutcome = analysis === null ? 'failed' : 'succeeded';
|
||||
} catch {
|
||||
analysis = null;
|
||||
structureOutcome = 'failed';
|
||||
}
|
||||
|
||||
if (wantsCallGraph) {
|
||||
try {
|
||||
const extractedCallGraph = registry.extractCallGraph(file.path, content);
|
||||
callGraph = mapCallGraph(extractedCallGraph);
|
||||
callGraphOutcome = Array.isArray(extractedCallGraph)
|
||||
? 'succeeded'
|
||||
: 'failed';
|
||||
} catch {
|
||||
callGraph = null;
|
||||
callGraphOutcome = 'failed';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { analysis, callGraph, structureOutcome, callGraphOutcome };
|
||||
}
|
||||
|
||||
export function buildResult(file, totalLines, nonEmptyLines, analysis, callGraph, batchImportData) {
|
||||
const base = {
|
||||
path: file.path,
|
||||
|
||||
@@ -20,9 +20,15 @@ import { createRequire } from 'node:module';
|
||||
import { dirname, resolve, join } from 'node:path';
|
||||
import { fileURLToPath, pathToFileURL } from 'node:url';
|
||||
import { existsSync, readFileSync, realpathSync, writeFileSync } from 'node:fs';
|
||||
import { buildResult as buildExtractResult } from './extract-structure-result.mjs';
|
||||
import {
|
||||
analyzeFileWithOutcomes,
|
||||
buildResult as buildExtractResult,
|
||||
} from './extract-structure-result.mjs';
|
||||
|
||||
export { buildResult } from './extract-structure-result.mjs';
|
||||
export {
|
||||
analyzeFileWithOutcomes,
|
||||
buildResult,
|
||||
} from './extract-structure-result.mjs';
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
// skills/understand/ -> plugin root is two dirs up
|
||||
@@ -77,6 +83,10 @@ async function main() {
|
||||
|
||||
const results = [];
|
||||
const filesSkipped = [];
|
||||
const analysisOutcomes = {
|
||||
structure: { succeeded: 0, failed: 0 },
|
||||
callGraph: { succeeded: 0, failed: 0, skipped: 0 },
|
||||
};
|
||||
|
||||
for (const file of batchFiles) {
|
||||
const absolutePath = join(projectRoot, file.path);
|
||||
@@ -97,56 +107,10 @@ async function main() {
|
||||
const totalLines = content.endsWith('\n') ? Math.max(0, lines.length - 1) : lines.length;
|
||||
const nonEmptyLines = lines.filter(l => l.trim().length > 0).length;
|
||||
|
||||
const wantsCallGraph =
|
||||
file.fileCategory === 'code' || file.fileCategory === 'script';
|
||||
|
||||
const mapCallGraph = cg =>
|
||||
cg && cg.length > 0
|
||||
? cg.map(entry => ({
|
||||
caller: entry.caller,
|
||||
callee: entry.callee,
|
||||
lineNumber: entry.lineNumber,
|
||||
}))
|
||||
: null;
|
||||
|
||||
let analysis = null;
|
||||
let callGraph = null;
|
||||
|
||||
// Single-parse fast path: when both structure and call graph are needed,
|
||||
// analyzeFileFull parses the file once instead of analyzeFile +
|
||||
// extractCallGraph parsing it twice (~40% less parse work on code files).
|
||||
// Falls back to the two separate calls (preserving their independent
|
||||
// degradation) when the registry/plugin lacks the combined method or it
|
||||
// throws.
|
||||
let full = null;
|
||||
if (wantsCallGraph && typeof registry.analyzeFileFull === 'function') {
|
||||
try {
|
||||
full = registry.analyzeFileFull(file.path, content);
|
||||
} catch {
|
||||
full = null;
|
||||
}
|
||||
}
|
||||
|
||||
if (full) {
|
||||
analysis = full.structure;
|
||||
callGraph = mapCallGraph(full.callGraph);
|
||||
} else {
|
||||
// Structural analysis via registry
|
||||
try {
|
||||
analysis = registry.analyzeFile(file.path, content);
|
||||
} catch {
|
||||
// If analysis throws, treat as degraded — still include basic metrics
|
||||
}
|
||||
|
||||
// Call graph extraction (code files only)
|
||||
if (wantsCallGraph) {
|
||||
try {
|
||||
callGraph = mapCallGraph(registry.extractCallGraph(file.path, content));
|
||||
} catch {
|
||||
// Call graph extraction failed — non-fatal
|
||||
}
|
||||
}
|
||||
}
|
||||
const { analysis, callGraph, structureOutcome, callGraphOutcome } =
|
||||
analyzeFileWithOutcomes(registry, file, content);
|
||||
analysisOutcomes.structure[structureOutcome] += 1;
|
||||
analysisOutcomes.callGraph[callGraphOutcome] += 1;
|
||||
|
||||
// Build result object
|
||||
const result = buildExtractResult(file, totalLines, nonEmptyLines, analysis, callGraph, batchImportData);
|
||||
@@ -158,6 +122,7 @@ async function main() {
|
||||
scriptCompleted: true,
|
||||
filesAnalyzed: results.length,
|
||||
filesSkipped,
|
||||
analysisOutcomes,
|
||||
results,
|
||||
};
|
||||
|
||||
|
||||
Reference in New Issue
Block a user