iamtoruk
d5e485f415
Every dated command re-walks and re-stats every provider tree before it can decide what the cache already covers. That sweep was strictly serial at every level -- one readdir, one stat, one state.json read at a time, and one provider after another -- so on a 21k-file / 9-provider corpus it owned most of a warm run's wall clock with the machine idle waiting on the kernel. Measured per warm `today` pass on that corpus before this change: 859 ms in discoverAllSessions (kimicode 467, codex 143, claude 135, grok 77) and 727 ms in the Claude project walk + fingerprint pass. - fs-utils: mapWithConcurrency + FS_SCAN_CONCURRENCY, one bounded, order- preserving helper for the whole sweep. - providers/index: run provider discovery concurrently, concatenated in registry order. - claude/codex/grok/kimicode: walk each level with the level fanned out, re-concatenated in readdir order before anything reconciles. - parser: the Claude dir walk and both fingerprint passes (scanProjectDirs and parseProviderSources) fan out, then reconcile serially in discovery order, which is what changedFiles ordering and the seenMsgIds pre-seed depend on. - parser: collectJsonlFiles reads entries with their types, so a plain file no longer costs a wasted subagents/ probe. - codex-cache: share one in-flight load between concurrent readers. The memo is only populated after the read + parse resolves, so concurrent discovery had every caller re-reading and re-parsing the same (here 59 MB) file. No reconciliation logic changed: the same fingerprints reach the same cache comparison in the same order. Warm `today` 5.30s -> 2.99s, cold 34.7s -> 31.1s (medians of 5 / 2, isolated HOME + cache). today/report -p month/models/sessions JSON, warm and cold, per-provider and combined, are byte-identical apart from the run's own `generated` timestamp. Closes #1104
67 lines
2.4 KiB
TypeScript
67 lines
2.4 KiB
TypeScript
// The codex result cache is a single (often hundreds-of-MB) JSON file, memoized
|
|
// in memory only once the read + parse resolves. Discovery now asks for it from
|
|
// many concurrent callers, so without a shared in-flight promise every one of
|
|
// them re-read and re-parsed the whole file.
|
|
|
|
import { mkdtemp, rm, writeFile, mkdir } from 'fs/promises'
|
|
import { tmpdir } from 'os'
|
|
import { join } from 'path'
|
|
|
|
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'
|
|
|
|
const readSpy = vi.hoisted(() => vi.fn())
|
|
|
|
vi.mock('../src/cache-dir.js', async (importOriginal) => {
|
|
const actual = await importOriginal<typeof import('../src/cache-dir.js')>()
|
|
return {
|
|
...actual,
|
|
readExistingTextFile: (path: string) => {
|
|
readSpy(path)
|
|
return actual.readExistingTextFile(path)
|
|
},
|
|
}
|
|
})
|
|
|
|
const { CODEX_CACHE_VERSION, clearCodexMemCaches, codexCacheFileName, getCachedCodexProject, withCodexCacheDirectory } =
|
|
await import('../src/codex-cache.js')
|
|
|
|
let cacheDir: string
|
|
let sessionDir: string
|
|
|
|
beforeEach(async () => {
|
|
readSpy.mockClear()
|
|
clearCodexMemCaches()
|
|
const root = await mkdtemp(join(tmpdir(), 'codeburn-codex-cache-'))
|
|
cacheDir = join(root, 'cache')
|
|
sessionDir = join(root, 'sessions')
|
|
await mkdir(cacheDir, { recursive: true })
|
|
await mkdir(sessionDir, { recursive: true })
|
|
})
|
|
|
|
afterEach(async () => {
|
|
clearCodexMemCaches()
|
|
await rm(join(cacheDir, '..'), { recursive: true, force: true })
|
|
})
|
|
|
|
describe('codex result cache under concurrent readers', () => {
|
|
it('reads the cache file once and answers every caller correctly', async () => {
|
|
const paths: string[] = []
|
|
const files: Record<string, unknown> = {}
|
|
for (let i = 0; i < 24; i++) {
|
|
const p = join(sessionDir, `rollout-${i}.jsonl`)
|
|
await writeFile(p, '{}\n')
|
|
paths.push(p)
|
|
const { statSync } = await import('fs')
|
|
const s = statSync(p)
|
|
files[p] = { dev: s.dev, ino: s.ino, mtimeMs: s.mtimeMs, sizeBytes: s.size, project: `proj-${i}`, calls: [] }
|
|
}
|
|
await writeFile(join(cacheDir, codexCacheFileName()), JSON.stringify({ version: CODEX_CACHE_VERSION, files }))
|
|
|
|
const projects = await withCodexCacheDirectory(cacheDir, () =>
|
|
Promise.all(paths.map(p => getCachedCodexProject(p))))
|
|
|
|
expect(projects).toEqual(paths.map((_, i) => `proj-${i}`))
|
|
expect(readSpy.mock.calls.filter(([p]) => String(p).includes('codex-results'))).toHaveLength(1)
|
|
})
|
|
})
|