fix(replay): preserve real sessionId + surface 200-file scan cap

Two bugs in the JSONL import path that combined to make
\`agentmemory import-jsonl\` against a real \`~/.claude/projects\` tree
quietly mis-import: ~10% of files scanned, each landing under a fresh
random session id even when the file's own header named one.

### #202 — \`parseJsonlText\` ignores the file's sessionId

The parser pre-populated \`sessionId\` from \`fallbackSessionId\` and then
guarded \`if (entry.sessionId && !sessionId)\`, so the file's real id
was never adopted. Each call returned a freshly generated session,
breaking merge-on-reimport (Session.observationCount accumulation,
endedAt extension, jsonl-import tag set) and inflating the session
count ~4× on real trees.

Fix: don't pre-populate sessionId; fall back to \`fallbackSessionId\`
and then \`generateId('sess')\` only after the loop has had a chance
to find the file's id.

### #203 — \`import-jsonl\` CLI silently caps at 200 files

\`findJsonlFiles\` had a hard 200-file safety cap (sensible given the
30s function timeout) but the CLI exposed no flag to override it and
the post-import summary never warned that files were skipped. On a
2167-file tree users got a silent ~10% import.

Fix:
- \`findJsonlFiles\` now returns \`{files, truncated, discovered}\` so
  the handler can keep counting beyond the cap without retaining paths
- Handler returns \`discovered\`, \`truncated\`, \`maxFiles\` in the response
- CLI accepts \`--max-files <N>\` (and \`--max-files=<N>\`)
- CLI warns on truncation: how many were skipped, suggested re-run
  with a larger cap

Adds three regression tests in \`test/replay.test.ts\` covering: file
sessionId beats fallback, two parses of the same file produce the same
id, fallback used only when file has no id.

Thanks @bloodcarter for the precise repros and root-cause pointers.

Closes #202
Closes #203
This commit is contained in:
Rohit Ghumare
2026-04-27 00:23:12 +01:00
committed by Rohit Ghumare
parent 47184fdf17
commit 6768b1055b
4 changed files with 100 additions and 9 deletions
+28
View File
@@ -36,6 +36,7 @@ Commands:
upgrade Upgrade local deps + iii runtime (best effort)
mcp Start standalone MCP server (no engine required)
import-jsonl [p] Import Claude Code JSONL transcripts (default: ~/.claude/projects)
Use --max-files <N> to override the 200-file scan cap (default: 200)
Options:
--help, -h Show this help
@@ -962,6 +963,20 @@ async function runMcp(): Promise<void> {
async function runImportJsonl(): Promise<void> {
const nonFlagArgs = args.slice(1).filter((a) => !a.startsWith("-"));
const pathArg = nonFlagArgs[0];
let maxFiles: number | undefined;
const flagIdx = args.findIndex((a) => a === "--max-files");
if (flagIdx !== -1 && args[flagIdx + 1]) {
const parsed = parseInt(args[flagIdx + 1]!, 10);
if (!Number.isNaN(parsed) && parsed > 0) maxFiles = parsed;
} else {
const eqArg = args.find((a) => a.startsWith("--max-files="));
if (eqArg) {
const parsed = parseInt(eqArg.slice("--max-files=".length), 10);
if (!Number.isNaN(parsed) && parsed > 0) maxFiles = parsed;
}
}
const port = getRestPort();
const base = `http://localhost:${port}`;
@@ -990,6 +1005,7 @@ async function runImportJsonl(): Promise<void> {
const body: Record<string, unknown> = {};
if (pathArg) body["path"] = pathArg;
if (maxFiles !== undefined) body["maxFiles"] = maxFiles;
const headers: Record<string, string> = { "content-type": "application/json" };
const secret = process.env["AGENTMEMORY_SECRET"];
@@ -1013,6 +1029,9 @@ async function runImportJsonl(): Promise<void> {
imported?: number;
sessionIds?: string[];
observations?: number;
discovered?: number;
truncated?: boolean;
maxFiles?: number;
} = {};
if (text.length > 0) {
try {
@@ -1050,6 +1069,15 @@ async function runImportJsonl(): Promise<void> {
spinner.stop(
`imported ${json.imported ?? 0} file(s), ${json.observations ?? 0} observation(s) across ${json.sessionIds?.length || 0} session(s)`,
);
if (json.truncated) {
const cap = json.maxFiles ?? 200;
const skipped = (json.discovered ?? 0) - (json.imported ?? 0);
p.log.warn(
`Hit the ${cap}-file scan cap; ${skipped} of ${json.discovered ?? "?"} discovered file(s) were skipped. ` +
`Re-run with --max-files=<N> (e.g. --max-files=${Math.max((json.discovered ?? cap) + 100, cap * 2)}) ` +
`or batch by subdirectory.`,
);
}
if (json.sessionIds && json.sessionIds.length > 0) {
p.log.info(`View at ${getViewerUrl()} → Replay tab`);
}
+31 -7
View File
@@ -200,10 +200,13 @@ async function loadObservations(
return rows.map((r) => (isRawShape(r) ? r : rawFromCompressed(r as CompressedObservation)));
}
async function findJsonlFiles(root: string, limit = 200): Promise<string[]> {
async function findJsonlFiles(
root: string,
limit = 200,
): Promise<{ files: string[]; truncated: boolean; discovered: number }> {
const out: string[] = [];
let discovered = 0;
async function walk(dir: string) {
if (out.length >= limit) return;
let names: string[];
try {
names = await readdir(dir);
@@ -211,7 +214,6 @@ async function findJsonlFiles(root: string, limit = 200): Promise<string[]> {
return;
}
for (const name of names) {
if (out.length >= limit) return;
const full = join(dir, name);
let st;
try {
@@ -223,12 +225,13 @@ async function findJsonlFiles(root: string, limit = 200): Promise<string[]> {
if (st.isDirectory()) {
await walk(full);
} else if (st.isFile() && name.endsWith(".jsonl")) {
out.push(full);
discovered++;
if (out.length < limit) out.push(full);
}
}
}
await walk(root);
return out;
return { files: out, truncated: discovered > out.length, discovered };
}
export function registerReplayFunctions(sdk: ISdk, kv: StateKV): void {
@@ -267,6 +270,9 @@ export function registerReplayFunctions(sdk: ISdk, kv: StateKV): void {
imported: number;
sessionIds: string[];
observations: number;
discovered: number;
truncated: boolean;
maxFiles: number;
}
| { success: false; error: string }
> => {
@@ -293,17 +299,32 @@ export function registerReplayFunctions(sdk: ISdk, kv: StateKV): void {
return { success: false, error: "path not found" };
}
const maxFiles = data.maxFiles && data.maxFiles > 0 ? data.maxFiles : 200;
let files: string[] = [];
let truncated = false;
let discovered = 0;
if (stat.isDirectory()) {
files = await findJsonlFiles(abs, data.maxFiles || 200);
const found = await findJsonlFiles(abs, maxFiles);
files = found.files;
truncated = found.truncated;
discovered = found.discovered;
} else if (stat.isFile() && abs.endsWith(".jsonl")) {
files = [abs];
discovered = 1;
} else {
return { success: false, error: "path must be a .jsonl file or directory" };
}
if (files.length === 0) {
return { success: true, imported: 0, sessionIds: [], observations: 0 };
return {
success: true,
imported: 0,
sessionIds: [],
observations: 0,
discovered,
truncated,
maxFiles,
};
}
const sessionIds: string[] = [];
@@ -399,6 +420,9 @@ export function registerReplayFunctions(sdk: ISdk, kv: StateKV): void {
imported: files.length,
sessionIds,
observations: observationCount,
discovered,
truncated,
maxFiles,
};
},
);
+2 -2
View File
@@ -90,7 +90,7 @@ export function parseJsonlText(text: string, fallbackSessionId?: string): Parsed
}
}
let sessionId = fallbackSessionId || "";
let sessionId = "";
let cwd = "";
let firstTs = "";
let lastTs = "";
@@ -164,7 +164,7 @@ export function parseJsonlText(text: string, fallbackSessionId?: string): Parsed
}
}
const effectiveSessionId = sessionId || generateId("sess");
const effectiveSessionId = sessionId || fallbackSessionId || generateId("sess");
for (const obs of observations) {
if (obs.sessionId === "imported") obs.sessionId = effectiveSessionId;
}
+39
View File
@@ -58,6 +58,45 @@ describe("parseJsonlText", () => {
const out = parseJsonlText("");
expect(out.observations).toHaveLength(0);
});
it("prefers the file's sessionId over the fallback", () => {
const text = [
JSON.stringify({
type: "user",
sessionId: "real-session-from-file",
timestamp: "2026-01-01T00:00:00.000Z",
message: { role: "user", content: [{ type: "text", text: "hi" }] },
}),
].join("\n");
const out = parseJsonlText(text, "fallback-should-be-ignored");
expect(out.sessionId).toBe("real-session-from-file");
for (const obs of out.observations) {
expect(obs.sessionId).toBe("real-session-from-file");
}
});
it("returns the same sessionId across repeated parses of one file", () => {
const text = JSON.stringify({
type: "user",
sessionId: "stable-id",
timestamp: "2026-01-01T00:00:00.000Z",
message: { role: "user", content: [{ type: "text", text: "hi" }] },
});
const a = parseJsonlText(text, "fb-1");
const b = parseJsonlText(text, "fb-2");
expect(a.sessionId).toBe("stable-id");
expect(b.sessionId).toBe("stable-id");
});
it("uses the fallback only when the file has no sessionId", () => {
const text = JSON.stringify({
type: "user",
timestamp: "2026-01-01T00:00:00.000Z",
message: { role: "user", content: [{ type: "text", text: "hi" }] },
});
const out = parseJsonlText(text, "fb-used");
expect(out.sessionId).toBe("fb-used");
});
});
describe("projectTimeline", () => {