fix(replay): preserve real sessionId + surface 200-file scan cap
Two bugs in the JSONL import path that combined to make \`agentmemory import-jsonl\` against a real \`~/.claude/projects\` tree quietly mis-import: ~10% of files scanned, each landing under a fresh random session id even when the file's own header named one. ### #202 — \`parseJsonlText\` ignores the file's sessionId The parser pre-populated \`sessionId\` from \`fallbackSessionId\` and then guarded \`if (entry.sessionId && !sessionId)\`, so the file's real id was never adopted. Each call returned a freshly generated session, breaking merge-on-reimport (Session.observationCount accumulation, endedAt extension, jsonl-import tag set) and inflating the session count ~4× on real trees. Fix: don't pre-populate sessionId; fall back to \`fallbackSessionId\` and then \`generateId('sess')\` only after the loop has had a chance to find the file's id. ### #203 — \`import-jsonl\` CLI silently caps at 200 files \`findJsonlFiles\` had a hard 200-file safety cap (sensible given the 30s function timeout) but the CLI exposed no flag to override it and the post-import summary never warned that files were skipped. On a 2167-file tree users got a silent ~10% import. Fix: - \`findJsonlFiles\` now returns \`{files, truncated, discovered}\` so the handler can keep counting beyond the cap without retaining paths - Handler returns \`discovered\`, \`truncated\`, \`maxFiles\` in the response - CLI accepts \`--max-files <N>\` (and \`--max-files=<N>\`) - CLI warns on truncation: how many were skipped, suggested re-run with a larger cap Adds three regression tests in \`test/replay.test.ts\` covering: file sessionId beats fallback, two parses of the same file produce the same id, fallback used only when file has no id. Thanks @bloodcarter for the precise repros and root-cause pointers. Closes #202 Closes #203
This commit is contained in:
committed by
Rohit Ghumare
parent
47184fdf17
commit
6768b1055b
+28
@@ -36,6 +36,7 @@ Commands:
|
||||
upgrade Upgrade local deps + iii runtime (best effort)
|
||||
mcp Start standalone MCP server (no engine required)
|
||||
import-jsonl [p] Import Claude Code JSONL transcripts (default: ~/.claude/projects)
|
||||
Use --max-files <N> to override the 200-file scan cap (default: 200)
|
||||
|
||||
Options:
|
||||
--help, -h Show this help
|
||||
@@ -962,6 +963,20 @@ async function runMcp(): Promise<void> {
|
||||
async function runImportJsonl(): Promise<void> {
|
||||
const nonFlagArgs = args.slice(1).filter((a) => !a.startsWith("-"));
|
||||
const pathArg = nonFlagArgs[0];
|
||||
|
||||
let maxFiles: number | undefined;
|
||||
const flagIdx = args.findIndex((a) => a === "--max-files");
|
||||
if (flagIdx !== -1 && args[flagIdx + 1]) {
|
||||
const parsed = parseInt(args[flagIdx + 1]!, 10);
|
||||
if (!Number.isNaN(parsed) && parsed > 0) maxFiles = parsed;
|
||||
} else {
|
||||
const eqArg = args.find((a) => a.startsWith("--max-files="));
|
||||
if (eqArg) {
|
||||
const parsed = parseInt(eqArg.slice("--max-files=".length), 10);
|
||||
if (!Number.isNaN(parsed) && parsed > 0) maxFiles = parsed;
|
||||
}
|
||||
}
|
||||
|
||||
const port = getRestPort();
|
||||
const base = `http://localhost:${port}`;
|
||||
|
||||
@@ -990,6 +1005,7 @@ async function runImportJsonl(): Promise<void> {
|
||||
|
||||
const body: Record<string, unknown> = {};
|
||||
if (pathArg) body["path"] = pathArg;
|
||||
if (maxFiles !== undefined) body["maxFiles"] = maxFiles;
|
||||
|
||||
const headers: Record<string, string> = { "content-type": "application/json" };
|
||||
const secret = process.env["AGENTMEMORY_SECRET"];
|
||||
@@ -1013,6 +1029,9 @@ async function runImportJsonl(): Promise<void> {
|
||||
imported?: number;
|
||||
sessionIds?: string[];
|
||||
observations?: number;
|
||||
discovered?: number;
|
||||
truncated?: boolean;
|
||||
maxFiles?: number;
|
||||
} = {};
|
||||
if (text.length > 0) {
|
||||
try {
|
||||
@@ -1050,6 +1069,15 @@ async function runImportJsonl(): Promise<void> {
|
||||
spinner.stop(
|
||||
`imported ${json.imported ?? 0} file(s), ${json.observations ?? 0} observation(s) across ${json.sessionIds?.length || 0} session(s)`,
|
||||
);
|
||||
if (json.truncated) {
|
||||
const cap = json.maxFiles ?? 200;
|
||||
const skipped = (json.discovered ?? 0) - (json.imported ?? 0);
|
||||
p.log.warn(
|
||||
`Hit the ${cap}-file scan cap; ${skipped} of ${json.discovered ?? "?"} discovered file(s) were skipped. ` +
|
||||
`Re-run with --max-files=<N> (e.g. --max-files=${Math.max((json.discovered ?? cap) + 100, cap * 2)}) ` +
|
||||
`or batch by subdirectory.`,
|
||||
);
|
||||
}
|
||||
if (json.sessionIds && json.sessionIds.length > 0) {
|
||||
p.log.info(`View at ${getViewerUrl()} → Replay tab`);
|
||||
}
|
||||
|
||||
+31
-7
@@ -200,10 +200,13 @@ async function loadObservations(
|
||||
return rows.map((r) => (isRawShape(r) ? r : rawFromCompressed(r as CompressedObservation)));
|
||||
}
|
||||
|
||||
async function findJsonlFiles(root: string, limit = 200): Promise<string[]> {
|
||||
async function findJsonlFiles(
|
||||
root: string,
|
||||
limit = 200,
|
||||
): Promise<{ files: string[]; truncated: boolean; discovered: number }> {
|
||||
const out: string[] = [];
|
||||
let discovered = 0;
|
||||
async function walk(dir: string) {
|
||||
if (out.length >= limit) return;
|
||||
let names: string[];
|
||||
try {
|
||||
names = await readdir(dir);
|
||||
@@ -211,7 +214,6 @@ async function findJsonlFiles(root: string, limit = 200): Promise<string[]> {
|
||||
return;
|
||||
}
|
||||
for (const name of names) {
|
||||
if (out.length >= limit) return;
|
||||
const full = join(dir, name);
|
||||
let st;
|
||||
try {
|
||||
@@ -223,12 +225,13 @@ async function findJsonlFiles(root: string, limit = 200): Promise<string[]> {
|
||||
if (st.isDirectory()) {
|
||||
await walk(full);
|
||||
} else if (st.isFile() && name.endsWith(".jsonl")) {
|
||||
out.push(full);
|
||||
discovered++;
|
||||
if (out.length < limit) out.push(full);
|
||||
}
|
||||
}
|
||||
}
|
||||
await walk(root);
|
||||
return out;
|
||||
return { files: out, truncated: discovered > out.length, discovered };
|
||||
}
|
||||
|
||||
export function registerReplayFunctions(sdk: ISdk, kv: StateKV): void {
|
||||
@@ -267,6 +270,9 @@ export function registerReplayFunctions(sdk: ISdk, kv: StateKV): void {
|
||||
imported: number;
|
||||
sessionIds: string[];
|
||||
observations: number;
|
||||
discovered: number;
|
||||
truncated: boolean;
|
||||
maxFiles: number;
|
||||
}
|
||||
| { success: false; error: string }
|
||||
> => {
|
||||
@@ -293,17 +299,32 @@ export function registerReplayFunctions(sdk: ISdk, kv: StateKV): void {
|
||||
return { success: false, error: "path not found" };
|
||||
}
|
||||
|
||||
const maxFiles = data.maxFiles && data.maxFiles > 0 ? data.maxFiles : 200;
|
||||
let files: string[] = [];
|
||||
let truncated = false;
|
||||
let discovered = 0;
|
||||
if (stat.isDirectory()) {
|
||||
files = await findJsonlFiles(abs, data.maxFiles || 200);
|
||||
const found = await findJsonlFiles(abs, maxFiles);
|
||||
files = found.files;
|
||||
truncated = found.truncated;
|
||||
discovered = found.discovered;
|
||||
} else if (stat.isFile() && abs.endsWith(".jsonl")) {
|
||||
files = [abs];
|
||||
discovered = 1;
|
||||
} else {
|
||||
return { success: false, error: "path must be a .jsonl file or directory" };
|
||||
}
|
||||
|
||||
if (files.length === 0) {
|
||||
return { success: true, imported: 0, sessionIds: [], observations: 0 };
|
||||
return {
|
||||
success: true,
|
||||
imported: 0,
|
||||
sessionIds: [],
|
||||
observations: 0,
|
||||
discovered,
|
||||
truncated,
|
||||
maxFiles,
|
||||
};
|
||||
}
|
||||
|
||||
const sessionIds: string[] = [];
|
||||
@@ -399,6 +420,9 @@ export function registerReplayFunctions(sdk: ISdk, kv: StateKV): void {
|
||||
imported: files.length,
|
||||
sessionIds,
|
||||
observations: observationCount,
|
||||
discovered,
|
||||
truncated,
|
||||
maxFiles,
|
||||
};
|
||||
},
|
||||
);
|
||||
|
||||
@@ -90,7 +90,7 @@ export function parseJsonlText(text: string, fallbackSessionId?: string): Parsed
|
||||
}
|
||||
}
|
||||
|
||||
let sessionId = fallbackSessionId || "";
|
||||
let sessionId = "";
|
||||
let cwd = "";
|
||||
let firstTs = "";
|
||||
let lastTs = "";
|
||||
@@ -164,7 +164,7 @@ export function parseJsonlText(text: string, fallbackSessionId?: string): Parsed
|
||||
}
|
||||
}
|
||||
|
||||
const effectiveSessionId = sessionId || generateId("sess");
|
||||
const effectiveSessionId = sessionId || fallbackSessionId || generateId("sess");
|
||||
for (const obs of observations) {
|
||||
if (obs.sessionId === "imported") obs.sessionId = effectiveSessionId;
|
||||
}
|
||||
|
||||
@@ -58,6 +58,45 @@ describe("parseJsonlText", () => {
|
||||
const out = parseJsonlText("");
|
||||
expect(out.observations).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("prefers the file's sessionId over the fallback", () => {
|
||||
const text = [
|
||||
JSON.stringify({
|
||||
type: "user",
|
||||
sessionId: "real-session-from-file",
|
||||
timestamp: "2026-01-01T00:00:00.000Z",
|
||||
message: { role: "user", content: [{ type: "text", text: "hi" }] },
|
||||
}),
|
||||
].join("\n");
|
||||
const out = parseJsonlText(text, "fallback-should-be-ignored");
|
||||
expect(out.sessionId).toBe("real-session-from-file");
|
||||
for (const obs of out.observations) {
|
||||
expect(obs.sessionId).toBe("real-session-from-file");
|
||||
}
|
||||
});
|
||||
|
||||
it("returns the same sessionId across repeated parses of one file", () => {
|
||||
const text = JSON.stringify({
|
||||
type: "user",
|
||||
sessionId: "stable-id",
|
||||
timestamp: "2026-01-01T00:00:00.000Z",
|
||||
message: { role: "user", content: [{ type: "text", text: "hi" }] },
|
||||
});
|
||||
const a = parseJsonlText(text, "fb-1");
|
||||
const b = parseJsonlText(text, "fb-2");
|
||||
expect(a.sessionId).toBe("stable-id");
|
||||
expect(b.sessionId).toBe("stable-id");
|
||||
});
|
||||
|
||||
it("uses the fallback only when the file has no sessionId", () => {
|
||||
const text = JSON.stringify({
|
||||
type: "user",
|
||||
timestamp: "2026-01-01T00:00:00.000Z",
|
||||
message: { role: "user", content: [{ type: "text", text: "hi" }] },
|
||||
});
|
||||
const out = parseJsonlText(text, "fb-used");
|
||||
expect(out.sessionId).toBe("fb-used");
|
||||
});
|
||||
});
|
||||
|
||||
describe("projectTimeline", () => {
|
||||
|
||||
Reference in New Issue
Block a user