fix(memory): sync FTS5 and vector indexes on CLI delete/edit/prune/purge
## Problem `headroom memory delete`, `prune`, `edit`, and `purge` all operate on the bare `SQLiteMemoryStore` — they update the primary `memories` table but never touch the FTS5 full-text index (`memory_fts` in `memory.db`) or the vector index (`vec_metadata` / `vec_embeddings` in `memory_vectors.db`). The index maintenance path lives in `HierarchicalMemory.delete()` / `.update()`, which the CLI never instantiates. **Symptoms (from #2856):** ```sql -- After deleting 16 of 46 memories via CLI: SELECT COUNT(*) FROM memories; -- 30 SELECT COUNT(*) FROM memory_fts; -- 46 ← orphans -- memory_vectors.db SELECT COUNT(*) FROM vec_metadata; -- 46 ← orphans ``` Deleted memories keep surfacing in `memory_search` results even after a full server restart, because server startup only re-embeds memories whose `embedding IS NULL` — it never removes orphaned index entries. Fixes #2856. ## Solution Add two best-effort helpers to `headroom/cli/memory.py` that use **direct SQLite** (no `sqlite-vec` extension, no embedder): - **`_remove_from_search_indexes(db_path, memory_ids)`**: removes specific IDs from `memory_fts` and from `vec_metadata` / `vec_embeddings`. Skips silently if an index doesn't exist. - **`_clear_all_search_indexes(db_path)`**: truncates both indexes completely (for purge). Wire these up in four commands: | Command | Change | |---|---| | `delete` | `_remove_from_search_indexes` after `store.delete_batch()` | | `prune` | `_remove_from_search_indexes` after `store.delete_batch()` | | `purge` | `_clear_all_search_indexes` after `store.clear_all()` | | `edit` | If content changed: remove stale entries, clear `embedding` (server re-embeds on next startup), re-add FTS5 entry with new content immediately | The edit path re-adds the FTS5 entry right away so keyword search reflects the new content without requiring a server restart. Vector search is deferred to the next startup re-embed cycle (same as what the server already does for missing embeddings). ## Changes - `headroom/cli/memory.py` — two new helpers; four command call sites - `tests/test_cli_memory_index_sync.py` (new) — 9 unit tests covering both helpers with FTS5 and a stub vector DB. No `sqlite-vec` or embedder required; tests run locally. ## Testing ``` $ python -m pytest tests/test_cli_memory_index_sync.py -v ... 9 passed in 2.38s ``` --------- Signed-off-by: Radhakrishnan Pachyappan <radhakrishnan.p@op.tech> Signed-off-by: Radhakrishnan Pachyappan <gingeekrishna@gmail.com> Co-authored-by: Tejas Chopra <tejas@Tejass-MacBook-Pro.local>
This commit is contained in:
Radhakrishnan Pachyappan
committed by
GitHub
GitHub
parent
65961827cf
commit
fd4628d821
+329
-3
@@ -30,6 +30,8 @@ from ._utils.formatting import (
|
||||
from ._utils.parsers import parse_duration
|
||||
from .main import main
|
||||
|
||||
_REINDEX_PAGE_SIZE = 1_000
|
||||
|
||||
|
||||
def _default_db_path() -> str:
|
||||
"""Resolve the memory DB the proxy/install actually use.
|
||||
@@ -66,6 +68,148 @@ def get_store(db_path: str) -> SQLiteMemoryStore:
|
||||
return SQLiteMemoryStore(db_path)
|
||||
|
||||
|
||||
def _sqlite_table_exists(conn: Any, table_name: str) -> bool:
|
||||
"""Return whether a SQLite table or virtual table has been initialized."""
|
||||
row = conn.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?",
|
||||
(table_name,),
|
||||
).fetchone()
|
||||
return row is not None
|
||||
|
||||
|
||||
def _remove_from_search_indexes(db_path: str, memory_ids: list[str]) -> bool:
|
||||
"""Remove specific memories from FTS5 and vector search indexes.
|
||||
|
||||
FTS5 cleanup uses a bare sqlite3 connection (FTS5 is built-in).
|
||||
Vector cleanup requires sqlite-vec to load the vec0 virtual-table
|
||||
module; when not installed a warning is printed.
|
||||
|
||||
Returns True when both indexes were fully synced, False when any part
|
||||
of the sync failed. Callers must treat False as a partial failure and
|
||||
surface it — typically by exiting with a non-zero code so the primary
|
||||
store mutation is not silently reported as fully successful.
|
||||
"""
|
||||
if not memory_ids:
|
||||
return True
|
||||
|
||||
import sqlite3
|
||||
|
||||
db = Path(db_path)
|
||||
ok = True
|
||||
|
||||
# FTS5 table lives in the same memory.db file (built-in, no extension needed).
|
||||
try:
|
||||
with sqlite3.connect(str(db)) as conn:
|
||||
if _sqlite_table_exists(conn, "memory_fts"):
|
||||
for i in range(0, len(memory_ids), 500):
|
||||
chunk = memory_ids[i : i + 500]
|
||||
placeholders = ",".join("?" * len(chunk))
|
||||
conn.execute(
|
||||
f"DELETE FROM memory_fts WHERE memory_id IN ({placeholders})",
|
||||
chunk,
|
||||
)
|
||||
conn.commit()
|
||||
except Exception as exc:
|
||||
print_warning(f"FTS5 index cleanup incomplete: {exc}")
|
||||
ok = False
|
||||
|
||||
# Vector DB is a sibling file: memory.db -> memory_vectors.db.
|
||||
# vec_embeddings is a vec0 virtual table — the sqlite-vec extension must be
|
||||
# loaded on every connection before touching it.
|
||||
vector_db = db.parent / f"{db.stem}_vectors.db"
|
||||
if not vector_db.exists():
|
||||
return ok
|
||||
|
||||
try:
|
||||
with sqlite3.connect(str(vector_db)) as conn:
|
||||
# A sibling database may exist before the optional vector index has
|
||||
# ever been initialized. That is a valid no-op, not a sync failure.
|
||||
if not _sqlite_table_exists(conn, "vec_metadata"):
|
||||
return ok
|
||||
|
||||
try:
|
||||
import sqlite_vec
|
||||
except ImportError:
|
||||
print_warning(
|
||||
"sqlite-vec is not installed; stale vector index entries may remain. "
|
||||
"Run 'headroom memory reindex' after installing sqlite-vec to repair."
|
||||
)
|
||||
return False
|
||||
|
||||
conn.enable_load_extension(True)
|
||||
sqlite_vec.load(conn)
|
||||
conn.enable_load_extension(False)
|
||||
for i in range(0, len(memory_ids), 500):
|
||||
chunk = memory_ids[i : i + 500]
|
||||
placeholders = ",".join("?" * len(chunk))
|
||||
rows = conn.execute(
|
||||
f"SELECT rowid FROM vec_metadata WHERE memory_id IN ({placeholders})",
|
||||
chunk,
|
||||
).fetchall()
|
||||
rowids = [r[0] for r in rows]
|
||||
if rowids:
|
||||
rph = ",".join("?" * len(rowids))
|
||||
conn.execute(f"DELETE FROM vec_embeddings WHERE rowid IN ({rph})", rowids)
|
||||
conn.execute(f"DELETE FROM vec_metadata WHERE rowid IN ({rph})", rowids)
|
||||
conn.commit()
|
||||
except Exception as exc:
|
||||
print_warning(f"Vector index cleanup incomplete: {exc}")
|
||||
ok = False
|
||||
|
||||
return ok
|
||||
|
||||
|
||||
def _clear_all_search_indexes(db_path: str) -> bool:
|
||||
"""Truncate both search indexes after a full purge.
|
||||
|
||||
Same extension-loading requirement as :func:`_remove_from_search_indexes`.
|
||||
Returns True on full success, False on any partial failure.
|
||||
"""
|
||||
import sqlite3
|
||||
|
||||
db = Path(db_path)
|
||||
ok = True
|
||||
|
||||
try:
|
||||
with sqlite3.connect(str(db)) as conn:
|
||||
if _sqlite_table_exists(conn, "memory_fts"):
|
||||
conn.execute("DELETE FROM memory_fts")
|
||||
conn.commit()
|
||||
except Exception as exc:
|
||||
print_warning(f"FTS5 index cleanup incomplete: {exc}")
|
||||
ok = False
|
||||
|
||||
vector_db = db.parent / f"{db.stem}_vectors.db"
|
||||
if not vector_db.exists():
|
||||
return ok
|
||||
|
||||
try:
|
||||
with sqlite3.connect(str(vector_db)) as conn:
|
||||
if not _sqlite_table_exists(conn, "vec_metadata"):
|
||||
return ok
|
||||
|
||||
try:
|
||||
import sqlite_vec
|
||||
except ImportError:
|
||||
print_warning(
|
||||
"sqlite-vec is not installed; stale vector index entries may remain. "
|
||||
"Run 'headroom memory reindex' after installing sqlite-vec to repair."
|
||||
)
|
||||
return False
|
||||
|
||||
conn.enable_load_extension(True)
|
||||
sqlite_vec.load(conn)
|
||||
conn.enable_load_extension(False)
|
||||
conn.execute("DELETE FROM vec_embeddings")
|
||||
conn.execute("DELETE FROM vec_metadata")
|
||||
conn.commit()
|
||||
except Exception as exc:
|
||||
print_warning(f"Vector index cleanup incomplete: {exc}")
|
||||
ok = False
|
||||
|
||||
return ok
|
||||
|
||||
|
||||
def _resolve_memory(store: SQLiteMemoryStore, memory_id: str) -> Memory:
|
||||
"""Resolve an exact or unambiguous partial memory ID."""
|
||||
memory = asyncio.run(store.get(memory_id))
|
||||
@@ -620,13 +764,37 @@ def edit_memory(
|
||||
mem = matches[0]
|
||||
|
||||
# Update fields
|
||||
content_changed = content is not None and content != mem.content
|
||||
if content is not None:
|
||||
mem.content = content
|
||||
if importance is not None:
|
||||
mem.importance = importance
|
||||
|
||||
index_ok = True
|
||||
if content_changed:
|
||||
# Clear the stale embedding so the memory MCP server re-embeds on
|
||||
# next startup. Also remove the old FTS5 and vector index entries
|
||||
# now to avoid serving stale search results until then.
|
||||
mem.embedding = None
|
||||
index_ok = _remove_from_search_indexes(db_path, [mem.id])
|
||||
# Re-index FTS5 immediately with new content (no embedder needed).
|
||||
try:
|
||||
from ..memory.adapters.fts5 import FTS5TextIndex
|
||||
|
||||
fts = FTS5TextIndex(db_path=db_path)
|
||||
asyncio.run(fts.index_memory(mem))
|
||||
except Exception as exc:
|
||||
print_warning(f"FTS5 re-index incomplete: {exc}")
|
||||
index_ok = False
|
||||
|
||||
# Save
|
||||
asyncio.run(store.save(mem))
|
||||
if not index_ok:
|
||||
print_warning(
|
||||
f"Updated memory {mem.id[:8]}, but search index sync incomplete. "
|
||||
"Run 'headroom memory reindex' to repair."
|
||||
)
|
||||
sys.exit(1)
|
||||
print_success(f"Updated memory {mem.id[:8]}")
|
||||
|
||||
except Exception as e:
|
||||
@@ -775,7 +943,14 @@ def delete_memories(
|
||||
|
||||
# Delete
|
||||
deleted = asyncio.run(store.delete_batch(resolved_ids))
|
||||
print_success(f"Deleted {deleted} memory(ies).")
|
||||
if _remove_from_search_indexes(db_path, resolved_ids):
|
||||
print_success(f"Deleted {deleted} memory(ies).")
|
||||
else:
|
||||
print_warning(
|
||||
f"Deleted {deleted} memory(ies) from store, but search index sync "
|
||||
"incomplete. Run 'headroom memory reindex' to repair."
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
except click.Abort:
|
||||
click.echo("Aborted.")
|
||||
@@ -897,7 +1072,14 @@ def prune_memories(
|
||||
# Delete
|
||||
ids_to_delete = [m.id for m in memories]
|
||||
deleted = asyncio.run(store.delete_batch(ids_to_delete))
|
||||
print_success(f"Deleted {deleted} memory(ies).")
|
||||
if _remove_from_search_indexes(db_path, ids_to_delete):
|
||||
print_success(f"Deleted {deleted} memory(ies).")
|
||||
else:
|
||||
print_warning(
|
||||
f"Deleted {deleted} memory(ies) from store, but search index sync "
|
||||
"incomplete. Run 'headroom memory reindex' to repair."
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
except click.BadParameter as e:
|
||||
print_error(str(e))
|
||||
@@ -952,7 +1134,14 @@ def purge_memories(ctx: click.Context, db_path: str, confirm_flag: bool) -> None
|
||||
|
||||
# Purge
|
||||
deleted = asyncio.run(store.clear_all())
|
||||
print_success(f"Purged {deleted} memory(ies).")
|
||||
if _clear_all_search_indexes(db_path):
|
||||
print_success(f"Purged {deleted} memory(ies).")
|
||||
else:
|
||||
print_warning(
|
||||
f"Purged {deleted} memory(ies) from store, but search index sync "
|
||||
"incomplete. Run 'headroom memory reindex' to repair."
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
except click.Abort:
|
||||
click.echo("Aborted.")
|
||||
@@ -962,6 +1151,143 @@ def purge_memories(ctx: click.Context, db_path: str, confirm_flag: bool) -> None
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
@memory.command("reindex")
|
||||
@db_path_option
|
||||
@click.pass_context
|
||||
def reindex_memories(ctx: click.Context, db_path: str) -> None:
|
||||
"""Rebuild FTS5 search index and remove orphaned vector entries.
|
||||
|
||||
Use this to repair an inconsistent index after a failed delete, prune,
|
||||
or purge. Run it after installing sqlite-vec to clean up any vector
|
||||
entries that could not be removed earlier.
|
||||
|
||||
Vector embeddings are not regenerated by this command — they are rebuilt
|
||||
automatically when the Headroom server next starts.
|
||||
|
||||
\b
|
||||
Example:
|
||||
headroom memory reindex
|
||||
"""
|
||||
import sqlite3
|
||||
|
||||
store = get_store(db_path)
|
||||
|
||||
try:
|
||||
# Page through the complete active store. A fixed cap is destructive:
|
||||
# clearing FTS and rebuilding only the first N rows drops valid search
|
||||
# coverage, while using the same truncated ID set for vector cleanup
|
||||
# misclassifies later primary rows as orphans.
|
||||
memories: list[Memory] = []
|
||||
offset = 0
|
||||
while True:
|
||||
page = asyncio.run(
|
||||
store.query(
|
||||
MemoryFilter(
|
||||
limit=_REINDEX_PAGE_SIZE,
|
||||
offset=offset,
|
||||
order_by="created_at",
|
||||
order_desc=False,
|
||||
)
|
||||
)
|
||||
)
|
||||
if not page:
|
||||
break
|
||||
memories.extend(page)
|
||||
offset += len(page)
|
||||
|
||||
db = Path(db_path)
|
||||
ok = True
|
||||
|
||||
# --- FTS5: wipe and rebuild from primary store ---
|
||||
from ..memory.adapters.fts5 import FTS5TextIndex
|
||||
|
||||
# Construction initializes an absent optional FTS table. Cleanup
|
||||
# helpers, by contrast, intentionally treat an absent table as a no-op.
|
||||
fts = FTS5TextIndex(db_path=db_path)
|
||||
try:
|
||||
with sqlite3.connect(str(db)) as conn:
|
||||
conn.execute("DELETE FROM memory_fts")
|
||||
conn.commit()
|
||||
except Exception as exc:
|
||||
print_error(f"Failed to clear FTS5 index: {exc}")
|
||||
sys.exit(1)
|
||||
|
||||
fts_indexed = 0
|
||||
for mem in memories:
|
||||
try:
|
||||
asyncio.run(fts.index_memory(mem))
|
||||
fts_indexed += 1
|
||||
except Exception as exc:
|
||||
print_warning(f"FTS5: failed to index {mem.id[:8]}: {exc}")
|
||||
ok = False
|
||||
|
||||
# --- Vector: remove orphaned entries (requires sqlite-vec) ---
|
||||
vector_db = db.parent / f"{db.stem}_vectors.db"
|
||||
vector_msg = ""
|
||||
if vector_db.exists():
|
||||
# Orphan detection is based on every primary row, including
|
||||
# superseded memories that are intentionally omitted from FTS.
|
||||
with store._get_conn() as conn:
|
||||
primary_ids = {row[0] for row in conn.execute("SELECT id FROM memories")}
|
||||
try:
|
||||
with sqlite3.connect(str(vector_db)) as conn:
|
||||
if not _sqlite_table_exists(conn, "vec_metadata"):
|
||||
vector_msg = ", vector index not initialized"
|
||||
else:
|
||||
import sqlite_vec
|
||||
|
||||
conn.enable_load_extension(True)
|
||||
sqlite_vec.load(conn)
|
||||
conn.enable_load_extension(False)
|
||||
rows = conn.execute("SELECT memory_id FROM vec_metadata").fetchall()
|
||||
orphan_ids = [r[0] for r in rows if r[0] not in primary_ids]
|
||||
if orphan_ids:
|
||||
for i in range(0, len(orphan_ids), 500):
|
||||
chunk = orphan_ids[i : i + 500]
|
||||
ph = ",".join("?" * len(chunk))
|
||||
vec_rows = conn.execute(
|
||||
f"SELECT rowid FROM vec_metadata WHERE memory_id IN ({ph})",
|
||||
chunk,
|
||||
).fetchall()
|
||||
rowids = [r[0] for r in vec_rows]
|
||||
if rowids:
|
||||
rph = ",".join("?" * len(rowids))
|
||||
conn.execute(
|
||||
f"DELETE FROM vec_embeddings WHERE rowid IN ({rph})",
|
||||
rowids,
|
||||
)
|
||||
conn.execute(
|
||||
f"DELETE FROM vec_metadata WHERE rowid IN ({rph})",
|
||||
rowids,
|
||||
)
|
||||
conn.commit()
|
||||
vector_msg = (
|
||||
f", removed {len(orphan_ids)} orphaned vector entry(ies)"
|
||||
if orphan_ids
|
||||
else ", vector index clean"
|
||||
)
|
||||
except ImportError:
|
||||
vector_msg = (
|
||||
" (vector index skipped: sqlite-vec not installed — "
|
||||
"install with: pip install sqlite-vec)"
|
||||
)
|
||||
ok = False
|
||||
except Exception as exc:
|
||||
vector_msg = f" (vector index cleanup failed: {exc})"
|
||||
ok = False
|
||||
|
||||
msg = f"Re-indexed {fts_indexed}/{len(memories)} memories{vector_msg}."
|
||||
if ok:
|
||||
print_success(msg)
|
||||
else:
|
||||
print_warning(msg)
|
||||
sys.exit(1)
|
||||
|
||||
except Exception as e:
|
||||
print_error(f"Failed to reindex: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
@memory.command("export")
|
||||
@db_path_option
|
||||
@click.option(
|
||||
|
||||
@@ -0,0 +1,508 @@
|
||||
"""Tests for memory CLI index synchronization (issue #2856).
|
||||
|
||||
Verifies that headroom memory delete/prune/purge/edit remove stale entries
|
||||
from the FTS5 and vector search indexes, not just from the primary store.
|
||||
|
||||
Vector index tests require sqlite-vec and are skipped when it is not installed.
|
||||
They exercise the real SQLiteVectorIndex schema (vec0 virtual table) so that
|
||||
the extension-aware connection path in _remove_from_search_indexes and
|
||||
_clear_all_search_indexes is exercised rather than a plain-table stand-in.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import sqlite3
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
from click.testing import CliRunner
|
||||
|
||||
import headroom.cli.memory as memory_cli
|
||||
from headroom.cli.main import main
|
||||
from headroom.cli.memory import (
|
||||
_clear_all_search_indexes,
|
||||
_remove_from_search_indexes,
|
||||
)
|
||||
from headroom.memory.adapters.fts5 import FTS5TextIndex
|
||||
from headroom.memory.adapters.sqlite import SQLiteMemoryStore
|
||||
from headroom.memory.models import Memory
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# sqlite-vec availability guard
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
try:
|
||||
from headroom.memory.adapters.sqlite_vector import (
|
||||
SQLiteVectorIndex,
|
||||
is_sqlite_vec_available,
|
||||
)
|
||||
|
||||
SQLITE_VEC_AVAILABLE = is_sqlite_vec_available()
|
||||
except ImportError:
|
||||
SQLITE_VEC_AVAILABLE = False
|
||||
SQLiteVectorIndex = None # type: ignore[assignment,misc]
|
||||
|
||||
requires_sqlite_vec = pytest.mark.skipif(
|
||||
not SQLITE_VEC_AVAILABLE, reason="sqlite-vec not available"
|
||||
)
|
||||
|
||||
_VEC_DIM = 4 # small dimension keeps test seeding fast
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_memory(memory_id: str, content: str = "test content") -> Memory:
|
||||
return Memory(
|
||||
id=memory_id,
|
||||
content=content,
|
||||
user_id="test-user",
|
||||
)
|
||||
|
||||
|
||||
def _seed_fts(db_path: Path, memories: list[Memory]) -> None:
|
||||
"""Index memories into the FTS5 table."""
|
||||
fts = FTS5TextIndex(db_path=str(db_path))
|
||||
for mem in memories:
|
||||
asyncio.run(fts.index_memory(mem))
|
||||
|
||||
|
||||
def _seed_vector(db_path: Path, memory_ids: list[str]) -> None:
|
||||
"""Seed the vector DB using the real SQLiteVectorIndex (requires sqlite-vec).
|
||||
|
||||
Creates the true vec0 virtual-table schema so the helpers under test
|
||||
exercise the extension-aware connection path.
|
||||
"""
|
||||
vector_db = db_path.parent / f"{db_path.stem}_vectors.db"
|
||||
index = SQLiteVectorIndex(dimension=_VEC_DIM, db_path=str(vector_db))
|
||||
for mid in memory_ids:
|
||||
embedding = list(
|
||||
np.random.default_rng(abs(hash(mid))).standard_normal(_VEC_DIM).astype(float)
|
||||
)
|
||||
mem = Memory(id=mid, content="test", user_id="u", embedding=embedding)
|
||||
asyncio.run(index.index(mem))
|
||||
|
||||
|
||||
def _fts_count(db_path: Path) -> int:
|
||||
with sqlite3.connect(str(db_path)) as conn:
|
||||
return conn.execute("SELECT COUNT(*) FROM memory_fts").fetchone()[0]
|
||||
|
||||
|
||||
def _fts_ids(db_path: Path) -> set[str]:
|
||||
with sqlite3.connect(str(db_path)) as conn:
|
||||
rows = conn.execute("SELECT memory_id FROM memory_fts").fetchall()
|
||||
return {r[0] for r in rows}
|
||||
|
||||
|
||||
def _vector_ids(db_path: Path) -> set[str]:
|
||||
"""Read surviving memory_ids from the metadata table (regular, no extension needed)."""
|
||||
vector_db = db_path.parent / f"{db_path.stem}_vectors.db"
|
||||
if not vector_db.exists():
|
||||
return set()
|
||||
with sqlite3.connect(str(vector_db)) as conn:
|
||||
rows = conn.execute("SELECT memory_id FROM vec_metadata").fetchall()
|
||||
return {r[0] for r in rows}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _remove_from_search_indexes — FTS5 (no sqlite-vec required)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_remove_from_search_indexes_clears_fts_entries(tmp_path):
|
||||
db_path = tmp_path / "memory.db"
|
||||
mems = [_make_memory(f"id-{i}") for i in range(3)]
|
||||
_seed_fts(db_path, mems)
|
||||
assert _fts_count(db_path) == 3
|
||||
|
||||
_remove_from_search_indexes(str(db_path), ["id-0", "id-2"])
|
||||
|
||||
assert _fts_ids(db_path) == {"id-1"}
|
||||
|
||||
|
||||
def test_remove_from_search_indexes_no_vector_db_is_noop(tmp_path):
|
||||
db_path = tmp_path / "memory.db"
|
||||
mems = [_make_memory("id-0")]
|
||||
_seed_fts(db_path, mems)
|
||||
|
||||
# No vector DB → should not raise
|
||||
_remove_from_search_indexes(str(db_path), ["id-0"])
|
||||
|
||||
assert _fts_count(db_path) == 0
|
||||
|
||||
|
||||
def test_remove_from_search_indexes_empty_list_is_noop(tmp_path):
|
||||
db_path = tmp_path / "memory.db"
|
||||
_seed_fts(db_path, [_make_memory("id-0")])
|
||||
assert _fts_count(db_path) == 1
|
||||
|
||||
_remove_from_search_indexes(str(db_path), [])
|
||||
|
||||
assert _fts_count(db_path) == 1
|
||||
|
||||
|
||||
def test_remove_from_search_indexes_absent_optional_indexes_is_noop(tmp_path):
|
||||
"""A primary-only store must not fail after its mutation already succeeded."""
|
||||
db_path = tmp_path / "memory.db"
|
||||
store = SQLiteMemoryStore(str(db_path))
|
||||
asyncio.run(store.save(_make_memory("id-0")))
|
||||
|
||||
assert _remove_from_search_indexes(str(db_path), ["id-0"]) is True
|
||||
assert _clear_all_search_indexes(str(db_path)) is True
|
||||
|
||||
|
||||
def test_empty_uninitialized_vector_database_is_noop_without_sqlite_vec(tmp_path, monkeypatch):
|
||||
db_path = tmp_path / "memory.db"
|
||||
store = SQLiteMemoryStore(str(db_path))
|
||||
asyncio.run(store.save(_make_memory("id-0")))
|
||||
(tmp_path / "memory_vectors.db").touch()
|
||||
monkeypatch.setitem(sys.modules, "sqlite_vec", None)
|
||||
|
||||
assert _remove_from_search_indexes(str(db_path), ["id-0"]) is True
|
||||
assert _clear_all_search_indexes(str(db_path)) is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _remove_from_search_indexes — vector index (real vec0 schema, requires sqlite-vec)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@requires_sqlite_vec
|
||||
def test_remove_from_search_indexes_clears_vector_entries(tmp_path):
|
||||
"""Exercise the real vec0 virtual-table schema so the extension-aware
|
||||
connection path in _remove_from_search_indexes is covered."""
|
||||
db_path = tmp_path / "memory.db"
|
||||
_seed_fts(db_path, []) # ensure memory.db exists
|
||||
_seed_vector(db_path, ["id-0", "id-1", "id-2"])
|
||||
assert _vector_ids(db_path) == {"id-0", "id-1", "id-2"}
|
||||
|
||||
_remove_from_search_indexes(str(db_path), ["id-0", "id-2"])
|
||||
|
||||
assert _vector_ids(db_path) == {"id-1"}
|
||||
|
||||
|
||||
@requires_sqlite_vec
|
||||
def test_remove_from_search_indexes_no_vector_rows_to_delete_is_noop(tmp_path):
|
||||
"""IDs not present in the vector index must be silently skipped."""
|
||||
db_path = tmp_path / "memory.db"
|
||||
_seed_fts(db_path, [])
|
||||
_seed_vector(db_path, ["id-0"])
|
||||
assert _vector_ids(db_path) == {"id-0"}
|
||||
|
||||
_remove_from_search_indexes(str(db_path), ["id-99"]) # not in index
|
||||
|
||||
assert _vector_ids(db_path) == {"id-0"}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _clear_all_search_indexes — FTS5 (no sqlite-vec required)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_clear_all_search_indexes_removes_all_fts_entries(tmp_path):
|
||||
db_path = tmp_path / "memory.db"
|
||||
_seed_fts(db_path, [_make_memory(f"id-{i}") for i in range(5)])
|
||||
assert _fts_count(db_path) == 5
|
||||
|
||||
_clear_all_search_indexes(str(db_path))
|
||||
|
||||
assert _fts_count(db_path) == 0
|
||||
|
||||
|
||||
def test_clear_all_search_indexes_no_vector_db_is_noop(tmp_path):
|
||||
db_path = tmp_path / "memory.db"
|
||||
_seed_fts(db_path, [_make_memory("id-0")])
|
||||
|
||||
_clear_all_search_indexes(str(db_path))
|
||||
|
||||
assert _fts_count(db_path) == 0 # FTS cleared; no vector DB is fine
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _clear_all_search_indexes — vector index (real vec0 schema, requires sqlite-vec)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@requires_sqlite_vec
|
||||
def test_clear_all_search_indexes_removes_all_vector_entries(tmp_path):
|
||||
"""Exercise the real vec0 virtual-table schema so the extension-aware
|
||||
connection path in _clear_all_search_indexes is covered."""
|
||||
db_path = tmp_path / "memory.db"
|
||||
_seed_fts(db_path, [])
|
||||
_seed_vector(db_path, ["id-0", "id-1"])
|
||||
assert _vector_ids(db_path) == {"id-0", "id-1"}
|
||||
|
||||
_clear_all_search_indexes(str(db_path))
|
||||
|
||||
assert _vector_ids(db_path) == set()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Integration: CLI commands wire up index sync correctly
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_delete_command_removes_from_fts(tmp_path):
|
||||
"""Simulate delete command: delete_batch then _remove_from_search_indexes."""
|
||||
db_path = tmp_path / "memory.db"
|
||||
store = SQLiteMemoryStore(str(db_path))
|
||||
mem = _make_memory("abc123")
|
||||
asyncio.run(store.save(mem))
|
||||
_seed_fts(db_path, [mem])
|
||||
assert _fts_count(db_path) == 1
|
||||
|
||||
asyncio.run(store.delete_batch(["abc123"]))
|
||||
_remove_from_search_indexes(str(db_path), ["abc123"])
|
||||
|
||||
assert _fts_count(db_path) == 0
|
||||
|
||||
|
||||
@requires_sqlite_vec
|
||||
def test_delete_command_removes_from_vector_index(tmp_path):
|
||||
"""Simulate delete command end-to-end with the real vec0 schema."""
|
||||
db_path = tmp_path / "memory.db"
|
||||
store = SQLiteMemoryStore(str(db_path))
|
||||
mem = _make_memory("abc123")
|
||||
asyncio.run(store.save(mem))
|
||||
_seed_fts(db_path, [mem])
|
||||
_seed_vector(db_path, ["abc123"])
|
||||
assert _vector_ids(db_path) == {"abc123"}
|
||||
|
||||
asyncio.run(store.delete_batch(["abc123"]))
|
||||
_remove_from_search_indexes(str(db_path), ["abc123"])
|
||||
|
||||
assert _fts_count(db_path) == 0
|
||||
assert _vector_ids(db_path) == set()
|
||||
|
||||
|
||||
def test_purge_command_clears_fts(tmp_path):
|
||||
"""Simulate purge command: clear_all then _clear_all_search_indexes."""
|
||||
db_path = tmp_path / "memory.db"
|
||||
store = SQLiteMemoryStore(str(db_path))
|
||||
for i in range(3):
|
||||
asyncio.run(store.save(_make_memory(f"id-{i}")))
|
||||
_seed_fts(db_path, [_make_memory(f"id-{i}") for i in range(3)])
|
||||
assert _fts_count(db_path) == 3
|
||||
|
||||
asyncio.run(store.clear_all())
|
||||
_clear_all_search_indexes(str(db_path))
|
||||
|
||||
assert _fts_count(db_path) == 0
|
||||
|
||||
|
||||
@requires_sqlite_vec
|
||||
def test_purge_command_clears_vector_index(tmp_path):
|
||||
"""Simulate purge command end-to-end with the real vec0 schema."""
|
||||
db_path = tmp_path / "memory.db"
|
||||
store = SQLiteMemoryStore(str(db_path))
|
||||
for i in range(3):
|
||||
asyncio.run(store.save(_make_memory(f"id-{i}")))
|
||||
_seed_fts(db_path, [_make_memory(f"id-{i}") for i in range(3)])
|
||||
_seed_vector(db_path, [f"id-{i}" for i in range(3)])
|
||||
assert _vector_ids(db_path) == {"id-0", "id-1", "id-2"}
|
||||
|
||||
asyncio.run(store.clear_all())
|
||||
_clear_all_search_indexes(str(db_path))
|
||||
|
||||
assert _fts_count(db_path) == 0
|
||||
assert _vector_ids(db_path) == set()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Failure-path: return value and exit-code impact
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_remove_from_search_indexes_fts_failure_returns_false(tmp_path):
|
||||
"""When FTS5 delete raises, the function returns False (not True)."""
|
||||
db_path = tmp_path / "memory.db"
|
||||
_seed_fts(db_path, [_make_memory("id-0")])
|
||||
|
||||
# sqlite3 is imported locally inside the helper so we patch the global module.
|
||||
original_connect = sqlite3.connect
|
||||
call_count = [0]
|
||||
|
||||
def failing_connect(path, **kwargs):
|
||||
call_count[0] += 1
|
||||
if call_count[0] == 1: # first call is the FTS5 db open
|
||||
raise sqlite3.OperationalError("simulated FTS5 failure")
|
||||
return original_connect(path, **kwargs)
|
||||
|
||||
with patch("sqlite3.connect", side_effect=failing_connect):
|
||||
result = _remove_from_search_indexes(str(db_path), ["id-0"])
|
||||
|
||||
assert result is False
|
||||
|
||||
|
||||
def test_remove_from_search_indexes_sqlite_vec_missing_returns_false(tmp_path):
|
||||
"""When sqlite_vec is absent and a vector DB exists, returns False."""
|
||||
db_path = tmp_path / "memory.db"
|
||||
_seed_fts(db_path, [])
|
||||
# Create a non-empty vector DB file so the code doesn't short-circuit.
|
||||
vector_db = tmp_path / "memory_vectors.db"
|
||||
vector_db.write_bytes(b"placeholder")
|
||||
|
||||
# Remove sqlite_vec from sys.modules so `import sqlite_vec` raises ImportError.
|
||||
with patch.dict(sys.modules, {"sqlite_vec": None}):
|
||||
result = _remove_from_search_indexes(str(db_path), ["id-0"])
|
||||
|
||||
assert result is False
|
||||
|
||||
|
||||
def test_clear_all_search_indexes_fts_failure_returns_false(tmp_path):
|
||||
"""When FTS5 DELETE raises, _clear_all_search_indexes returns False."""
|
||||
db_path = tmp_path / "memory.db"
|
||||
_seed_fts(db_path, [_make_memory("id-0")])
|
||||
|
||||
original_connect = sqlite3.connect
|
||||
call_count = [0]
|
||||
|
||||
def failing_connect(path, **kwargs):
|
||||
call_count[0] += 1
|
||||
if call_count[0] == 1:
|
||||
raise sqlite3.OperationalError("simulated FTS5 failure")
|
||||
return original_connect(path, **kwargs)
|
||||
|
||||
with patch("sqlite3.connect", side_effect=failing_connect):
|
||||
result = _clear_all_search_indexes(str(db_path))
|
||||
|
||||
assert result is False
|
||||
|
||||
|
||||
def test_clear_all_search_indexes_sqlite_vec_missing_returns_false(tmp_path):
|
||||
"""When sqlite_vec is absent and a vector DB exists, returns False."""
|
||||
db_path = tmp_path / "memory.db"
|
||||
_seed_fts(db_path, [])
|
||||
vector_db = tmp_path / "memory_vectors.db"
|
||||
vector_db.write_bytes(b"placeholder")
|
||||
|
||||
with patch.dict(sys.modules, {"sqlite_vec": None}):
|
||||
result = _clear_all_search_indexes(str(db_path))
|
||||
|
||||
assert result is False
|
||||
|
||||
|
||||
def test_reindex_pages_through_complete_store(tmp_path, monkeypatch):
|
||||
"""Records beyond the first page remain represented in rebuilt FTS."""
|
||||
db_path = tmp_path / "memory.db"
|
||||
store = SQLiteMemoryStore(str(db_path))
|
||||
memories = [_make_memory(f"id-{i}", f"content {i}") for i in range(5)]
|
||||
for memory in memories:
|
||||
asyncio.run(store.save(memory))
|
||||
_seed_fts(db_path, memories[:2])
|
||||
monkeypatch.setattr(memory_cli, "_REINDEX_PAGE_SIZE", 2)
|
||||
|
||||
result = CliRunner().invoke(main, ["memory", "reindex", "--db-path", str(db_path)])
|
||||
|
||||
assert result.exit_code == 0, result.output
|
||||
assert _fts_ids(db_path) == {memory.id for memory in memories}
|
||||
assert "Re-indexed 5/5 memories" in result.output
|
||||
|
||||
|
||||
@requires_sqlite_vec
|
||||
def test_reindex_keeps_valid_vectors_beyond_first_page(tmp_path, monkeypatch):
|
||||
"""Complete primary IDs, not one page, determine vector orphans."""
|
||||
db_path = tmp_path / "memory.db"
|
||||
store = SQLiteMemoryStore(str(db_path))
|
||||
memories = [_make_memory(f"id-{i}", f"content {i}") for i in range(5)]
|
||||
for memory in memories:
|
||||
asyncio.run(store.save(memory))
|
||||
_seed_vector(db_path, [memory.id for memory in memories] + ["orphan"])
|
||||
monkeypatch.setattr(memory_cli, "_REINDEX_PAGE_SIZE", 2)
|
||||
|
||||
result = CliRunner().invoke(main, ["memory", "reindex", "--db-path", str(db_path)])
|
||||
|
||||
assert result.exit_code == 0, result.output
|
||||
assert _vector_ids(db_path) == {memory.id for memory in memories}
|
||||
|
||||
|
||||
def test_delete_command_exits_nonzero_when_index_sync_fails(tmp_path, monkeypatch):
|
||||
"""The real Click command must not report a partially synced delete as success."""
|
||||
db_path = tmp_path / "memory.db"
|
||||
store = SQLiteMemoryStore(str(db_path))
|
||||
memory = _make_memory("abc123")
|
||||
asyncio.run(store.save(memory))
|
||||
monkeypatch.setattr(memory_cli, "_remove_from_search_indexes", lambda *_args: False)
|
||||
|
||||
result = CliRunner().invoke(
|
||||
main,
|
||||
["memory", "delete", memory.id, "--force", "--db-path", str(db_path)],
|
||||
)
|
||||
|
||||
assert result.exit_code == 1
|
||||
assert asyncio.run(store.get(memory.id)) is None
|
||||
assert "index sync incomplete" in result.output
|
||||
|
||||
|
||||
def test_edit_command_exits_nonzero_when_index_sync_fails(tmp_path, monkeypatch):
|
||||
db_path = tmp_path / "memory.db"
|
||||
store = SQLiteMemoryStore(str(db_path))
|
||||
memory = _make_memory("abc123", "before")
|
||||
asyncio.run(store.save(memory))
|
||||
monkeypatch.setattr(memory_cli, "_remove_from_search_indexes", lambda *_args: False)
|
||||
|
||||
result = CliRunner().invoke(
|
||||
main,
|
||||
[
|
||||
"memory",
|
||||
"edit",
|
||||
memory.id,
|
||||
"--content",
|
||||
"after",
|
||||
"--db-path",
|
||||
str(db_path),
|
||||
],
|
||||
)
|
||||
|
||||
assert result.exit_code == 1
|
||||
assert asyncio.run(store.get(memory.id)).content == "after"
|
||||
assert "index sync incomplete" in result.output
|
||||
|
||||
|
||||
def test_prune_command_exits_nonzero_when_index_sync_fails(tmp_path, monkeypatch):
|
||||
db_path = tmp_path / "memory.db"
|
||||
store = SQLiteMemoryStore(str(db_path))
|
||||
memory = _make_memory("abc123")
|
||||
asyncio.run(store.save(memory))
|
||||
monkeypatch.setattr(memory_cli, "_remove_from_search_indexes", lambda *_args: False)
|
||||
|
||||
result = CliRunner().invoke(
|
||||
main,
|
||||
[
|
||||
"memory",
|
||||
"prune",
|
||||
"--low-importance",
|
||||
"1.0",
|
||||
"--force",
|
||||
"--db-path",
|
||||
str(db_path),
|
||||
],
|
||||
)
|
||||
|
||||
assert result.exit_code == 1
|
||||
assert asyncio.run(store.get(memory.id)) is None
|
||||
assert "index sync incomplete" in result.output
|
||||
|
||||
|
||||
def test_purge_command_exits_nonzero_when_index_sync_fails(tmp_path, monkeypatch):
|
||||
db_path = tmp_path / "memory.db"
|
||||
store = SQLiteMemoryStore(str(db_path))
|
||||
memory = _make_memory("abc123")
|
||||
asyncio.run(store.save(memory))
|
||||
monkeypatch.setattr(memory_cli, "_clear_all_search_indexes", lambda *_args: False)
|
||||
|
||||
result = CliRunner().invoke(
|
||||
main,
|
||||
["memory", "purge", "--confirm", "--db-path", str(db_path)],
|
||||
input="y\n",
|
||||
)
|
||||
|
||||
assert result.exit_code == 1
|
||||
assert asyncio.run(store.get(memory.id)) is None
|
||||
assert "index sync incomplete" in result.output
|
||||
Reference in New Issue
Block a user