fix: resolve all clippy lints and verify tests passing

This commit is contained in:
pankaj
2026-01-23 20:54:00 +05:45
parent 3461877851
commit 9a7b3c2275
80 changed files with 871 additions and 677 deletions
+10 -1
View File
@@ -76,10 +76,19 @@ git commit -m "docs: update README examples"
- Follow standard Rust idioms and conventions
- Use `rustfmt` for formatting (`cargo fmt`)
- Use `clippy` for linting (`cargo clippy`)
- Use `clippy` for linting (`cargo clippy`). We maintain a **zero-warning policy**.
- Prefer explicit types for public APIs
- Use `thiserror` for error definitions
### Linting & Safety
We enforce strict linting to ensure safety and portability:
1. **Zero Warnings**: CI will fail on any warning. Run `cargo clippy --workspace --all-targets -- -D warnings` locally.
2. **No Panics**: `unwrap()` and `expect()` are **denied** in library code. Use `Result` propagation (`?`) or graceful error handling. They are allowed in `tests/`.
3. **No Truncation**: `cast_possible_truncation` is denied. Use `try_from` when converting `u64` to `usize`/`u32`.
4. **Exceptions**: We allow pragmatic lints (e.g., `cast_precision_loss` for ML math) in `src/lib.rs`. Do not add global `#![allow]` without discussion.
### Documentation
- Add doc comments (`///`) to all public functions, structs, and modules
+1 -1
View File
@@ -39,7 +39,7 @@ fn setup_corpus(size: usize) -> std::path::PathBuf {
let content = format!("Document {} about {}", i, topics[i % topics.len()]);
mem.put_bytes_with_options(
content.as_bytes(),
PutOptions::builder().title(&format!("Doc {}", i)).build(),
PutOptions::builder().title(format!("Doc {}", i)).build(),
)
.unwrap();
+1 -1
View File
@@ -27,7 +27,7 @@ fn main() -> memvid_core::Result<()> {
{
eprintln!("This example requires the 'clip' feature.");
eprintln!("Run with: cargo run --example clip_visual_search --features clip");
return Ok(());
Ok(())
}
#[cfg(feature = "clip")]
+1 -1
View File
@@ -33,7 +33,7 @@ fn main() -> memvid_core::Result<()> {
mem.put_bytes_with_options(
content.as_bytes(),
PutOptions::builder()
.title(&format!("Doc {} - {}", i, topic.0))
.title(format!("Doc {} - {}", i, topic.0))
.build(),
)?;
+1 -1
View File
@@ -63,7 +63,7 @@ fn main() -> Result<()> {
let options = PutOptions::builder()
.title(&title)
.uri(&format!(
.uri(format!(
"mv2://pdfs/{}",
pdf_path.file_name().unwrap_or_default().to_string_lossy()
))
+75 -72
View File
@@ -40,75 +40,77 @@ impl AutoTagger {
}
fn extract_keywords(text: &str, limit: usize) -> Vec<String> {
static TOKEN_RE: std::sync::LazyLock<Regex> = std::sync::LazyLock::new(|| Regex::new(r"(?i)[a-z0-9][a-z0-9'-]+").unwrap());
static STOPWORDS: std::sync::LazyLock<BTreeSet<&'static str>> = std::sync::LazyLock::new(|| {
[
"the",
"and",
"for",
"with",
"that",
"from",
"this",
"were",
"have",
"has",
"will",
"shall",
"into",
"about",
"without",
"within",
"between",
"because",
"over",
"under",
"after",
"before",
"until",
"while",
"their",
"there",
"these",
"those",
"your",
"into",
"such",
"been",
"where",
"when",
"which",
"using",
"also",
"than",
"could",
"would",
"should",
"might",
"cannot",
"however",
"therefore",
"thereof",
"hereby",
"herein",
"hereof",
"based",
"system",
"application",
"service",
"provide",
"provided",
"including",
"include",
"includes",
"version",
"update",
"updates",
"usage",
]
.into_iter()
.collect()
});
static TOKEN_RE: std::sync::LazyLock<Regex> =
std::sync::LazyLock::new(|| Regex::new(r"(?i)[a-z0-9][a-z0-9'-]+").unwrap());
static STOPWORDS: std::sync::LazyLock<BTreeSet<&'static str>> =
std::sync::LazyLock::new(|| {
[
"the",
"and",
"for",
"with",
"that",
"from",
"this",
"were",
"have",
"has",
"will",
"shall",
"into",
"about",
"without",
"within",
"between",
"because",
"over",
"under",
"after",
"before",
"until",
"while",
"their",
"there",
"these",
"those",
"your",
"into",
"such",
"been",
"where",
"when",
"which",
"using",
"also",
"than",
"could",
"would",
"should",
"might",
"cannot",
"however",
"therefore",
"thereof",
"hereby",
"herein",
"hereof",
"based",
"system",
"application",
"service",
"provide",
"provided",
"including",
"include",
"includes",
"version",
"update",
"updates",
"usage",
]
.into_iter()
.collect()
});
let mut counts: BTreeMap<String, u32> = BTreeMap::new();
for token in TOKEN_RE.find_iter(text) {
@@ -132,8 +134,9 @@ fn extract_keywords(text: &str, limit: usize) -> Vec<String> {
}
fn derive_labels(text: &str, limit: usize) -> Vec<String> {
static PHRASE_RE: std::sync::LazyLock<Regex> =
std::sync::LazyLock::new(|| Regex::new(r"(?m)^(?P<phrase>[A-Z][A-Za-z0-9 &/-]{3,})$").unwrap());
static PHRASE_RE: std::sync::LazyLock<Regex> = std::sync::LazyLock::new(|| {
Regex::new(r"(?m)^(?P<phrase>[A-Z][A-Za-z0-9 &/-]{3,})$").unwrap()
});
let mut labels = BTreeSet::new();
for caps in PHRASE_RE.captures_iter(text) {
@@ -218,7 +221,7 @@ mod tests {
#[test]
fn produces_keywords_and_labels() {
let text = "Rust memory engines power efficient systems. Memory safety ensures reliability in 2025.";
let result = AutoTagger::default().analyse(text, true);
let result = AutoTagger.analyse(text, true);
assert!(result.tags.iter().any(|tag| tag.contains("memory")));
assert!(!result.content_dates.is_empty());
}
+6 -6
View File
@@ -84,13 +84,13 @@ pub static NER_MODELS: &[NerModelInfo] = &[NerModelInfo {
}];
/// Get NER model info by name
#[must_use]
#[must_use]
pub fn get_ner_model_info(name: &str) -> Option<&'static NerModelInfo> {
NER_MODELS.iter().find(|m| m.name == name)
}
/// Get default NER model info
#[must_use]
#[must_use]
pub fn default_ner_model_info() -> &'static NerModelInfo {
NER_MODELS
.iter()
@@ -119,7 +119,7 @@ pub struct ExtractedEntity {
impl ExtractedEntity {
/// Convert the raw entity type to our `EntityKind` enum
#[must_use]
#[must_use]
pub fn to_entity_kind(&self) -> EntityKind {
match self.entity_type.to_uppercase().as_str() {
"PER" | "PERSON" | "B-PER" | "I-PER" => EntityKind::Person,
@@ -562,19 +562,19 @@ impl NerModel {
// ============================================================================
/// Get the expected path for the NER model in the models directory
#[must_use]
#[must_use]
pub fn ner_model_path(models_dir: &Path) -> PathBuf {
models_dir.join(NER_MODEL_NAME).join("model.onnx")
}
/// Get the expected path for the NER tokenizer in the models directory
#[must_use]
#[must_use]
pub fn ner_tokenizer_path(models_dir: &Path) -> PathBuf {
models_dir.join(NER_MODEL_NAME).join("tokenizer.json")
}
/// Check if NER model is installed
#[must_use]
#[must_use]
pub fn is_ner_model_installed(models_dir: &Path) -> bool {
ner_model_path(models_dir).exists() && ner_tokenizer_path(models_dir).exists()
}
+12 -11
View File
@@ -35,6 +35,7 @@ use crate::{MemvidError, Result, types::FrameId};
// ============================================================================
/// CLIP index decode limit (512MB max)
#[allow(clippy::cast_possible_truncation)]
const CLIP_DECODE_LIMIT: usize = crate::MAX_INDEX_BYTES as usize;
/// MobileCLIP-S2 embedding dimensions
@@ -140,7 +141,7 @@ pub static CLIP_MODELS: &[ClipModelInfo] = &[
];
/// Get model info by name, defaults to mobileclip-s2
#[must_use]
#[must_use]
pub fn get_model_info(name: &str) -> &'static ClipModelInfo {
CLIP_MODELS
.iter()
@@ -154,7 +155,7 @@ pub fn get_model_info(name: &str) -> &'static ClipModelInfo {
}
/// Get the default model info
#[must_use]
#[must_use]
pub fn default_model_info() -> &'static ClipModelInfo {
CLIP_MODELS
.iter()
@@ -185,7 +186,7 @@ pub struct ClipIndexBuilder {
}
impl ClipIndexBuilder {
#[must_use]
#[must_use]
pub fn new() -> Self {
Self::default()
}
@@ -210,7 +211,7 @@ impl ClipIndexBuilder {
let dimension = self
.documents
.first()
.map_or(0, |doc| doc.embedding.len() as u32);
.map_or(0, |doc| u32::try_from(doc.embedding.len()).unwrap_or(0));
Ok(ClipIndexArtifact {
bytes,
@@ -248,7 +249,7 @@ impl Default for ClipIndex {
impl ClipIndex {
/// Create a new empty CLIP index
#[must_use]
#[must_use]
pub fn new() -> Self {
Self {
documents: Vec::new(),
@@ -297,7 +298,7 @@ impl ClipIndex {
}
/// Search for similar embeddings using L2 distance
#[must_use]
#[must_use]
pub fn search(&self, query: &[f32], limit: usize) -> Vec<ClipSearchHit> {
if query.is_empty() {
return Vec::new();
@@ -333,7 +334,7 @@ impl ClipIndex {
}
/// Get embedding for a specific frame
#[must_use]
#[must_use]
pub fn embedding_for(&self, frame_id: FrameId) -> Option<&[f32]> {
self.documents
.iter()
@@ -347,13 +348,13 @@ impl ClipIndex {
}
/// Number of documents in the index
#[must_use]
#[must_use]
pub fn len(&self) -> usize {
self.documents.len()
}
/// Check if index is empty
#[must_use]
#[must_use]
pub fn is_empty(&self) -> bool {
self.documents.is_empty()
}
@@ -366,7 +367,7 @@ impl ClipIndex {
let dimension = self
.documents
.first()
.map_or(0, |doc| doc.embedding.len() as u32);
.map_or(0, |doc| u32::try_from(doc.embedding.len()).unwrap_or(0));
Ok(ClipIndexArtifact {
bytes,
@@ -411,7 +412,7 @@ pub struct ImageInfo {
impl ImageInfo {
/// Check if this image should be processed for CLIP embedding
#[must_use]
#[must_use]
pub fn should_embed(&self) -> bool {
// Skip tiny images (icons, bullets)
if self.width < MIN_IMAGE_DIM || self.height < MIN_IMAGE_DIM {
+2 -2
View File
@@ -149,10 +149,10 @@ mod tests {
struct TestEngine;
impl EnrichmentEngine for TestEngine {
fn kind(&self) -> &str {
fn kind(&self) -> &'static str {
"test"
}
fn version(&self) -> &str {
fn version(&self) -> &'static str {
"1.0.0"
}
fn enrich(&self, _ctx: &EnrichmentContext) -> EnrichmentResult {
+14 -10
View File
@@ -72,7 +72,7 @@ pub struct EnrichmentWorkerHandle {
impl EnrichmentWorkerHandle {
/// Create a new worker handle.
#[must_use]
#[must_use]
pub fn new() -> Self {
Self {
stop_signal: Arc::new(AtomicBool::new(false)),
@@ -90,19 +90,19 @@ impl EnrichmentWorkerHandle {
}
/// Check if stop was requested.
#[must_use]
#[must_use]
pub fn should_stop(&self) -> bool {
self.stop_signal.load(Ordering::SeqCst)
}
/// Check if worker is currently running.
#[must_use]
#[must_use]
pub fn is_running(&self) -> bool {
self.is_running.load(Ordering::SeqCst)
}
/// Get current statistics.
#[must_use]
#[must_use]
pub fn stats(&self) -> EnrichmentWorkerStats {
EnrichmentWorkerStats {
frames_processed: self.frames_processed.load(Ordering::Relaxed),
@@ -141,7 +141,7 @@ impl EnrichmentWorkerHandle {
}
/// Clone the handle for sharing with the worker thread.
#[must_use]
#[must_use]
pub fn clone_handle(&self) -> Self {
Self {
stop_signal: Arc::clone(&self.stop_signal),
@@ -266,7 +266,7 @@ pub struct EnrichmentProcessor {
impl EnrichmentProcessor {
/// Create a new enrichment processor.
#[must_use]
#[must_use]
pub fn new(config: EnrichmentWorkerConfig) -> Self {
Self { config }
}
@@ -306,9 +306,11 @@ impl EnrichmentProcessor {
};
// Read current frame state
let (text, is_skim, _needs_embedding) = if let Some(data) = read_frame(task.frame_id) { data } else {
let (text, is_skim, _needs_embedding) = if let Some(data) = read_frame(task.frame_id) {
data
} else {
result.error = Some("Frame not found".to_string());
result.elapsed_ms = start.elapsed().as_millis() as u64;
result.elapsed_ms = start.elapsed().as_millis().try_into().unwrap_or(u64::MAX);
return result;
};
@@ -337,7 +339,7 @@ impl EnrichmentProcessor {
result.error = Some(format!("Index update failed: {err}"));
}
result.elapsed_ms = start.elapsed().as_millis() as u64;
result.elapsed_ms = start.elapsed().as_millis().try_into().unwrap_or(u64::MAX);
result
}
}
@@ -374,7 +376,9 @@ pub fn run_worker_loop<G, P, M, C>(
while !handle.should_stop() {
// Get next task
let task = if let Some(task) = get_next_task() { task } else {
let task = if let Some(task) = get_next_task() {
task
} else {
// Queue is empty, wait and check again
std::thread::sleep(Duration::from_millis(config.task_delay_ms * 10));
continue;
+7 -8
View File
@@ -30,7 +30,7 @@ pub struct ExtractedDocument {
}
impl ExtractedDocument {
#[must_use]
#[must_use]
pub fn empty() -> Self {
Self {
text: None,
@@ -285,7 +285,7 @@ impl Default for DocumentProcessor {
#[cfg(not(feature = "extractous"))]
impl DocumentProcessor {
#[must_use]
#[must_use]
pub fn new(config: ProcessorConfig) -> Self {
Self {
max_length: config.max_text_chars,
@@ -790,12 +790,11 @@ fn pdf_text_extract_lopdf(bytes: &[u8]) -> Result<Option<String>> {
})?;
// Try to decrypt if encrypted (empty password for unprotected PDFs)
if document.is_encrypted()
&& document.decrypt("").is_err() {
return Err(MemvidError::ExtractionFailed {
reason: "cannot decrypt password-protected PDF".into(),
});
}
if document.is_encrypted() && document.decrypt("").is_err() {
return Err(MemvidError::ExtractionFailed {
reason: "cannot decrypt password-protected PDF".into(),
});
}
// Decompress streams for better text extraction
let () = document.decompress();
+16 -17
View File
@@ -43,7 +43,7 @@ impl Default for ExtractionBudget {
impl ExtractionBudget {
/// Create a budget with custom milliseconds.
#[must_use]
#[must_use]
pub fn with_ms(ms: u64) -> Self {
Self {
budget: Duration::from_millis(ms),
@@ -52,7 +52,7 @@ impl ExtractionBudget {
}
/// Create an unlimited budget (extract everything).
#[must_use]
#[must_use]
pub fn unlimited() -> Self {
Self {
budget: Duration::from_secs(3600), // 1 hour = effectively unlimited
@@ -81,13 +81,13 @@ pub struct BudgetedExtractionResult {
impl BudgetedExtractionResult {
/// Check if we got meaningful content.
#[must_use]
#[must_use]
pub fn has_content(&self) -> bool {
!self.text.trim().is_empty()
}
/// Check if this is a skim (partial) extraction.
#[must_use]
#[must_use]
pub fn is_skim(&self) -> bool {
!self.completed && self.sections_extracted < self.sections_total
}
@@ -138,7 +138,7 @@ pub fn extract_pdf_budgeted(
sections_extracted: estimated_pages,
sections_total: estimated_pages,
completed,
elapsed_ms: start.elapsed().as_millis() as u64,
elapsed_ms: start.elapsed().as_millis().try_into().unwrap_or(u64::MAX),
coverage: 1.0,
});
}
@@ -180,7 +180,7 @@ pub fn extract_pdf_budgeted(
sections_extracted: estimated_pages,
sections_total: estimated_pages,
completed,
elapsed_ms: start.elapsed().as_millis() as u64,
elapsed_ms: start.elapsed().as_millis().try_into().unwrap_or(u64::MAX),
coverage: 1.0,
});
}
@@ -233,12 +233,11 @@ fn extract_pdf_budgeted_lopdf(
})?;
// Handle encryption
if document.is_encrypted()
&& document.decrypt("").is_err() {
return Err(MemvidError::ExtractionFailed {
reason: "cannot decrypt password-protected PDF".into(),
});
}
if document.is_encrypted() && document.decrypt("").is_err() {
return Err(MemvidError::ExtractionFailed {
reason: "cannot decrypt password-protected PDF".into(),
});
}
// Decompress for better extraction
let () = document.decompress();
@@ -251,7 +250,7 @@ fn extract_pdf_budgeted_lopdf(
sections_extracted: 0,
sections_total: 0,
completed: true,
elapsed_ms: start.elapsed().as_millis() as u64,
elapsed_ms: start.elapsed().as_millis().try_into().unwrap_or(u64::MAX),
coverage: 1.0,
});
}
@@ -360,7 +359,7 @@ pub fn extract_text_budgeted(
sections_extracted: sections,
sections_total: sections,
completed: true,
elapsed_ms: start.elapsed().as_millis() as u64,
elapsed_ms: start.elapsed().as_millis().try_into().unwrap_or(u64::MAX),
coverage: 1.0,
})
}
@@ -429,7 +428,7 @@ fn extract_ooxml_budgeted(
sections_extracted: sections,
sections_total: sections,
completed: true,
elapsed_ms: start.elapsed().as_millis() as u64,
elapsed_ms: start.elapsed().as_millis().try_into().unwrap_or(u64::MAX),
coverage: 1.0,
})
}
@@ -442,7 +441,7 @@ fn extract_ooxml_budgeted(
sections_extracted: 0,
sections_total: 0,
completed: true,
elapsed_ms: start.elapsed().as_millis() as u64,
elapsed_ms: start.elapsed().as_millis().try_into().unwrap_or(u64::MAX),
coverage: 1.0,
})
}
@@ -588,7 +587,7 @@ fn finish_extraction(
sections_extracted,
sections_total: total_pages,
completed,
elapsed_ms: start.elapsed().as_millis() as u64,
elapsed_ms: start.elapsed().as_millis().try_into().unwrap_or(u64::MAX),
coverage,
})
}
+1 -1
View File
@@ -91,7 +91,7 @@ pub fn find_last_valid_footer(bytes: &[u8]) -> Option<FooterSlice<'_>> {
let candidate = &bytes[pos..pos + FOOTER_SIZE];
if let Some(footer) = CommitFooter::decode(candidate) {
let toc_end = pos;
let toc_len = footer.toc_len as usize;
let toc_len = usize::try_from(footer.toc_len).unwrap_or(0);
if toc_len == 0 || toc_len > toc_end {
search_end = pos;
continue;
+2 -2
View File
@@ -193,13 +193,13 @@ pub struct GraphMatcher<'a> {
impl<'a> GraphMatcher<'a> {
/// Create a new graph matcher.
#[must_use]
#[must_use]
pub fn new(memvid: &'a Memvid) -> Self {
Self { memvid }
}
/// Execute a graph pattern and return matching results.
#[must_use]
#[must_use]
pub fn execute(&self, pattern: &GraphPattern) -> Vec<GraphMatchResult> {
let mut results = Vec::new();
+2 -1
View File
@@ -102,7 +102,8 @@ impl ManifestWal {
let checksum = hash(&payload);
self.file.seek(SeekFrom::Start(self.write_offset))?;
// Safe: validated payload.len() <= MAX_RECORD_BYTES (4MB) on line 96
self.file.write_all(&(payload.len() as u32).to_le_bytes())?;
self.file
.write_all(&(u32::try_from(payload.len()).unwrap_or(u32::MAX)).to_le_bytes())?;
self.file.write_all(checksum.as_bytes())?;
self.file.write_all(&payload)?;
+1
View File
@@ -96,6 +96,7 @@ pub fn read_track<R: Read + Seek>(
}
// Safe: count validated by checked_mul and payload_bytes comparison above
#[allow(clippy::cast_possible_truncation)]
let mut entries = Vec::with_capacity(count as usize);
let mut prev: Option<TimeIndexEntry> = None;
for _ in 0..count {
+6 -4
View File
@@ -231,7 +231,8 @@ impl EmbeddedWal {
let digest = blake3::hash(payload);
let mut header = [0u8; ENTRY_HEADER_SIZE];
header[..8].copy_from_slice(&sequence.to_le_bytes());
header[8..12].copy_from_slice(&(payload.len() as u32).to_le_bytes());
header[8..12]
.copy_from_slice(&(u32::try_from(payload.len()).unwrap_or(u32::MAX)).to_le_bytes());
header[16..48].copy_from_slice(digest.as_bytes());
// Atomic write: combine header and payload into single buffer
@@ -264,6 +265,7 @@ impl EmbeddedWal {
if remaining < ENTRY_HEADER_SIZE as u64 {
if remaining > 0 {
// Safe: remaining < ENTRY_HEADER_SIZE (48) so always fits in usize
#[allow(clippy::cast_possible_truncation)]
let zero_tail = vec![0u8; remaining as usize];
self.seek_and_write(pos, &zero_tail)?;
}
@@ -458,13 +460,13 @@ mod tests {
let (file, mut header) = prepare_wal(size);
let mut wal = EmbeddedWal::open(&file, &header).expect("open wal");
wal.append_entry(&vec![0xAA; 32]).expect("append a");
wal.append_entry(&vec![0xBB; 32]).expect("append b");
wal.append_entry(&[0xAA; 32]).expect("append a");
wal.append_entry(&[0xBB; 32]).expect("append b");
wal.record_checkpoint(&mut header).expect("checkpoint");
assert!(wal.pending_records().expect("pending").is_empty());
wal.append_entry(&vec![0xCC; 32]).expect("append c");
wal.append_entry(&[0xCC; 32]).expect("append c");
let records = wal.pending_records().expect("after append");
assert_eq!(records.len(), 1);
assert_eq!(records[0].payload, vec![0xCC; 32]);
+4 -3
View File
@@ -15,6 +15,7 @@ fn lex_config() -> impl bincode::config::Config {
.with_little_endian()
}
#[allow(clippy::cast_possible_truncation)]
const LEX_DECODE_LIMIT: usize = crate::MAX_INDEX_BYTES as usize;
const LEX_SECTION_SOFT_CHARS: usize = 900;
const LEX_SECTION_HARD_CHARS: usize = 1400;
@@ -27,7 +28,7 @@ pub struct LexIndexBuilder {
}
impl LexIndexBuilder {
#[must_use]
#[must_use]
pub fn new() -> Self {
Self::default()
}
@@ -150,7 +151,7 @@ impl LexIndex {
Self { documents }
}
#[must_use]
#[must_use]
pub fn search(&self, query: &str, limit: usize) -> Vec<LexSearchHit> {
let mut query_tokens = tokenize(query);
query_tokens.retain(|token| !token.is_empty());
@@ -684,7 +685,7 @@ mod tests {
let artifact = builder.finish().expect("finish");
assert_eq!(artifact.doc_count, 2);
assert!(artifact.bytes.len() > 0);
assert!(!artifact.bytes.is_empty());
let index = LexIndex::decode(&artifact.bytes).expect("decode");
let hits = index.search("rust", 10);
+16 -6
View File
@@ -1,5 +1,15 @@
#![deny(clippy::all, clippy::pedantic)]
#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used))]
#![cfg_attr(
test,
allow(
clippy::useless_vec,
clippy::uninlined_format_args,
clippy::cast_possible_truncation,
clippy::float_cmp,
clippy::cast_precision_loss
)
)]
#![allow(clippy::module_name_repetitions)]
//
// Strategic lint exceptions - these are allowed project-wide for pragmatic reasons:
@@ -13,7 +23,6 @@
// Cast safety: All casts in this codebase are carefully reviewed and bounded by
// real-world constraints (file sizes, frame counts, etc). Using try_into() everywhere
// would add significant complexity without safety benefits in our use case.
#![allow(clippy::cast_possible_truncation)]
#![allow(clippy::cast_precision_loss)]
#![allow(clippy::cast_possible_wrap)]
#![allow(clippy::cast_sign_loss)]
@@ -322,6 +331,7 @@ const MAX_FRAME_BYTES: u64 = 256 * 1024 * 1024;
const DEFAULT_SEARCH_TEXT_LIMIT: usize = 32_768;
#[cfg(test)]
#[allow(clippy::non_std_lazy_statics)]
static SERIAL_TEST_MUTEX: Lazy<Mutex<()>> = Lazy::new(|| Mutex::new(()));
#[cfg(test)]
@@ -1160,7 +1170,7 @@ mod tests {
.expect("open file");
file.seek(SeekFrom::Start(manifest.bytes_offset))
.expect("seek");
let zeros = vec![0u8; manifest.bytes_length as usize];
let zeros = vec![0u8; usize::try_from(manifest.bytes_length).unwrap_or(0)];
file.write_all(&zeros).expect("corrupt time index");
file.flush().expect("flush");
file.sync_all().expect("sync");
@@ -1178,7 +1188,7 @@ mod tests {
assert_eq!(report.overall_status, VerificationStatus::Failed);
}
Err(e) => {
println!("test: verify failed with error (expected): {}", e);
println!("test: verify failed with error (expected): {e}");
}
}
@@ -1330,6 +1340,7 @@ mod tests {
}
#[test]
#[allow(deprecated)]
fn ticket_sequence_enforced() {
run_serial_test(|| {
let dir = tempdir().expect("tmp");
@@ -1347,6 +1358,7 @@ mod tests {
}
#[test]
#[allow(deprecated)]
fn capacity_limit_enforced() {
run_serial_test(|| {
let dir = tempdir().expect("tmp");
@@ -1360,9 +1372,7 @@ mod tests {
mem.put_bytes(&vec![0xFF; 32]).expect("first put");
mem.commit().expect("commit");
let err = mem
.put_bytes(&vec![0xFF; 40])
.expect_err("capacity exceeded");
let err = mem.put_bytes(&[0xFF; 40]).expect_err("capacity exceeded");
assert!(matches!(err, MemvidError::CapacityExceeded { .. }));
});
}
+1 -1
View File
@@ -81,7 +81,7 @@ impl FileLock {
Ok(self.file.try_clone()?)
}
#[must_use]
#[must_use]
pub fn mode(&self) -> LockMode {
self.mode
}
+9 -4
View File
@@ -502,12 +502,17 @@ impl Memvid {
.map(|manifest| manifest.dimension)
.filter(|dim| *dim > 0)
.or_else(|| {
self.vec_index
.as_ref()
.and_then(|index| index.entries().next().map(|(_, emb)| emb.len() as u32))
self.vec_index.as_ref().and_then(|index| {
index
.entries()
.next()
.map(|(_, emb)| u32::try_from(emb.len()).unwrap_or(0))
})
})
.unwrap_or(0);
if stored_dimension > 0 && query_embedding.len() as u32 != stored_dimension {
if stored_dimension > 0
&& u32::try_from(query_embedding.len()).unwrap_or(u32::MAX) != stored_dimension
{
return Err(MemvidError::VecDimensionMismatch {
expected: stored_dimension,
actual: query_embedding.len(),
+4 -4
View File
@@ -297,7 +297,7 @@ mod tests {
#[test]
fn structural_chunking_keeps_small_table_whole() {
let text = r#"# Small Report
let text = r"# Small Report
Introduction paragraph.
@@ -307,7 +307,7 @@ Introduction paragraph.
| Orange | $2 |
Conclusion.
"#
"
.repeat(50); // Repeat to meet minimum size
let plan = plan_document_chunks(text.as_bytes()).expect("chunk plan");
@@ -327,7 +327,7 @@ Conclusion.
#[test]
fn structural_chunking_detects_code_blocks() {
let text = r#"# Code Example
let text = r"# Code Example
Here is some code:
@@ -347,7 +347,7 @@ class DataProcessor:
self.data.append(item)
```
More explanation here. "#
More explanation here. "
.repeat(20);
let plan = plan_document_chunks(text.as_bytes()).expect("chunk plan");
+12 -4
View File
@@ -106,6 +106,7 @@ fn try_recover_from_wal_corruption(path: &Path) -> Result<Memvid> {
);
// Zero out the entire WAL region to create a clean slate
#[allow(clippy::cast_possible_truncation)]
let wal_size = header.wal_size as usize;
let zeros = vec![0u8; min(1024 * 1024, wal_size)]; // Write in 1MB chunks
let mut written = 0;
@@ -619,6 +620,7 @@ impl DoctorPlanner {
));
return;
}
#[allow(clippy::cast_possible_truncation)]
let mut buf = vec![0u8; manifest.bytes_length as usize];
if let Err(err) = file.seek(SeekFrom::Start(manifest.bytes_offset)) {
probe.index.needs_lex = true;
@@ -728,6 +730,7 @@ impl DoctorPlanner {
}
// Read and validate segment
#[allow(clippy::cast_possible_truncation)]
let mut buf = vec![0u8; segment.common.bytes_length as usize];
if let Err(err) = file.seek(SeekFrom::Start(segment.common.bytes_offset)) {
probe.index.needs_vec = true;
@@ -799,6 +802,7 @@ impl DoctorPlanner {
return;
}
#[allow(clippy::cast_possible_truncation)]
let mut buf = vec![0u8; manifest.bytes_length as usize];
if let Err(err) = file.seek(SeekFrom::Start(manifest.bytes_offset)) {
probe.index.needs_vec = true;
@@ -1225,7 +1229,11 @@ impl DoctorExecutor {
metrics.phase_durations.push(DoctorPhaseDuration {
phase: phase.phase,
duration_ms: phase_start.elapsed().as_millis() as u64,
duration_ms: phase_start
.elapsed()
.as_millis()
.try_into()
.unwrap_or(u64::MAX),
});
metrics.actions_completed += actions
.iter()
@@ -1251,7 +1259,7 @@ impl DoctorExecutor {
}
}
metrics.total_duration_ms = start.elapsed().as_millis() as u64;
metrics.total_duration_ms = start.elapsed().as_millis().try_into().unwrap_or(u64::MAX);
if overall_failed {
if let Some(original) = &original_header {
@@ -1499,10 +1507,10 @@ impl DoctorExecutor {
);
let mut remaining = mem.header.wal_size;
let mut offset = mem.header.wal_offset;
let chunk_size = min(remaining as usize, 4096).max(1);
let chunk_size = (remaining.min(4096) as usize).max(1);
let zeros = vec![0u8; chunk_size];
while remaining > 0 {
let write_len = min(remaining as usize, zeros.len());
let write_len = usize::try_from(remaining.min(zeros.len() as u64)).unwrap_or(0);
mem.file.seek(SeekFrom::Start(offset))?;
mem.file.write_all(&zeros[..write_len])?;
remaining -= write_len as u64;
+17 -17
View File
@@ -30,7 +30,7 @@ pub struct EnrichmentHandle {
impl EnrichmentHandle {
/// Stop the worker and wait for it to finish.
#[must_use]
#[must_use]
pub fn stop_and_wait(mut self) -> EnrichmentWorkerStats {
self.handle.stop();
if let Some(thread) = self.thread.take() {
@@ -40,13 +40,13 @@ impl EnrichmentHandle {
}
/// Check if worker is still running.
#[must_use]
#[must_use]
pub fn is_running(&self) -> bool {
self.handle.is_running()
}
/// Get current statistics.
#[must_use]
#[must_use]
pub fn stats(&self) -> EnrichmentWorkerStats {
self.handle.stats()
}
@@ -216,19 +216,19 @@ where
impl Memvid {
/// Get the number of frames pending enrichment.
#[must_use]
#[must_use]
pub fn enrichment_queue_len(&self) -> usize {
self.toc.enrichment_queue.len()
}
/// Check if any frames need enrichment.
#[must_use]
#[must_use]
pub fn has_pending_enrichment(&self) -> bool {
!self.toc.enrichment_queue.is_empty()
}
/// Get the next task from the enrichment queue.
#[must_use]
#[must_use]
pub fn next_enrichment_task(&self) -> Option<EnrichmentTask> {
self.toc.enrichment_queue.tasks.first().cloned()
}
@@ -242,7 +242,7 @@ impl Memvid {
/// Read frame data needed for enrichment.
///
/// Returns (`search_text`, `is_skim`, `needs_embedding`) if frame exists.
#[must_use]
#[must_use]
pub fn read_frame_for_enrichment(&self, frame_id: FrameId) -> Option<(String, bool, bool)> {
let frame = self
.toc
@@ -401,7 +401,7 @@ impl Memvid {
// Mark frame as enriched
self.mark_frame_enriched(task.frame_id);
result.elapsed_ms = start.elapsed().as_millis() as u64;
result.elapsed_ms = start.elapsed().as_millis().try_into().unwrap_or(u64::MAX);
result
}
@@ -437,7 +437,7 @@ impl Memvid {
}
/// Get enrichment statistics.
#[must_use]
#[must_use]
pub fn enrichment_stats(&self) -> EnrichmentStats {
let total_frames = self
.toc
@@ -602,7 +602,7 @@ impl Memvid {
frames_processed += 1;
// Update checkpoint in queue for crash recovery
let chunks_done = embeddings_generated as u32;
let chunks_done = u32::try_from(embeddings_generated).unwrap_or(u32::MAX);
self.toc.enrichment_queue.update_checkpoint(
task.frame_id,
chunks_done,
@@ -638,19 +638,19 @@ impl Memvid {
}
/// Check if vector embeddings are enabled.
#[must_use]
#[must_use]
pub fn has_embeddings(&self) -> bool {
self.vec_enabled && self.vec_index.is_some()
}
/// Get vector count from the index.
#[must_use]
#[must_use]
pub fn vector_count(&self) -> usize {
self.toc
.indexes
.vec
.as_ref()
.map_or(0, |m| m.vector_count as usize)
self.toc.indexes.vec.as_ref().map_or(0, |m| {
#[allow(clippy::cast_possible_truncation)]
let count = m.vector_count as usize;
count
})
}
}
+69 -44
View File
@@ -70,6 +70,7 @@ impl Read for BlobReader {
if remaining == 0 {
return Ok(0);
}
#[allow(clippy::cast_possible_truncation)]
let to_read = remaining.min(buf.len() as u64) as usize;
file.seek(SeekFrom::Start(*start + *pos))?;
let read = file.read(&mut buf[..to_read])?;
@@ -161,9 +162,11 @@ fn mime_is_text(mime: &str) -> bool {
impl Memvid {
pub fn frame_by_id(&self, frame_id: FrameId) -> Result<Frame> {
let index =
usize::try_from(frame_id).map_err(|_| MemvidError::FrameNotFound { frame_id })?;
self.toc
.frames
.get(frame_id as usize)
.get(index)
.cloned()
.ok_or(MemvidError::FrameNotFound { frame_id })
}
@@ -201,7 +204,7 @@ impl Memvid {
/// we can skip re-ingestion. The hash is computed from the original file bytes.
///
/// Returns `None` if no matching frame is found.
#[must_use]
#[must_use]
pub fn find_frame_by_hash(&self, hash: &[u8; 32]) -> Option<&Frame> {
self.toc
.frames
@@ -254,11 +257,17 @@ impl Memvid {
}
pub fn frame_preview_by_id(&mut self, frame_id: FrameId) -> Result<String> {
let frame = self.toc.frames.get(frame_id as usize).cloned().ok_or(
MemvidError::InvalidTimeIndex {
let index = usize::try_from(frame_id).map_err(|_| MemvidError::InvalidTimeIndex {
reason: "frame id too large".into(),
})?;
let frame = self
.toc
.frames
.get(index)
.cloned()
.ok_or(MemvidError::InvalidTimeIndex {
reason: "frame id out of range".into(),
},
)?;
})?;
self.frame_preview(&frame)
}
@@ -267,11 +276,17 @@ impl Memvid {
/// Unlike `frame_preview_by_id` which truncates for display purposes,
/// this returns the complete text content suitable for LLM processing.
pub fn frame_text_by_id(&mut self, frame_id: FrameId) -> Result<String> {
let frame = self.toc.frames.get(frame_id as usize).cloned().ok_or(
MemvidError::InvalidTimeIndex {
let index = usize::try_from(frame_id).map_err(|_| MemvidError::InvalidTimeIndex {
reason: "frame id too large".into(),
})?;
let frame = self
.toc
.frames
.get(index)
.cloned()
.ok_or(MemvidError::InvalidTimeIndex {
reason: "frame id out of range".into(),
},
)?;
})?;
self.frame_content(&frame)
}
@@ -337,19 +352,24 @@ impl Memvid {
return Ok(None);
}
self.ensure_vec_index()?;
Ok(self.vec_index.as_ref().and_then(|index| {
index
.embedding_for(frame_id)
.map(<[f32]>::to_vec)
}))
Ok(self
.vec_index
.as_ref()
.and_then(|index| index.embedding_for(frame_id).map(<[f32]>::to_vec)))
}
pub fn frame_context(&mut self, frame_id: FrameId, query: &str) -> Result<(String, usize)> {
let frame = self.toc.frames.get(frame_id as usize).cloned().ok_or(
MemvidError::InvalidTimeIndex {
let index = usize::try_from(frame_id).map_err(|_| MemvidError::InvalidTimeIndex {
reason: "frame id too large".into(),
})?;
let frame = self
.toc
.frames
.get(index)
.cloned()
.ok_or(MemvidError::InvalidTimeIndex {
reason: "frame id out of range".into(),
},
)?;
})?;
let preview = self.frame_preview(&frame)?;
let content = self.frame_content(&frame)?;
let count = query
@@ -362,15 +382,14 @@ impl Memvid {
}
pub(crate) fn frame_canonical_bytes(&mut self, frame: &Frame) -> Result<Vec<u8>> {
if frame.role == FrameRole::Document
&& frame.chunk_manifest.is_some() {
let chunks = self.document_chunk_payloads(frame)?;
let mut buffer = Vec::new();
for (_, bytes) in chunks {
buffer.extend_from_slice(&bytes);
}
return Ok(buffer);
if frame.role == FrameRole::Document && frame.chunk_manifest.is_some() {
let chunks = self.document_chunk_payloads(frame)?;
let mut buffer = Vec::new();
for (_, bytes) in chunks {
buffer.extend_from_slice(&bytes);
}
return Ok(buffer);
}
let raw = self.read_frame_payload_bytes(frame)?;
let decoded = crate::decode_canonical_bytes(&raw, frame.canonical_encoding, frame.id)?;
if let Some(expected) = frame.canonical_length {
@@ -410,6 +429,7 @@ impl Memvid {
.canonical_length
.or(Some(frame.payload_length))
.unwrap_or(frame.payload_length);
#[allow(clippy::cast_possible_truncation)]
return Ok(Self::render_binary_summary(logical as usize));
}
}
@@ -529,24 +549,27 @@ impl Memvid {
FrameRole::DocumentChunk => {
// Try to resolve via parent's chunk manifest (new format)
if let Some(parent_id) = frame.parent_id {
if let Some(parent) = self.toc.frames.get(parent_id as usize).cloned() {
if parent.chunk_manifest.is_some() {
if let Ok(payloads) = self.document_chunk_payloads(&parent) {
if let Some(idx) = frame.chunk_index {
let idx = idx as usize;
if idx < payloads.len() {
let mut offset = 0usize;
for (_, bytes) in payloads.iter().take(idx) {
offset += bytes.len();
// Safe frame lookup
if let Ok(index) = usize::try_from(parent_id) {
if let Some(parent) = self.toc.frames.get(index).cloned() {
if parent.chunk_manifest.is_some() {
if let Ok(payloads) = self.document_chunk_payloads(&parent) {
if let Some(idx) = frame.chunk_index {
let idx = idx as usize;
if idx < payloads.len() {
let mut offset = 0usize;
for (_, bytes) in payloads.iter().take(idx) {
offset += bytes.len();
}
let (_, bytes) = &payloads[idx];
let text = String::from_utf8_lossy(bytes).into_owned();
let end = offset + bytes.len();
return Ok(ChunkInfo {
start: offset,
end,
text,
});
}
let (_, bytes) = &payloads[idx];
let text = String::from_utf8_lossy(bytes).into_owned();
let end = offset + bytes.len();
return Ok(ChunkInfo {
start: offset,
end,
text,
});
}
}
}
@@ -588,6 +611,8 @@ impl Memvid {
pub(crate) fn read_frame_payload_bytes(&mut self, frame: &Frame) -> Result<Vec<u8>> {
self.validate_frame_bounds(frame)?;
self.file.seek(SeekFrom::Start(frame.payload_offset))?;
// Safe: guarded by MAX_FRAME_BYTES check
#[allow(clippy::cast_possible_truncation)]
let mut buf = vec![0u8; frame.payload_length as usize];
self.file.read_exact(&mut buf)?;
Ok(buf)
+1 -1
View File
@@ -13,7 +13,7 @@ use crate::{MemvidError, Result};
impl Memvid {
/// Returns the vector index dimension stored in the MV2 file, if available.
/// This is useful for auto-detecting which embedding model was used to create the file.
#[must_use]
#[must_use]
pub fn vec_index_dimension(&self) -> Option<u32> {
self.toc
.indexes
+34 -16
View File
@@ -99,8 +99,7 @@ pub struct Memvid {
}
/// Controls read-only open behaviour for `.mv2` memories.
#[derive(Debug, Clone, Copy)]
#[derive(Default)]
#[derive(Debug, Clone, Copy, Default)]
pub struct OpenReadOptions {
pub allow_repair: bool,
}
@@ -126,7 +125,6 @@ impl Default for LockSettings {
}
}
impl Memvid {
/// Create a new, empty `.mv2` file with an embedded WAL and empty TOC.
/// The file is locked exclusively for the lifetime of the handle.
@@ -255,7 +253,7 @@ impl Memvid {
Ok(memvid)
}
#[must_use]
#[must_use]
pub fn lock_settings(&self) -> &LockSettings {
&self.lock_settings
}
@@ -271,7 +269,7 @@ impl Memvid {
}
/// Get the current vector compression mode
#[must_use]
#[must_use]
pub fn vector_compression(&self) -> &VectorCompression {
&self.vec_compression
}
@@ -281,7 +279,7 @@ impl Memvid {
/// Frame IDs are dense indices into `toc.frames`. When a memory is mutable, inserts are first
/// appended to the embedded WAL and only materialized into `toc.frames` on commit. This helper
/// lets frontends allocate stable frame IDs before an explicit commit.
#[must_use]
#[must_use]
pub fn next_frame_id(&self) -> u64 {
(self.toc.frames.len() as u64).saturating_add(self.pending_frame_inserts)
}
@@ -619,6 +617,13 @@ impl Memvid {
};
// Read the compressed data from the file
if manifest.bytes_length > crate::MAX_INDEX_BYTES {
return Err(MemvidError::InvalidToc {
reason: "memories track exceeds safety limit".into(),
});
}
// Safe: guarded by MAX_INDEX_BYTES check above
#[allow(clippy::cast_possible_truncation)]
let mut buf = vec![0u8; manifest.bytes_length as usize];
self.file
.seek(std::io::SeekFrom::Start(manifest.bytes_offset))?;
@@ -646,6 +651,13 @@ impl Memvid {
};
// Read the serialized data from the file
if manifest.bytes_length > crate::MAX_INDEX_BYTES {
return Err(MemvidError::InvalidToc {
reason: "logic mesh exceeds safety limit".into(),
});
}
// Safe: guarded by MAX_INDEX_BYTES check above
#[allow(clippy::cast_possible_truncation)]
let mut buf = vec![0u8; manifest.bytes_length as usize];
self.file
.seek(std::io::SeekFrom::Start(manifest.bytes_offset))?;
@@ -760,7 +772,7 @@ impl Memvid {
///
/// Returns the binding if this file is bound to a dashboard memory,
/// or None if unbound.
#[must_use]
#[must_use]
pub fn get_memory_binding(&self) -> Option<&crate::types::MemoryBinding> {
self.toc.memory_binding.as_ref()
}
@@ -860,7 +872,14 @@ pub(crate) fn read_toc(file: &mut File, header: &Header) -> Result<Toc> {
// Read the entire region from footer_offset to EOF (includes TOC + footer)
file.seek(SeekFrom::Start(header.footer_offset))?;
// Safe: total_size bounded by file length, and we check MAX_INDEX_BYTES before reading
#[allow(clippy::cast_possible_truncation)]
let total_size = (len - header.footer_offset) as usize;
if total_size as u64 > crate::MAX_INDEX_BYTES {
return Err(MemvidError::InvalidToc {
reason: "toc region exceeds safety limit".into(),
});
}
if total_size < FOOTER_SIZE {
return Err(MemvidError::InvalidToc {
@@ -880,6 +899,7 @@ pub(crate) fn read_toc(file: &mut File, header: &Header) -> Result<Toc> {
// Extract only the TOC bytes (excluding the footer)
let toc_bytes = &buf[..footer_start];
#[allow(clippy::cast_possible_truncation)]
if toc_bytes.len() != footer.toc_len as usize {
return Err(MemvidError::InvalidToc {
reason: "toc length mismatch".into(),
@@ -1024,6 +1044,8 @@ pub(crate) fn recover_toc(file: &mut File, hint: Option<u64>) -> Result<(Toc, u6
if let Some(hint_offset) = hint {
use crate::footer::FOOTER_SIZE;
// Safe: file successfully mmapped so length fits in usize
#[allow(clippy::cast_possible_truncation)]
let start = (hint_offset.min(len)) as usize;
if mmap.len().saturating_sub(start) >= FOOTER_SIZE {
let toc_end = mmap.len().saturating_sub(FOOTER_SIZE);
@@ -1047,6 +1069,8 @@ pub(crate) fn recover_toc(file: &mut File, hint: Option<u64>) -> Result<(Toc, u6
// Fallback to manual scan if footer-based recovery failed
let mut ranges = Vec::new();
if let Some(hint_offset) = hint {
// Safe: file successfully mmapped so length fits in usize
#[allow(clippy::cast_possible_truncation)]
let hint_idx = hint_offset.min(len) as usize;
ranges.push((hint_idx, mmap.len()));
if hint_idx > 0 {
@@ -1439,9 +1463,7 @@ fn validate_segment_integrity(toc: &Toc, header: &Header, file_len: u64) -> Resu
let end = offset
.checked_add(length)
.ok_or_else(|| MemvidError::Doctor {
reason: format!(
"Tantivy segment {idx} offset overflow: {offset} + {length}"
),
reason: format!("Tantivy segment {idx} offset overflow: {offset} + {length}"),
})?;
if end > file_len || end > data_limit {
@@ -1465,9 +1487,7 @@ fn validate_segment_integrity(toc: &Toc, header: &Header, file_len: u64) -> Resu
let end = offset
.checked_add(length)
.ok_or_else(|| MemvidError::Doctor {
reason: format!(
"Time segment {idx} offset overflow: {offset} + {length}"
),
reason: format!("Time segment {idx} offset overflow: {offset} + {length}"),
})?;
if end > file_len || end > data_limit {
@@ -1491,9 +1511,7 @@ fn validate_segment_integrity(toc: &Toc, header: &Header, file_len: u64) -> Resu
let end = offset
.checked_add(length)
.ok_or_else(|| MemvidError::Doctor {
reason: format!(
"Vec segment {idx} offset overflow: {offset} + {length}"
),
reason: format!("Vec segment {idx} offset overflow: {offset} + {length}"),
})?;
if end > file_len || end > data_limit {
+9 -4
View File
@@ -413,7 +413,7 @@ impl Memvid {
///
/// # Returns
/// A vector of (index, error) tuples for invalid cards.
#[must_use]
#[must_use]
pub fn validate_cards(&self, cards: &[MemoryCard]) -> Vec<(usize, SchemaError)> {
cards
.iter()
@@ -540,8 +540,9 @@ impl Memvid {
.into_iter()
.map(|schema| {
let stats = predicate_stats.get(&schema.id);
let (entity_count, value_count, unique_values) = stats
.map_or((0, 0, 0), |s| (s.entities.len(), s.value_count, s.unique_values.len()));
let (entity_count, value_count, unique_values) = stats.map_or((0, 0, 0), |s| {
(s.entities.len(), s.value_count, s.unique_values.len())
});
// Check if there's an existing (builtin) schema
let is_builtin = self
@@ -590,7 +591,11 @@ impl Memvid {
for frame_id in unenriched {
// Get frame data
let Some(frame) = self.toc.frames.get(frame_id as usize) else {
// Safe frame lookup
let Ok(index) = usize::try_from(frame_id) else {
continue;
};
let Some(frame) = self.toc.frames.get(index) else {
continue;
};
let frame = frame.clone();
+9 -9
View File
@@ -95,7 +95,7 @@ impl Memvid {
///
/// # Returns
/// A list of entities found by traversing the relationships.
#[must_use]
#[must_use]
pub fn follow(&self, start: &str, link: &str, hops: usize) -> Vec<FollowResult> {
self.logic_mesh.follow(start, link, hops)
}
@@ -107,7 +107,7 @@ impl Memvid {
///
/// # Returns
/// The matching node if found.
#[must_use]
#[must_use]
pub fn find_entity(&self, name: &str) -> Option<&MeshNode> {
self.logic_mesh.find_node(name)
}
@@ -119,7 +119,7 @@ impl Memvid {
///
/// # Returns
/// A list of entity nodes that have mentions in the specified frame.
#[must_use]
#[must_use]
pub fn frame_entities(&self, frame_id: FrameId) -> Vec<&MeshNode> {
self.logic_mesh
.nodes
@@ -135,7 +135,7 @@ impl Memvid {
///
/// # Returns
/// A list of entity nodes matching the specified kind.
#[must_use]
#[must_use]
pub fn entities_by_kind(&self, kind: EntityKind) -> Vec<&MeshNode> {
self.logic_mesh
.nodes
@@ -148,7 +148,7 @@ impl Memvid {
///
/// # Returns
/// Statistics including node count, edge count, and breakdowns by kind/link type.
#[must_use]
#[must_use]
pub fn logic_mesh_stats(&self) -> LogicMeshStats {
self.logic_mesh.stats()
}
@@ -157,19 +157,19 @@ impl Memvid {
///
/// # Returns
/// `true` if the mesh has nodes or edges.
#[must_use]
#[must_use]
pub fn has_logic_mesh(&self) -> bool {
!self.logic_mesh.is_empty()
}
/// Get the number of entity nodes in the mesh.
#[must_use]
#[must_use]
pub fn mesh_node_count(&self) -> usize {
self.logic_mesh.nodes.len()
}
/// Get the number of relationship edges in the mesh.
#[must_use]
#[must_use]
pub fn mesh_edge_count(&self) -> usize {
self.logic_mesh.edges.len()
}
@@ -177,7 +177,7 @@ impl Memvid {
/// Get entities for a frame as `SearchHitEntity` for search metadata.
///
/// Returns entities from the Logic-Mesh that appear in the given frame.
#[must_use]
#[must_use]
pub fn frame_entities_for_search(&self, frame_id: FrameId) -> Vec<SearchHitEntity> {
self.logic_mesh
.nodes
+84 -55
View File
@@ -381,7 +381,7 @@ fn finalize_reader_output(output: ReaderOutput, start: Instant) -> ExtractedDocu
fn log_reader_result(reader: &str, diagnostics: &ReaderDiagnostics, elapsed: Duration) {
let duration_ms = diagnostics
.duration_ms
.unwrap_or(elapsed.as_millis() as u64);
.unwrap_or(elapsed.as_millis().try_into().unwrap_or(u64::MAX));
let warnings = diagnostics.warnings.len();
let pages = diagnostics.pages_processed;
@@ -671,9 +671,11 @@ impl Memvid {
let chunk = min(remaining, buffer.len() as u64);
let src = data_start + remaining - chunk;
self.file.seek(SeekFrom::Start(src))?;
#[allow(clippy::cast_possible_truncation)]
self.file.read_exact(&mut buffer[..chunk as usize])?;
let dst = src + delta;
self.file.seek(SeekFrom::Start(dst))?;
#[allow(clippy::cast_possible_truncation)]
self.file.write_all(&buffer[..chunk as usize])?;
remaining -= chunk;
}
@@ -683,6 +685,7 @@ impl Memvid {
let mut remaining = delta;
while remaining > 0 {
let write = min(remaining, zero_buf.len() as u64);
#[allow(clippy::cast_possible_truncation)]
self.file.write_all(&zero_buf[..write as usize])?;
remaining -= write;
}
@@ -1127,7 +1130,13 @@ impl Memvid {
reason: "reused payload entry contained inline bytes",
});
}
let source = self.toc.frames.get(source_id as usize).cloned().ok_or(
let source_idx = usize::try_from(source_id).map_err(|_| {
MemvidError::InvalidFrame {
frame_id: source_id,
reason: "frame id too large for memory",
}
})?;
let source = self.toc.frames.get(source_idx).cloned().ok_or(
MemvidError::InvalidFrame {
frame_id: source_id,
reason: "reused payload source missing",
@@ -1248,21 +1257,21 @@ impl Memvid {
if entry.role == FrameRole::DocumentChunk {
// Look backwards through recently inserted frames
for &candidate_id in delta.inserted_frames.iter().rev() {
if let Some(candidate) =
self.toc.frames.get(candidate_id as usize)
{
if candidate.role == FrameRole::Document
&& candidate.chunk_manifest.is_some()
{
// Found a parent document - use it
frame.parent_id = Some(candidate_id);
tracing::debug!(
chunk_frame_id = frame_id,
parent_frame_id = candidate_id,
parent_seq = parent_seq,
"resolved chunk parent via fallback"
);
break;
if let Ok(idx) = usize::try_from(candidate_id) {
if let Some(candidate) = self.toc.frames.get(idx) {
if candidate.role == FrameRole::Document
&& candidate.chunk_manifest.is_some()
{
// Found a parent document - use it
frame.parent_id = Some(candidate_id);
tracing::debug!(
chunk_frame_id = frame_id,
parent_frame_id = candidate_id,
parent_seq = parent_seq,
"resolved chunk parent via fallback"
);
break;
}
}
}
}
@@ -1366,18 +1375,21 @@ impl Memvid {
.inserted_frames
.iter()
.filter_map(|&frame_id| {
let frame = self.toc.frames.get(frame_id as usize)?;
let idx = usize::try_from(frame_id).ok()?;
let frame = self.toc.frames.get(idx)?;
if frame.role != FrameRole::DocumentChunk || frame.parent_id.is_some() {
return None;
}
// Find the most recent Document frame before this chunk that has a manifest
for candidate_id in (0..frame_id).rev() {
if let Some(candidate) = self.toc.frames.get(candidate_id as usize) {
if candidate.role == FrameRole::Document
&& candidate.chunk_manifest.is_some()
&& candidate.status == FrameStatus::Active
{
return Some((frame_id, candidate_id));
if let Ok(idx) = usize::try_from(candidate_id) {
if let Some(candidate) = self.toc.frames.get(idx) {
if candidate.role == FrameRole::Document
&& candidate.chunk_manifest.is_some()
&& candidate.status == FrameStatus::Active
{
return Some((frame_id, candidate_id));
}
}
}
}
@@ -1387,13 +1399,15 @@ impl Memvid {
// Now apply the resolutions
for (chunk_id, parent_id) in orphan_resolutions {
if let Some(frame) = self.toc.frames.get_mut(chunk_id as usize) {
frame.parent_id = Some(parent_id);
tracing::debug!(
chunk_frame_id = chunk_id,
parent_frame_id = parent_id,
"resolved orphan chunk parent in second pass"
);
if let Ok(idx) = usize::try_from(chunk_id) {
if let Some(frame) = self.toc.frames.get_mut(idx) {
frame.parent_id = Some(parent_id);
tracing::debug!(
chunk_frame_id = chunk_id,
parent_frame_id = parent_id,
"resolved orphan chunk parent in second pass"
);
}
}
}
@@ -1884,14 +1898,18 @@ impl Memvid {
}
fn mark_frame_superseded(&mut self, frame_id: FrameId, successor_id: FrameId) -> Result<()> {
let frame =
self.toc
.frames
.get_mut(frame_id as usize)
.ok_or(MemvidError::InvalidFrame {
frame_id,
reason: "supersede target missing",
})?;
let index = usize::try_from(frame_id).map_err(|_| MemvidError::InvalidFrame {
frame_id,
reason: "frame id too large",
})?;
let frame = self
.toc
.frames
.get_mut(index)
.ok_or(MemvidError::InvalidFrame {
frame_id,
reason: "supersede target missing",
})?;
frame.status = FrameStatus::Superseded;
frame.superseded_by = Some(successor_id);
self.remove_frame_from_indexes(frame_id)
@@ -2311,6 +2329,7 @@ impl Memvid {
bytes_offset: sketch_offset,
bytes_length: sketch_length,
entry_count: stats.entry_count,
#[allow(clippy::cast_possible_truncation)]
entry_size: stats.variant.entry_size() as u16,
flags: 0,
checksum: sketch_checksum,
@@ -2471,14 +2490,18 @@ impl Memvid {
}
fn mark_frame_deleted(&mut self, frame_id: FrameId) -> Result<()> {
let frame =
self.toc
.frames
.get_mut(frame_id as usize)
.ok_or(MemvidError::InvalidFrame {
frame_id,
reason: "delete target missing",
})?;
let index = usize::try_from(frame_id).map_err(|_| MemvidError::InvalidFrame {
frame_id,
reason: "frame id too large",
})?;
let frame = self
.toc
.frames
.get_mut(index)
.ok_or(MemvidError::InvalidFrame {
frame_id,
reason: "delete target missing",
})?;
frame.status = FrameStatus::Deleted;
frame.superseded_by = None;
self.remove_frame_from_indexes(frame_id)
@@ -2500,9 +2523,12 @@ impl Memvid {
}
pub(crate) fn frame_is_active(&self, frame_id: FrameId) -> bool {
let Ok(index) = usize::try_from(frame_id) else {
return false;
};
self.toc
.frames
.get(frame_id as usize)
.get(index)
.is_some_and(|frame| frame.status == FrameStatus::Active)
}
@@ -3087,7 +3113,9 @@ impl Memvid {
if let Some(ref vector) = embedding {
if !vector.is_empty() {
dim = Some(vector.len() as u32);
#[allow(clippy::cast_possible_truncation)]
let len = vector.len() as u32;
dim = Some(len);
}
}
@@ -3096,7 +3124,7 @@ impl Memvid {
if vector.is_empty() {
continue;
}
let vec_dim = vector.len() as u32;
let vec_dim = u32::try_from(vector.len()).unwrap_or(0);
match dim {
None => dim = Some(vec_dim),
Some(existing) if existing == vec_dim => {}
@@ -3168,6 +3196,7 @@ impl Memvid {
.unwrap_or(0)
});
#[allow(unused_assignments)]
let mut reuse_bytes: Option<Vec<u8>> = None;
let payload_for_processing = if let Some(bytes) = payload {
Some(bytes)
@@ -3412,7 +3441,7 @@ impl Memvid {
let triplet_title = title_value.clone();
if let Some(plan) = chunk_plan.as_ref() {
let chunk_total = plan.chunks.len() as u32;
let chunk_total = u32::try_from(plan.chunks.len()).unwrap_or(0);
parent_chunk_manifest = Some(plan.manifest.clone());
parent_chunk_count = Some(chunk_total);
@@ -3470,7 +3499,7 @@ impl Memvid {
chunk_manifest: None,
role: FrameRole::DocumentChunk,
parent_sequence: None,
chunk_index: Some(idx as u32),
chunk_index: Some(u32::try_from(idx).unwrap_or(0)),
chunk_count: Some(chunk_total),
op: FrameWalOp::Insert,
target_frame_id: None,
@@ -3494,9 +3523,9 @@ impl Memvid {
// Since frame.id corresponds to the array index, we need to find the sequence
// For now, we'll use the frame_id + WAL_START_SEQUENCE as an approximation
// This works because sequence numbers are assigned incrementally
self.toc
.frames
.get(parent_id as usize)
usize::try_from(parent_id)
.ok()
.and_then(|idx| self.toc.frames.get(idx))
.map(|_| parent_id + 2) // WAL sequences start at 2
} else {
None
+38 -11
View File
@@ -99,15 +99,24 @@ impl Memvid {
return Err(MemvidError::VecNotEnabled);
}
let mut ensured_vec_index = false;
let expected_dim = if let Some(dim) = self.effective_vec_index_dimension()? { dim } else {
let expected_dim = if let Some(dim) = self.effective_vec_index_dimension()? {
dim
} else {
self.ensure_vec_index()?;
ensured_vec_index = true;
self.vec_index
.as_ref()
.and_then(|index| index.entries().next().map(|(_, emb)| emb.len() as u32))
.and_then(|index| {
index
.entries()
.next()
.map(|(_, emb)| u32::try_from(emb.len()).unwrap_or(0))
})
.unwrap_or(0)
};
if expected_dim > 0 && query.len() as u32 != expected_dim {
// Safe: embedding dimensions are small (< few thousands)
#[allow(clippy::cast_possible_truncation)]
if expected_dim > 0 && (query.len() as u32) != expected_dim {
return Err(MemvidError::VecDimensionMismatch {
expected: expected_dim,
actual: query.len(),
@@ -246,15 +255,24 @@ impl Memvid {
// Validate embedding dimension BEFORE searching to prevent silent wrong results.
// For segment-only memories, dimension may only be discoverable after loading segments.
let mut ensured_vec_index = false;
let expected_dim = if let Some(dim) = self.effective_vec_index_dimension()? { dim } else {
let expected_dim = if let Some(dim) = self.effective_vec_index_dimension()? {
dim
} else {
self.ensure_vec_index()?;
ensured_vec_index = true;
self.vec_index
.as_ref()
.and_then(|index| index.entries().next().map(|(_, emb)| emb.len() as u32))
.and_then(|index| {
index
.entries()
.next()
.map(|(_, emb)| u32::try_from(emb.len()).unwrap_or(0))
})
.unwrap_or(0)
};
if expected_dim > 0 && query_embedding.len() as u32 != expected_dim {
// Safe: embedding dimensions are small
#[allow(clippy::cast_possible_truncation)]
if expected_dim > 0 && (query_embedding.len() as u32) != expected_dim {
return Err(MemvidError::VecDimensionMismatch {
expected: expected_dim,
actual: query_embedding.len(),
@@ -297,7 +315,14 @@ impl Memvid {
for vec_hit in vec_hits {
// Apply scope filter if provided
let frame = match self.toc.frames.get(vec_hit.frame_id as usize) {
// Apply scope filter if provided
let frame_idx = if let Ok(idx) = usize::try_from(vec_hit.frame_id) {
idx
} else {
continue;
};
let frame = match self.toc.frames.get(frame_idx) {
Some(f) => f.clone(),
None => continue,
};
@@ -690,6 +715,8 @@ impl Memvid {
self.file.seek(SeekFrom::Start(segment.bytes_offset))?;
let mut remaining = segment.bytes_length;
while remaining > 0 {
// Safe: chunk is at most buffer.len() which is usize
#[allow(clippy::cast_possible_truncation)]
let chunk = remaining.min(buffer.len() as u64) as usize;
if let Err(err) = self.file.read_exact(&mut buffer[..chunk]) {
return Err(MemvidError::Tantivy {
@@ -800,7 +827,7 @@ impl Memvid {
Ok(())
}
#[must_use]
#[must_use]
pub fn vec_segment_descriptor(&self, segment_id: u64) -> Option<VecSegmentDescriptor> {
self.toc
.segment_catalog
@@ -830,7 +857,7 @@ impl Memvid {
pub const DEFAULT_MAX_INDEX_PAYLOAD: u64 = 256 * 1024 * 1024;
/// Get the maximum indexable payload size from environment or use default
#[must_use]
#[must_use]
pub fn max_index_payload() -> u64 {
std::env::var("MEMVID_MAX_INDEX_PAYLOAD")
.ok()
@@ -839,7 +866,7 @@ pub fn max_index_payload() -> u64 {
}
/// Check if a MIME type represents text-based content that should be indexed
#[must_use]
#[must_use]
pub fn is_text_indexable_mime(mime: &str) -> bool {
let mime_lower = mime.to_lowercase();
@@ -891,7 +918,7 @@ pub fn is_text_indexable_mime(mime: &str) -> bool {
}
/// Check if a frame should be indexed for text search
#[must_use]
#[must_use]
pub fn is_frame_text_indexable(frame: &crate::types::Frame) -> bool {
// Must be active
if frame.status != crate::types::FrameStatus::Active {
+28 -16
View File
@@ -96,11 +96,14 @@ impl Memvid {
return Ok(());
}
let bytes = if let Ok(bytes) = self.read_range(manifest.bytes_offset, manifest.bytes_length) { bytes } else {
// Don't disable lex if loading fails - keep it enabled
self.lex_index = None;
return Ok(());
};
let bytes =
if let Ok(bytes) = self.read_range(manifest.bytes_offset, manifest.bytes_length) {
bytes
} else {
// Don't disable lex if loading fails - keep it enabled
self.lex_index = None;
return Ok(());
};
match LexIndex::decode(&bytes) {
Ok(mut index) => {
self.hydrate_lex_index_metadata(&mut index);
@@ -127,12 +130,15 @@ impl Memvid {
return Ok(());
}
let bytes = if let Ok(bytes) = self.read_range(manifest.bytes_offset, manifest.bytes_length) { bytes } else {
self.vec_index = None;
// Don't disable vec if loading fails - keep it enabled
// self.vec_enabled = false;
return Ok(());
};
let bytes =
if let Ok(bytes) = self.read_range(manifest.bytes_offset, manifest.bytes_length) {
bytes
} else {
self.vec_index = None;
// Don't disable vec if loading fails - keep it enabled
// self.vec_enabled = false;
return Ok(());
};
match catch_unwind(AssertUnwindSafe(|| VecIndex::decode(&bytes))) {
Ok(Ok(index)) => self.vec_index = Some(index),
Ok(Err(_)) | Err(_) => {
@@ -160,10 +166,13 @@ impl Memvid {
return Ok(());
}
let bytes = if let Ok(bytes) = self.read_range(manifest.bytes_offset, manifest.bytes_length) { bytes } else {
self.clip_index = None;
return Ok(());
};
let bytes =
if let Ok(bytes) = self.read_range(manifest.bytes_offset, manifest.bytes_length) {
bytes
} else {
self.clip_index = None;
return Ok(());
};
match catch_unwind(AssertUnwindSafe(|| ClipIndex::decode(&bytes))) {
Ok(Ok(index)) => self.clip_index = Some(index),
Ok(Err(_)) | Err(_) => {
@@ -187,6 +196,8 @@ impl Memvid {
});
}
self.file.seek(SeekFrom::Start(offset))?;
// Safe: length is checked against MAX_INDEX_BYTES above
#[allow(clippy::cast_possible_truncation)]
let mut buf = vec![0u8; length as usize];
self.file.read_exact(&mut buf)?;
Ok(buf)
@@ -264,7 +275,8 @@ impl Memvid {
fn hydrate_lex_index_metadata(&self, index: &mut LexIndex) {
for document in index.documents_mut() {
let frame_meta = self.toc.frames.get(document.frame_id as usize);
let frame_idx = usize::try_from(document.frame_id).ok();
let frame_meta = frame_idx.and_then(|idx| self.toc.frames.get(idx));
if document.uri.is_none() {
let derived = frame_meta
+15 -7
View File
@@ -45,11 +45,12 @@ pub(super) fn search_with_lex_fallback(
continue;
}
}
let frame_meta = memvid.toc.frames.get(matched.frame_id as usize).ok_or(
MemvidError::InvalidTimeIndex {
let frame_meta = usize::try_from(matched.frame_id)
.ok()
.and_then(|idx| memvid.toc.frames.get(idx))
.ok_or(MemvidError::InvalidTimeIndex {
reason: "frame id out of range".into(),
},
)?;
})?;
let content_lower = matched.content.to_ascii_lowercase();
let ctx = EvaluationContext {
frame: frame_meta,
@@ -88,14 +89,21 @@ pub(super) fn search_with_lex_fallback(
let frame_meta = memvid
.toc
.frames
.get(matched.frame_id as usize)
.get(usize::try_from(matched.frame_id).unwrap_or(usize::MAX))
.cloned()
.ok_or(MemvidError::InvalidTimeIndex {
reason: "frame id out of range".into(),
})?;
let canonical = memvid.frame_content(&frame_meta)?;
let canonical_limit = frame_meta
.canonical_length.map_or_else(|| canonical.len(), |len| len as usize);
let canonical_limit = frame_meta.canonical_length.map_or_else(
|| canonical.len(),
|len| {
// Safe: canonical length is reasonably small string length
#[allow(clippy::cast_possible_truncation)]
let l = len as usize;
l
},
);
let canonical_len = canonical.len();
let effective_len = canonical_limit.min(canonical_len);
let uri = matched
+4 -1
View File
@@ -406,7 +406,10 @@ pub(super) fn enrich_hits_with_entities(hits: &mut [SearchHit], memvid: &Memvid)
// If no entities found and this is a chunk, check the parent frame
if entities.is_empty() {
if let Some(frame) = memvid.toc.frames.get(hit.frame_id as usize) {
if let Some(frame) = usize::try_from(hit.frame_id)
.ok()
.and_then(|idx| memvid.toc.frames.get(idx))
{
if let Some(parent_id) = frame.parent_id {
entities = memvid.frame_entities_for_search(parent_id);
}
+4 -3
View File
@@ -123,7 +123,7 @@ pub(super) fn try_tantivy_search(
let frame_meta = memvid
.toc
.frames
.get(hit.frame_id as usize)
.get(usize::try_from(hit.frame_id).unwrap_or(usize::MAX))
.cloned()
.ok_or(MemvidError::InvalidTimeIndex {
reason: "frame id out of range".into(),
@@ -201,6 +201,7 @@ pub(super) fn try_tantivy_search(
.map(|(hit, occurrences, slices, chunk_info, timestamp)| {
let bm25_score = hit.score;
// Age relative to the most recent document in results
#[allow(clippy::cast_precision_loss)]
let age_seconds = (max_ts - timestamp).max(0) as f32;
// Decay factor: half-life of ~1 day for aggressive recency preference
// This ensures even a few days difference has significant impact
@@ -268,7 +269,7 @@ pub(super) fn try_tantivy_search(
let frame_meta = memvid
.toc
.frames
.get(hit.frame_id as usize)
.get(usize::try_from(hit.frame_id).unwrap_or(usize::MAX))
.cloned()
.ok_or(MemvidError::InvalidTimeIndex {
reason: "frame id out of range".into(),
@@ -383,7 +384,7 @@ fn uri_matches(candidate: Option<&str>, expected: &str) -> bool {
/// Parse content dates (from frame metadata) to find the most relevant timestamp.
/// Content dates are strings like "2023/06/30 (Fri) 14:20", ISO dates, or spelled-out dates.
/// Returns the most recent timestamp found, or None if parsing fails.
#[must_use]
#[must_use]
pub fn parse_content_date_to_timestamp(content_dates: &[String]) -> Option<i64> {
if content_dates.is_empty() {
return None;
+11 -8
View File
@@ -103,12 +103,15 @@ impl Memvid {
let mut builder = LexIndexBuilder::new();
let empty_tags = std::collections::HashMap::new();
for frame_id in frame_ids {
let frame = self.toc.frames.get(*frame_id as usize).cloned().ok_or(
MemvidError::InvalidFrame {
let frame = self
.toc
.frames
.get(usize::try_from(*frame_id).unwrap_or(0))
.cloned()
.ok_or(MemvidError::InvalidFrame {
frame_id: *frame_id,
reason: "frame id out of range for lex segment",
},
)?;
})?;
if frame.status != FrameStatus::Active {
continue;
@@ -174,7 +177,7 @@ impl Memvid {
continue;
}
non_empty_count = non_empty_count.saturating_add(1);
let vec_dim = vector.len() as u32;
let vec_dim = u32::try_from(vector.len()).unwrap_or(0);
match dimension {
None => dimension = Some(vec_dim),
Some(existing) if existing == vec_dim => {}
@@ -568,9 +571,9 @@ impl Memvid {
let existing = latest
.get(path.as_str())
.map(|descriptor| (*descriptor).clone());
let requires_append = existing.as_ref().is_none_or(|descriptor| {
descriptor.common.checksum != blob.checksum
});
let requires_append = existing
.as_ref()
.is_none_or(|descriptor| descriptor.common.checksum != blob.checksum);
let artifact = if requires_append {
Some(TantivySegmentArtifact {
+10 -10
View File
@@ -189,10 +189,10 @@ impl Memvid {
.filter(|(_, score)| *score >= opts.min_score)
.map(|(frame_id, score)| {
let entry = self.sketch_track.get(frame_id);
let hamming_distance = entry
.map_or(64, |e| e.hamming_distance(query_sketch.simhash));
let matching_top_terms = entry
.map_or(0, |e| e.count_matching_top_terms(&query_sketch.top_terms));
let hamming_distance =
entry.map_or(64, |e| e.hamming_distance(query_sketch.simhash));
let matching_top_terms =
entry.map_or(0, |e| e.count_matching_top_terms(&query_sketch.top_terms));
SketchCandidate {
frame_id,
@@ -255,10 +255,10 @@ impl Memvid {
.into_iter()
.map(|(frame_id, score)| {
let entry = self.sketch_track.get(frame_id);
let hamming_distance = entry
.map_or(64, |e| e.hamming_distance(query_sketch.simhash));
let matching_top_terms = entry
.map_or(0, |e| e.count_matching_top_terms(&query_sketch.top_terms));
let hamming_distance =
entry.map_or(64, |e| e.hamming_distance(query_sketch.simhash));
let matching_top_terms =
entry.map_or(0, |e| e.count_matching_top_terms(&query_sketch.top_terms));
SketchCandidate {
frame_id,
@@ -274,7 +274,7 @@ impl Memvid {
term_filter_hits,
simhash_hits,
candidates_returned,
scan_us: start.elapsed().as_micros() as u64,
scan_us: u64::try_from(start.elapsed().as_micros()).unwrap_or(u64::MAX),
};
(result, stats)
@@ -303,7 +303,7 @@ mod tests {
let candidates = mem.find_sketch_candidates("cats pets", None);
// Should find some candidates
assert!(!candidates.is_empty() || mem.sketches().len() > 0);
assert!(!candidates.is_empty() || !mem.sketches().is_empty());
}
#[test]
+3 -8
View File
@@ -86,12 +86,7 @@ impl Memvid {
}
// CLIP image count from clip index manifest
let clip_image_count = self
.toc
.indexes
.clip
.as_ref()
.map_or(0, |c| c.vector_count);
let clip_image_count = self.toc.indexes.clip.as_ref().map_or(0, |c| c.vector_count);
Ok(Stats {
frame_count: self.toc.frames.len() as u64,
@@ -247,13 +242,13 @@ impl Memvid {
Ok(())
}
#[must_use]
#[must_use]
pub fn current_ticket(&self) -> TicketRef {
self.toc.ticket_ref.clone()
}
/// Returns a reference to the Logic-Mesh manifest, if present.
#[must_use]
#[must_use]
pub fn logic_mesh_manifest(&self) -> Option<&crate::types::LogicMeshManifest> {
self.toc.logic_mesh.as_ref()
}
+4 -2
View File
@@ -88,7 +88,9 @@ pub(crate) fn build_timeline(
entries.reverse();
}
let limit = limit.map_or(entries.len(), |nz| nz.get() as usize);
let limit = limit.map_or(entries.len(), |nz| {
usize::try_from(nz.get()).unwrap_or(usize::MAX)
});
let mut result = Vec::with_capacity(entries.len().min(limit));
#[cfg(feature = "temporal_track")]
let temporal_track_snapshot = memvid.temporal_track_ref()?.cloned();
@@ -96,7 +98,7 @@ pub(crate) fn build_timeline(
let frame = memvid
.toc
.frames
.get(entry.frame_id as usize)
.get(usize::try_from(entry.frame_id).unwrap_or(usize::MAX))
.ok_or(MemvidError::InvalidTimeIndex {
reason: "frame id out of range".into(),
})?
+3 -5
View File
@@ -62,13 +62,11 @@ impl ModelVerification {
}
}
#[derive(Debug, Clone)]
#[derive(Default)]
#[derive(Debug, Clone, Default)]
pub struct ModelVerifyOptions {
pub run_onnx_smoke: bool,
}
#[derive(Debug, Clone, Deserialize)]
#[serde(default, rename_all = "kebab-case")]
pub struct ModelManifest {
@@ -106,7 +104,6 @@ pub struct ModelManifestEntry {
pub kind: Option<String>,
}
pub fn verify_models(root: &Path, options: &ModelVerifyOptions) -> Result<Vec<ModelVerification>> {
if !root.exists() {
return Ok(Vec::new());
@@ -317,7 +314,8 @@ fn normalize_sha256(value: &str, context: &str) -> Result<String> {
fn digest_from_dir_name(path: &Path) -> Option<String> {
let name = path.file_name()?.to_str()?;
name.strip_prefix("sha256-").map(std::string::ToString::to_string)
name.strip_prefix("sha256-")
.map(std::string::ToString::to_string)
}
fn compute_sha256_hex(path: &Path) -> Result<String> {
+5 -7
View File
@@ -44,13 +44,11 @@ impl DocumentReader for DocxReader {
fn supports(&self, hint: &ReaderHint<'_>) -> bool {
matches!(hint.format, Some(DocumentFormat::Docx))
|| hint
.mime
.is_some_and(|mime| {
mime.eq_ignore_ascii_case(
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
)
})
|| hint.mime.is_some_and(|mime| {
mime.eq_ignore_ascii_case(
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
)
})
}
fn extract(&self, bytes: &[u8], hint: &ReaderHint<'_>) -> Result<ReaderOutput> {
+2 -2
View File
@@ -33,7 +33,7 @@ pub enum DocumentFormat {
}
impl DocumentFormat {
#[must_use]
#[must_use]
pub fn label(self) -> &'static str {
match self {
Self::Pdf => "pdf",
@@ -126,7 +126,7 @@ impl ReaderDiagnostics {
self.fallback = true;
}
#[must_use]
#[must_use]
pub fn with_metadata(mut self, value: Value) -> Self {
self.extra_metadata = value;
self
+9 -7
View File
@@ -18,8 +18,12 @@ impl PassthroughReader {
fn supported_format(format: Option<DocumentFormat>) -> bool {
matches!(
format,
Some(DocumentFormat::Pdf | DocumentFormat::PlainText |
DocumentFormat::Markdown | DocumentFormat::Html) | None
Some(
DocumentFormat::Pdf
| DocumentFormat::PlainText
| DocumentFormat::Markdown
| DocumentFormat::Html
) | None
)
}
}
@@ -31,11 +35,9 @@ impl DocumentReader for PassthroughReader {
fn supports(&self, hint: &ReaderHint<'_>) -> bool {
Self::supported_format(hint.format)
|| hint
.mime
.is_none_or(|mime| {
mime.eq_ignore_ascii_case("application/pdf") || mime.starts_with("text/")
})
|| hint.mime.is_none_or(|mime| {
mime.eq_ignore_ascii_case("application/pdf") || mime.starts_with("text/")
})
}
fn extract(&self, bytes: &[u8], _hint: &ReaderHint<'_>) -> Result<ReaderOutput> {
+1 -1
View File
@@ -105,7 +105,7 @@ impl PdfReader {
pages += 1;
}
let duration_ms = start.elapsed().as_millis() as u64;
let duration_ms = start.elapsed().as_millis().try_into().unwrap_or(u64::MAX);
let trimmed = combined.trim();
if trimmed.is_empty() {
return Err(crate::MemvidError::ExtractionFailed {
+5 -7
View File
@@ -60,13 +60,11 @@ impl DocumentReader for PptxReader {
fn supports(&self, hint: &ReaderHint<'_>) -> bool {
matches!(hint.format, Some(DocumentFormat::Pptx))
|| hint
.mime
.is_some_and(|mime| {
mime.eq_ignore_ascii_case(
"application/vnd.openxmlformats-officedocument.presentationml.presentation",
)
})
|| hint.mime.is_some_and(|mime| {
mime.eq_ignore_ascii_case(
"application/vnd.openxmlformats-officedocument.presentationml.presentation",
)
})
}
fn extract(&self, bytes: &[u8], hint: &ReaderHint<'_>) -> Result<ReaderOutput> {
+5 -7
View File
@@ -60,13 +60,11 @@ impl DocumentReader for XlsxReader {
fn supports(&self, hint: &ReaderHint<'_>) -> bool {
matches!(hint.format, Some(DocumentFormat::Xlsx))
|| hint
.mime
.is_some_and(|mime| {
mime.eq_ignore_ascii_case(
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
)
})
|| hint.mime.is_some_and(|mime| {
mime.eq_ignore_ascii_case(
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
)
})
}
fn extract(&self, bytes: &[u8], hint: &ReaderHint<'_>) -> Result<ReaderOutput> {
+3 -9
View File
@@ -153,11 +153,7 @@ fn registry_root() -> Result<PathBuf> {
}
Err(last_err
.unwrap_or_else(|| {
io::Error::other(
"failed to establish memvid lock registry directory",
)
})
.unwrap_or_else(|| io::Error::other("failed to establish memvid lock registry directory"))
.into())
}
@@ -216,8 +212,7 @@ pub fn write_record(record: &LockRecord) -> Result<()> {
.create(true)
.truncate(true)
.open(path)?;
serde_json::to_writer(&mut file, record)
.map_err(io::Error::other)?;
serde_json::to_writer(&mut file, record).map_err(io::Error::other)?;
file.flush()?;
file.sync_all()?;
Ok(())
@@ -240,8 +235,7 @@ pub fn read_record(file_id: &FileId) -> Result<Option<LockRecord>> {
Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(None),
Err(err) => return Err(err.into()),
};
let record: LockRecord =
serde_json::from_reader(file).map_err(io::Error::other)?;
let record: LockRecord = serde_json::from_reader(file).map_err(io::Error::other)?;
Ok(Some(record))
}
+18 -12
View File
@@ -50,13 +50,13 @@ pub struct ReplayResult {
impl ReplayResult {
/// Check if the replay was successful (all actions matched).
#[must_use]
#[must_use]
pub fn is_success(&self) -> bool {
self.mismatched_actions == 0
}
/// Get the match rate as a percentage.
#[must_use]
#[must_use]
pub fn match_rate(&self) -> f64 {
if self.total_actions == 0 {
100.0
@@ -366,9 +366,8 @@ impl<'a> ReplayEngine<'a> {
// If model override is set, show it (CLI handles actual LLM re-execution)
if let Some(ref override_model) = self.config.use_model {
details.push_str(&format!(
"\n Override Model: {override_model}"
));
details
.push_str(&format!("\n Override Model: {override_model}"));
}
// Show original answer
@@ -377,9 +376,8 @@ impl<'a> ReplayEngine<'a> {
} else {
original_answer.clone()
};
details.push_str(&format!(
"\n Original Answer: \"{answer_preview}\""
));
details
.push_str(&format!("\n Original Answer: \"{answer_preview}\""));
// In audit mode with frozen frames, we consider it verified
if action.affected_frames.is_empty() {
@@ -503,7 +501,11 @@ impl<'a> ReplayEngine<'a> {
}
}
action_result.duration_ms = action_start.elapsed().as_millis() as u64;
action_result.duration_ms = action_start
.elapsed()
.as_millis()
.try_into()
.unwrap_or(u64::MAX);
result.action_results.push(action_result);
// Stop on mismatch if configured
@@ -515,7 +517,11 @@ impl<'a> ReplayEngine<'a> {
}
}
result.total_duration_ms = start_time.elapsed().as_millis() as u64;
result.total_duration_ms = start_time
.elapsed()
.as_millis()
.try_into()
.unwrap_or(u64::MAX);
if self.config.verbose {
tracing::info!(
@@ -530,7 +536,7 @@ impl<'a> ReplayEngine<'a> {
}
/// Compare two sessions to find differences.
#[must_use]
#[must_use]
pub fn compare_sessions(
session_a: &ReplaySession,
session_b: &ReplaySession,
@@ -643,7 +649,7 @@ pub struct SessionComparison {
impl SessionComparison {
/// Check if the sessions are identical.
#[must_use]
#[must_use]
pub fn is_identical(&self) -> bool {
self.actions_only_in_a.is_empty()
&& self.actions_only_in_b.is_empty()
+6 -6
View File
@@ -116,7 +116,7 @@ impl ActiveSession {
}
/// End the session and return it
#[must_use]
#[must_use]
pub fn end(mut self) -> ReplaySession {
self.session.end();
self.session
@@ -131,7 +131,7 @@ impl ActiveSession {
/// Storage operations for replay segments
pub mod storage {
use super::{REPLAY_SEGMENT_MAGIC, REPLAY_SEGMENT_VERSION, Result, MemvidError, ReplaySession};
use super::{MemvidError, REPLAY_SEGMENT_MAGIC, REPLAY_SEGMENT_VERSION, ReplaySession, Result};
use bincode::config::{self, Config};
use std::io::{Read, Write};
@@ -235,11 +235,11 @@ pub mod storage {
}
let header = ReplaySegmentHeader::new(
sessions.len() as u32,
u32::try_from(sessions.len()).unwrap_or(u32::MAX),
ReplaySegmentHeader::SIZE as u64 + total_session_bytes,
);
let mut segment = Vec::with_capacity(header.total_size as usize);
let mut segment = Vec::with_capacity(usize::try_from(header.total_size).unwrap_or(0));
header.write(&mut segment)?;
// Write each session with length prefix
@@ -260,7 +260,7 @@ pub mod storage {
for _ in 0..header.session_count {
let mut len_bytes = [0u8; 8];
cursor.read_exact(&mut len_bytes)?;
let len = u64::from_le_bytes(len_bytes) as usize;
let len = usize::try_from(u64::from_le_bytes(len_bytes)).unwrap_or(0);
let mut session_data = vec![0u8; len];
cursor.read_exact(&mut session_data)?;
@@ -302,7 +302,7 @@ pub mod storage {
reason: "Invalid active session magic".into(),
});
}
let len = u64::from_le_bytes(data[8..16].try_into().unwrap()) as usize;
let len = usize::try_from(u64::from_le_bytes(data[8..16].try_into().unwrap())).unwrap_or(0);
if data.len() < 16 + len {
return Err(MemvidError::InvalidToc {
reason: "Active session data truncated".into(),
+6 -10
View File
@@ -158,16 +158,12 @@ impl QueryPlanner<'_> {
)))
}
FieldTerm::DateRange(range) => {
let lower = range
.start
.map_or(Bound::Unbounded, |value| {
Bound::Included(Term::from_field_i64(self.engine.timestamp, value))
});
let upper = range
.end
.map_or(Bound::Unbounded, |value| {
Bound::Included(Term::from_field_i64(self.engine.timestamp, value))
});
let lower = range.start.map_or(Bound::Unbounded, |value| {
Bound::Included(Term::from_field_i64(self.engine.timestamp, value))
});
let upper = range.end.map_or(Bound::Unbounded, |value| {
Bound::Included(Term::from_field_i64(self.engine.timestamp, value))
});
Ok(Box::new(RangeQuery::new(lower, upper)))
}
}
+2 -2
View File
@@ -11,7 +11,7 @@ use wide::f32x8;
/// Uses 8-wide SIMD lanes (AVX2 on `x86_64`, NEON on aarch64).
/// Falls back to scalar for remainder elements.
#[cfg(feature = "simd")]
#[must_use]
#[must_use]
pub fn l2_distance_squared_simd(a: &[f32], b: &[f32]) -> f32 {
debug_assert_eq!(a.len(), b.len(), "vectors must have same length");
@@ -64,7 +64,7 @@ pub fn l2_distance_squared_simd(a: &[f32], b: &[f32]) -> f32 {
/// Compute L2 distance (with sqrt) using SIMD.
#[cfg(feature = "simd")]
#[must_use]
#[must_use]
pub fn l2_distance_simd(a: &[f32], b: &[f32]) -> f32 {
l2_distance_squared_simd(a, b).sqrt()
}
+21 -21
View File
@@ -40,13 +40,13 @@ impl Default for StructuralChunker {
impl StructuralChunker {
/// Create a new chunker with the given options.
#[must_use]
#[must_use]
pub fn new(options: ChunkingOptions) -> Self {
Self { options }
}
/// Create a chunker with default options and custom max chars.
#[must_use]
#[must_use]
pub fn with_max_chars(max_chars: usize) -> Self {
Self {
options: ChunkingOptions {
@@ -57,7 +57,7 @@ impl StructuralChunker {
}
/// Chunk a structured document.
#[must_use]
#[must_use]
pub fn chunk(&self, doc: &StructuredDocument) -> ChunkingResult {
let mut result = ChunkingResult::empty();
let mut current_text = String::new();
@@ -331,7 +331,7 @@ impl StructuralChunker {
index,
&table.id,
part as u32,
total_parts as u32,
u32::try_from(total_parts).unwrap_or(0),
&header_text,
char_start,
char_end,
@@ -526,7 +526,7 @@ impl StructuralChunker {
index,
element_id: None,
part: Some(1),
total_parts: Some(total_parts as u32),
total_parts: Some(u32::try_from(total_parts).unwrap_or(0)),
context: language.map(std::string::ToString::to_string),
char_start,
char_end,
@@ -537,8 +537,8 @@ impl StructuralChunker {
chunk_type: ChunkType::CodeBlockContinuation,
index,
element_id: None,
part: Some((i + 1) as u32),
total_parts: Some(total_parts as u32),
part: Some(u32::try_from(i + 1).unwrap_or(0)),
total_parts: Some(u32::try_from(total_parts).unwrap_or(0)),
context: language.map(std::string::ToString::to_string),
char_start,
char_end,
@@ -617,8 +617,8 @@ impl StructuralChunker {
chunk_type,
index,
element_id: None,
part: Some((i + 1) as u32),
total_parts: Some(total_parts as u32),
part: Some(u32::try_from(i + 1).unwrap_or(0)),
total_parts: Some(u32::try_from(total_parts).unwrap_or(0)),
context: language.map(std::string::ToString::to_string),
char_start,
char_end,
@@ -628,13 +628,13 @@ impl StructuralChunker {
}
/// Convenience function to chunk text with default options.
#[must_use]
#[must_use]
pub fn chunk_structured(doc: &StructuredDocument) -> ChunkingResult {
StructuralChunker::default().chunk(doc)
}
/// Convenience function to chunk text with custom max chars.
#[must_use]
#[must_use]
pub fn chunk_structured_with_max(doc: &StructuredDocument, max_chars: usize) -> ChunkingResult {
StructuralChunker::with_max_chars(max_chars).chunk(doc)
}
@@ -656,14 +656,14 @@ mod tests {
#[test]
fn test_table_preserved_when_small() {
let text = r#"Introduction.
let text = r"Introduction.
| Name | Age |
|------|-----|
| Alice | 30 |
| Bob | 25 |
Conclusion."#;
Conclusion.";
let doc = detect_structure(text);
let result = chunk_structured(&doc);
@@ -688,12 +688,12 @@ Conclusion."#;
}
let text = format!(
r#"Introduction.
r"Introduction.
| Column A | Column B | Column C |
|----------|----------|----------|
{}
Conclusion."#,
Conclusion.",
rows
);
@@ -781,10 +781,10 @@ All done."#;
#[test]
fn test_table_header_formatting() {
let text = r#"| Col1 | Col2 | Col3 |
let text = r"| Col1 | Col2 | Col3 |
|------|------|------|
| A1 | A2 | A3 |
| B1 | B2 | B3 |"#;
| B1 | B2 | B3 |";
let doc = detect_structure(text);
let table = doc.tables().next().unwrap();
@@ -802,9 +802,9 @@ All done."#;
}
let text = format!(
r#"| Header1 | Header2 |
r"| Header1 | Header2 |
|---------|---------|
{}"#,
{}",
rows
);
@@ -825,7 +825,7 @@ All done."#;
#[test]
fn test_chunking_result_stats() {
let text = r#"| A | B |
let text = r"| A | B |
|---|---|
| 1 | 2 |
@@ -835,7 +835,7 @@ x = 1
| C | D |
|---|---|
| 3 | 4 |"#;
| 3 | 4 |";
let doc = detect_structure(text);
let result = chunk_structured(&doc);
+12 -11
View File
@@ -390,7 +390,8 @@ fn try_detect_heading(
patterns: &Patterns,
) -> Option<DocumentElement> {
let caps = patterns.heading.captures(line)?;
let level = caps.get(1)?.as_str().len() as u8;
// Safe: markdown headers are ### (max few chars).
let level = u8::try_from(caps.get(1)?.as_str().len()).unwrap_or(0);
let text = caps.get(2)?.as_str().to_string();
let char_end = char_start + line.len();
@@ -592,14 +593,14 @@ mod tests {
#[test]
fn test_detect_markdown_table() {
let text = r#"Some text before.
let text = r"Some text before.
| Name | Age | City |
|------|-----|------|
| Alice | 30 | NYC |
| Bob | 25 | LA |
Some text after."#;
Some text after.";
let doc = detect_structure(text);
assert_eq!(doc.table_count, 1);
@@ -641,7 +642,7 @@ And more text."#;
#[test]
fn test_detect_lists() {
let text = r#"Shopping list:
let text = r"Shopping list:
- Apples
- Bananas
@@ -651,7 +652,7 @@ Steps:
1. First step
2. Second step
3. Third step"#;
3. Third step";
let doc = detect_structure(text);
@@ -666,7 +667,7 @@ Steps:
#[test]
fn test_detect_headings() {
let text = r#"# Main Title
let text = r"# Main Title
Some intro text.
@@ -676,7 +677,7 @@ Content here.
### Subsection
More content."#;
More content.";
let doc = detect_structure(text);
@@ -696,7 +697,7 @@ More content."#;
#[test]
fn test_complex_document() {
let text = r#"# Report
let text = r"# Report
## Summary
@@ -724,7 +725,7 @@ def calculate_growth(current, previous):
## Conclusion
Strong performance overall."#;
Strong performance overall.";
let doc = detect_structure(text);
@@ -772,10 +773,10 @@ Strong performance overall."#;
#[test]
fn test_ascii_table_detection() {
let text = r#"Name Age City
let text = r"Name Age City
Alice 30 NYC
Bob 25 LA
Charlie 35 SF"#;
Charlie 35 SF";
let tables = detect_ascii_tables(text);
assert_eq!(tables.len(), 1);
+3 -3
View File
@@ -320,7 +320,7 @@ pub fn extract_pdf_layout(bytes: &[u8], max_pages: usize) -> Result<Vec<PageLayo
let mut layouts = Vec::with_capacity(pages_to_process);
for page_idx in 0..pages_to_process {
let page_number = (page_idx + 1) as u32;
let page_number = u32::try_from(page_idx + 1).unwrap_or(0);
// Get page dimensions (default to standard US Letter if not available)
let (width, height) = get_page_dimensions(&document, page_idx).unwrap_or((612.0, 792.0));
@@ -452,7 +452,7 @@ fn parse_line_into_columns(
#[cfg(not(feature = "pdfium"))]
fn get_page_dimensions(document: &lopdf::Document, page_idx: usize) -> Option<(f32, f32)> {
let pages = document.get_pages();
let page_id = *pages.get(&((page_idx + 1) as u32))?;
let page_id = *pages.get(&u32::try_from(page_idx + 1).unwrap_or(0))?;
if let Ok(page) = document.get_dictionary(page_id) {
if let Ok(media_box) = page.get(b"MediaBox") {
@@ -480,7 +480,7 @@ fn get_page_dimensions(document: &lopdf::Document, page_idx: usize) -> Option<(f
///
/// Groups values that are within `threshold` of each other
/// and returns cluster centroids.
#[must_use]
#[must_use]
pub fn cluster_values(values: &[f32], threshold: f32) -> Vec<f32> {
if values.is_empty() {
return Vec::new();
+3 -3
View File
@@ -19,7 +19,7 @@ use super::types::{ExtractedTable, TableExtractionOptions, TableQuality, TableRo
///
/// # Returns
/// Vector of tables with multi-page tables merged
#[must_use]
#[must_use]
pub fn merge_multi_page_tables(
tables: Vec<ExtractedTable>,
options: &TableExtractionOptions,
@@ -263,7 +263,7 @@ fn combined_quality(q1: TableQuality, q2: TableQuality) -> TableQuality {
///
/// This is a utility function that identifies tables that might
/// be continuations of other tables without actually merging them.
#[must_use]
#[must_use]
pub fn find_continuation_candidates(
tables: &[ExtractedTable],
options: &TableExtractionOptions,
@@ -301,7 +301,7 @@ mod tests {
let mut table = ExtractedTable::new(id, source);
table.page_start = page_start;
table.page_end = page_end;
table.headers = headers.iter().map(|s| s.to_string()).collect();
table.headers = headers.iter().map(|s| (*s).to_string()).collect();
table.n_cols = headers.len();
table.n_rows = n_rows;
+3 -3
View File
@@ -41,7 +41,7 @@ pub fn extract_tables_from_pdf(
// Extract page layouts
let layouts = extract_pdf_layout(bytes, options.max_pages)?;
let pages_processed = layouts.len() as u32;
let pages_processed = u32::try_from(layouts.len()).unwrap_or(0);
let mut all_tables = Vec::new();
let mut warnings = Vec::new();
@@ -102,7 +102,7 @@ pub fn extract_tables_from_pdf(
}
}
let total_ms = start.elapsed().as_millis() as u64;
let total_ms = start.elapsed().as_millis().try_into().unwrap_or(u64::MAX);
if all_tables.is_empty() && pages_processed > 0 {
warnings.push("No tables detected in document".to_string());
@@ -651,7 +651,7 @@ fn extract_raw_text(bytes: &[u8]) -> Option<String> {
let pages = document.get_pages();
let mut all_text = String::new();
for page_num in 1..=pages.len() as u32 {
for page_num in 1..=u32::try_from(pages.len()).unwrap_or(0) {
if let Ok(text) = document.extract_text(&[page_num]) {
all_text.push_str(&text);
all_text.push('\n');
+22 -8
View File
@@ -305,7 +305,10 @@ pub fn list_tables(mem: &mut Memvid) -> Result<Vec<TableSummary>> {
.to_string();
let page_start = meta["page_start"].as_u64().unwrap_or(0);
let page_end = meta["page_end"].as_u64().unwrap_or(0);
// Safe: table dimensions fit throughout supported platforms
#[allow(clippy::cast_possible_truncation)]
let n_rows = meta["n_rows"].as_u64().unwrap_or(0) as usize;
#[allow(clippy::cast_possible_truncation)]
let n_cols = meta["n_cols"].as_u64().unwrap_or(0) as usize;
let quality = meta["quality"].as_str().unwrap_or("unknown").to_string();
let headers = meta["headers"]
@@ -321,8 +324,8 @@ pub fn list_tables(mem: &mut Memvid) -> Result<Vec<TableSummary>> {
summaries.push(TableSummary {
table_id,
source_file,
page_start: page_start as u32,
page_end: page_end as u32,
page_start: u32::try_from(page_start).unwrap_or(0),
page_end: u32::try_from(page_end).unwrap_or(0),
n_rows,
n_cols,
quality: quality.parse().unwrap_or(TableQuality::Medium),
@@ -377,11 +380,20 @@ pub fn get_table(mem: &mut Memvid, table_id: &str) -> Result<Option<ExtractedTab
);
table.source_uri = meta["source_uri"].as_str().map(String::from);
table.page_start = meta["page_start"].as_u64().unwrap_or(1) as u32;
table.page_end = meta["page_end"].as_u64().unwrap_or(1) as u32;
table.n_cols = meta["n_cols"].as_u64().unwrap_or(0) as usize;
table.n_rows = meta["n_rows"].as_u64().unwrap_or(0) as usize;
table.confidence_score = meta["confidence_score"].as_f64().unwrap_or(0.5) as f32;
table.page_start = u32::try_from(meta["page_start"].as_u64().unwrap_or(1)).unwrap_or(1);
#[allow(clippy::cast_possible_truncation)]
{
table.page_end = meta["page_end"].as_u64().unwrap_or(1) as u32;
}
#[allow(clippy::cast_possible_truncation)]
{
table.n_cols = meta["n_cols"].as_u64().unwrap_or(0) as usize;
table.n_rows = meta["n_rows"].as_u64().unwrap_or(0) as usize;
}
#[allow(clippy::cast_possible_truncation)]
{
table.confidence_score = meta["confidence_score"].as_f64().unwrap_or(0.5) as f32;
}
table.extraction_ms = meta["extraction_ms"].as_u64().unwrap_or(0);
table.headers = meta["headers"]
@@ -443,7 +455,9 @@ pub fn get_table(mem: &mut Memvid, table_id: &str) -> Result<Option<ExtractedTab
reason: format!("failed to parse row data: {e}"),
})?;
#[allow(clippy::cast_possible_truncation)]
let row_index = row_data["row_index"].as_u64().unwrap_or(0) as usize;
#[allow(clippy::cast_possible_truncation)]
let page = row_data["page"].as_u64().unwrap_or(1) as u32;
let cells: Vec<super::types::TableCell> =
@@ -480,7 +494,7 @@ pub fn get_table(mem: &mut Memvid, table_id: &str) -> Result<Option<ExtractedTab
///
/// # Returns
/// CSV formatted string
#[must_use]
#[must_use]
pub fn export_to_csv(table: &ExtractedTable) -> String {
let mut output = String::new();
+1 -3
View File
@@ -313,8 +313,7 @@ impl TableExtractionOptions {
}
/// Builder for `TableExtractionOptions`.
#[derive(Debug, Clone)]
#[derive(Default)]
#[derive(Debug, Clone, Default)]
pub struct TableExtractionOptionsBuilder {
inner: TableExtractionOptions,
}
@@ -390,7 +389,6 @@ impl TableExtractionOptionsBuilder {
}
}
/// Extraction mode controls quality vs coverage tradeoff.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum ExtractionMode {
+5 -2
View File
@@ -10,6 +10,7 @@ use crate::{
},
};
#[allow(clippy::cast_possible_truncation)]
fn canonical_config() -> impl bincode::config::Config {
bincode::config::standard()
.with_fixed_int_encoding()
@@ -121,7 +122,9 @@ impl Toc {
}
// Try V2 format (with memories_track/logic_mesh, without replay_manifest)
if let Ok((legacy, bytes_read)) = decode_from_slice::<LegacyTocV2, _>(bytes, canonical_config()) {
if let Ok((legacy, bytes_read)) =
decode_from_slice::<LegacyTocV2, _>(bytes, canonical_config())
{
if bytes_read != bytes.len() {
return Err(MemvidError::InvalidToc {
reason: "unexpected trailing bytes in V2 format".into(),
@@ -185,7 +188,7 @@ impl LegacyTocV2 {
impl Toc {
/// Computes the BLAKE3 checksum used for the TOC integrity field.
#[must_use]
#[must_use]
pub fn calculate_checksum(bytes: &[u8]) -> [u8; 32] {
let mut hasher = Hasher::new();
hasher.update(bytes);
+2 -2
View File
@@ -128,7 +128,7 @@ impl TripletExtractor {
0
};
let elapsed_ms = start.elapsed().as_millis() as u64;
let elapsed_ms = start.elapsed().as_millis().try_into().unwrap_or(u64::MAX);
let rules_count = all_cards.len();
// Deduplicate cards with same entity:slot
@@ -144,7 +144,7 @@ impl TripletExtractor {
/// Extract triplets from an existing `EnrichmentContext`.
///
/// This is useful when you already have a context from the enrichment pipeline.
#[must_use]
#[must_use]
pub fn extract_from_context(
&self,
ctx: &EnrichmentContext,
+10 -10
View File
@@ -86,7 +86,7 @@ impl Default for AdaptiveConfig {
impl AdaptiveConfig {
/// Create a config with absolute threshold strategy.
#[must_use]
#[must_use]
pub fn with_absolute_threshold(min_score: f32) -> Self {
Self {
strategy: CutoffStrategy::AbsoluteThreshold { min_score },
@@ -95,7 +95,7 @@ impl AdaptiveConfig {
}
/// Create a config with relative threshold strategy.
#[must_use]
#[must_use]
pub fn with_relative_threshold(min_ratio: f32) -> Self {
Self {
strategy: CutoffStrategy::RelativeThreshold { min_ratio },
@@ -104,7 +104,7 @@ impl AdaptiveConfig {
}
/// Create a config with score cliff detection.
#[must_use]
#[must_use]
pub fn with_score_cliff(max_drop_ratio: f32) -> Self {
Self {
strategy: CutoffStrategy::ScoreCliff { max_drop_ratio },
@@ -113,7 +113,7 @@ impl AdaptiveConfig {
}
/// Create a config with automatic elbow detection.
#[must_use]
#[must_use]
pub fn with_elbow_detection() -> Self {
Self {
strategy: CutoffStrategy::Elbow { sensitivity: 1.0 },
@@ -122,7 +122,7 @@ impl AdaptiveConfig {
}
/// Create a combined strategy (recommended for production).
#[must_use]
#[must_use]
pub fn combined(min_ratio: f32, max_drop: f32, min_score: f32) -> Self {
Self {
strategy: CutoffStrategy::Combined {
@@ -236,7 +236,7 @@ pub struct AdaptiveStats {
impl<T> AdaptiveResult<T> {
/// Create an empty result.
#[must_use]
#[must_use]
pub fn empty() -> Self {
Self {
results: Vec::new(),
@@ -365,9 +365,9 @@ pub fn compute_embedding_quality(embeddings: &[(u64, Vec<f32>)]) -> EmbeddingQua
while similarities.len() < max_pairs {
// Simple LCG random
rng_state = rng_state.wrapping_mul(6364136223846793005).wrapping_add(1);
let i = (rng_state as usize) % vector_count;
let i = usize::try_from(rng_state % (vector_count as u64)).unwrap_or(0);
rng_state = rng_state.wrapping_mul(6364136223846793005).wrapping_add(1);
let j = (rng_state as usize) % vector_count;
let j = usize::try_from(rng_state % (vector_count as u64)).unwrap_or(0);
if i != j {
let pair = if i < j { (i, j) } else { (j, i) };
@@ -500,7 +500,7 @@ fn cosine_similarity(a: &[f32], b: &[f32]) -> f32 {
///
/// Returns (`cutoff_index`, `triggered_by_strategy`).
/// Results at indices `0..cutoff_index` should be included.
#[must_use]
#[must_use]
pub fn find_adaptive_cutoff(scores: &[f32], config: &AdaptiveConfig) -> (usize, String) {
if scores.is_empty() {
return (0, "no_results".to_string());
@@ -758,7 +758,7 @@ mod tests {
// Should stop either at relative threshold (50% of 0.95 = 0.475)
// or at cliff (0.75 -> 0.40 is ~47% drop)
assert!(cutoff >= 4 && cutoff <= 6);
assert!((4..=6).contains(&cutoff));
}
#[test]
+11 -12
View File
@@ -130,7 +130,7 @@ pub struct AuditReport {
impl AuditReport {
/// Format the report as human-readable text.
#[must_use]
#[must_use]
pub fn to_text(&self) -> String {
let mut output = String::new();
@@ -270,7 +270,7 @@ impl AuditReport {
}
/// Format the report as Markdown.
#[must_use]
#[must_use]
pub fn to_markdown(&self) -> String {
let mut output = String::new();
@@ -415,13 +415,12 @@ fn format_snippet_block(text: &str, width: usize, prefix: &str) -> String {
let mut line = String::new();
for word in cleaned.split_whitespace() {
if line.len() + word.len() + 1 > width
&& !line.is_empty() {
result.push_str(prefix);
result.push_str(&line);
result.push('\n');
line.clear();
}
if line.len() + word.len() + 1 > width && !line.is_empty() {
result.push_str(prefix);
result.push_str(&line);
result.push('\n');
line.clear();
}
if !line.is_empty() {
line.push(' ');
}
@@ -477,6 +476,8 @@ fn format_timestamp(ts: i64) -> String {
// Approximate year/month/day calculation
let mut year = 1970i32;
// Safe: days will fit in i32 for any reasonable usage (millions of years)
#[allow(clippy::cast_possible_truncation)]
let mut remaining_days = days as i32;
loop {
@@ -505,9 +506,7 @@ fn format_timestamp(ts: i64) -> String {
let day = remaining_days + 1;
format!(
"{year:04}-{month:02}-{day:02}T{hours:02}:{minutes:02}:{seconds:02}Z"
)
format!("{year:04}-{month:02}-{day:02}T{hours:02}:{minutes:02}:{seconds:02}Z")
}
fn is_leap_year(year: i32) -> bool {
+4 -4
View File
@@ -18,7 +18,7 @@ pub enum CanonicalEncoding {
}
impl CanonicalEncoding {
#[must_use]
#[must_use]
pub const fn from_byte(value: u8) -> Self {
match value {
0 => CanonicalEncoding::Plain,
@@ -27,7 +27,7 @@ impl CanonicalEncoding {
}
}
#[must_use]
#[must_use]
pub const fn as_byte(self) -> u8 {
match self {
CanonicalEncoding::Plain => 0,
@@ -143,13 +143,13 @@ pub enum EnrichmentState {
impl EnrichmentState {
/// Returns true if this frame needs background enrichment.
#[must_use]
#[must_use]
pub fn needs_enrichment(&self) -> bool {
matches!(self, Self::Searchable)
}
/// Returns true if this frame has full semantic search capability.
#[must_use]
#[must_use]
pub fn has_embeddings(&self) -> bool {
matches!(self, Self::Enriched)
}
+2
View File
@@ -205,10 +205,12 @@ mod tests {
}
impl EmbeddingProvider for MockProvider {
#[allow(clippy::unnecessary_literal_bound)]
fn kind(&self) -> &str {
"mock"
}
#[allow(clippy::unnecessary_literal_bound)]
fn model(&self) -> &str {
"mock-model"
}
+8 -10
View File
@@ -19,8 +19,7 @@ use super::{
// Note: AnchorSource is always defined (not feature-gated) to maintain binary compatibility
/// Timeline query parameters for scanning frames chronologically or in reverse.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[derive(Default)]
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub struct TimelineQuery {
pub limit: Option<NonZeroU64>,
pub since: Option<i64>,
@@ -33,38 +32,37 @@ pub struct TimelineQuery {
impl TimelineQuery {
/// Start a fluent builder for timeline queries.
#[must_use]
#[must_use]
pub fn builder() -> TimelineQueryBuilder {
TimelineQueryBuilder::default()
}
}
#[derive(Debug, Default)]
pub struct TimelineQueryBuilder {
inner: TimelineQuery,
}
impl TimelineQueryBuilder {
#[must_use]
#[must_use]
pub fn limit(mut self, limit: NonZeroU64) -> Self {
self.inner.limit = Some(limit);
self
}
#[must_use]
#[must_use]
pub fn since(mut self, ts: i64) -> Self {
self.inner.since = Some(ts);
self
}
#[must_use]
#[must_use]
pub fn until(mut self, ts: i64) -> Self {
self.inner.until = Some(ts);
self
}
#[must_use]
#[must_use]
pub fn reverse(mut self, reverse: bool) -> Self {
self.inner.reverse = reverse;
self
@@ -76,13 +74,13 @@ impl TimelineQueryBuilder {
self
}
#[must_use]
#[must_use]
pub fn no_limit(mut self) -> Self {
self.inner.limit = None;
self
}
#[must_use]
#[must_use]
pub fn build(mut self) -> TimelineQuery {
if self.inner.limit.is_none() {
self.inner.limit = NonZeroU64::new(100);
+5 -5
View File
@@ -20,7 +20,7 @@ pub struct TriplePattern {
impl TriplePattern {
/// Create a new triple pattern.
#[must_use]
#[must_use]
pub fn new(subject: PatternTerm, predicate: PatternTerm, object: PatternTerm) -> Self {
Self {
subject,
@@ -30,7 +30,7 @@ impl TriplePattern {
}
/// Create a pattern matching entity:slot = value
#[must_use]
#[must_use]
pub fn entity_slot_value(entity: &str, slot: &str, value: &str) -> Self {
Self {
subject: PatternTerm::Literal(entity.to_lowercase()),
@@ -40,7 +40,7 @@ impl TriplePattern {
}
/// Create a pattern matching entity:slot = ?var (any value)
#[must_use]
#[must_use]
pub fn entity_slot_any(entity: &str, slot: &str, var: &str) -> Self {
Self {
subject: PatternTerm::Literal(entity.to_lowercase()),
@@ -50,7 +50,7 @@ impl TriplePattern {
}
/// Create a pattern matching ?entity:slot = value (find entities with this value)
#[must_use]
#[must_use]
pub fn any_slot_value(var: &str, slot: &str, value: &str) -> Self {
Self {
subject: PatternTerm::Variable(var.to_string()),
@@ -252,7 +252,7 @@ pub struct GraphMatchResult {
impl GraphMatchResult {
/// Create a new match result.
#[must_use]
#[must_use]
pub fn new(entity: String, frame_ids: Vec<FrameId>, confidence: f32) -> Self {
Self {
entity,
+16 -9
View File
@@ -43,7 +43,7 @@ pub struct MeshNode {
impl MeshNode {
/// Create a new mesh node with computed ID.
#[must_use]
#[must_use]
pub fn new(
canonical_name: String,
display_name: String,
@@ -54,12 +54,15 @@ impl MeshNode {
byte_len: u16,
) -> Self {
let id = compute_node_id(&canonical_name, kind);
#[allow(clippy::cast_possible_truncation)]
let confidence = (confidence * 100.0).min(100.0) as u8;
Self {
id,
canonical_name,
display_name,
kind,
confidence: (confidence * 100.0).min(100.0) as u8,
confidence,
frame_ids: vec![frame_id],
mentions: vec![(frame_id, byte_start, byte_len)],
}
@@ -144,7 +147,7 @@ pub struct MeshEdge {
impl MeshEdge {
/// Create a new edge.
#[must_use]
#[must_use]
pub fn new(
from_node: u64,
to_node: u64,
@@ -152,11 +155,14 @@ impl MeshEdge {
confidence: f32,
frame_id: FrameId,
) -> Self {
#[allow(clippy::cast_possible_truncation)]
let confidence = (confidence * 100.0).min(100.0) as u8;
Self {
from_node,
to_node,
link,
confidence: (confidence * 100.0).min(100.0) as u8,
confidence,
frame_id,
}
}
@@ -372,11 +378,12 @@ impl LogicMesh {
});
}
let compressed_len = u64::from_le_bytes(bytes[6..14].try_into().map_err(|_| {
MemvidError::InvalidLogicMesh {
let compressed_len = usize::try_from(u64::from_le_bytes(bytes[6..14].try_into().map_err(
|_| MemvidError::InvalidLogicMesh {
reason: "invalid length header".into(),
}
})?) as usize;
},
)?))
.unwrap_or(0);
if bytes.len() < 14 + compressed_len {
return Err(MemvidError::InvalidLogicMesh {
@@ -448,7 +455,7 @@ impl LogicMesh {
}
/// Follow edges from a start node.
#[must_use]
#[must_use]
pub fn follow(&self, start: &str, link: &str, hops: usize) -> Vec<FollowResult> {
let Some(start_node) = self.find_node(start) else {
return Vec::new();
+7 -15
View File
@@ -244,8 +244,7 @@ pub struct IndexSegmentRef {
/// Segment category emitted by the parallel builder.
/// Always defined for backwards compatibility.
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
#[derive(Default)]
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Default)]
pub enum SegmentKind {
#[default]
Lexical,
@@ -255,7 +254,6 @@ pub enum SegmentKind {
Tantivy,
}
/// Build-time metrics captured for a sealed segment.
/// Always defined for backwards compatibility.
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
@@ -673,8 +671,7 @@ impl<'de> Deserialize<'de> for TantivySegmentDescriptor {
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[derive(Default)]
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub struct IndexManifests {
pub lex: Option<LexIndexManifest>,
#[serde(default)]
@@ -685,7 +682,6 @@ pub struct IndexManifests {
pub clip: Option<crate::clip::ClipIndexManifest>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct LexIndexManifest {
pub doc_count: u64,
@@ -704,15 +700,13 @@ pub struct LexSegmentManifest {
pub checksum: [u8; 32],
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
#[derive(Default)]
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)]
pub enum VectorCompression {
#[default]
None, // Full f32 vectors (1,536 bytes for 384 dims)
Pq96, // Product quantization with 96 subspaces (96 bytes)
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct VecIndexManifest {
pub vector_count: u64,
@@ -725,8 +719,7 @@ pub struct VecIndexManifest {
pub compression_mode: VectorCompression,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[derive(Default)]
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub enum SegmentCompression {
#[default]
None,
@@ -734,7 +727,6 @@ pub enum SegmentCompression {
Lz4,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SegmentMeta {
pub id: u64,
@@ -877,7 +869,7 @@ pub struct EnrichmentQueueManifest {
impl EnrichmentQueueManifest {
/// Create a new empty enrichment queue.
#[must_use]
#[must_use]
pub fn new() -> Self {
Self {
tasks: Vec::new(),
@@ -926,13 +918,13 @@ impl EnrichmentQueueManifest {
}
/// Check if any enrichment work is pending.
#[must_use]
#[must_use]
pub fn is_empty(&self) -> bool {
self.tasks.is_empty()
}
/// Get count of pending tasks.
#[must_use]
#[must_use]
pub fn len(&self) -> usize {
self.tasks.len()
}
+3 -2
View File
@@ -533,11 +533,12 @@ impl MemoriesTrack {
});
}
let len = u64::from_le_bytes([
let len = usize::try_from(u64::from_le_bytes([
data[6], data[7], data[8], data[9], data[10], data[11], data[12],
data[13],
// Safe: checked on next line that data.len() >= 14 + len, so len fits in available memory
]) as usize;
]))
.unwrap_or(0);
if data.len() < 14 + len {
return Err(MemvidError::InvalidHeader {
reason: "memories track data truncated".into(),
+15 -17
View File
@@ -107,21 +107,19 @@ impl Default for PutOptions {
impl PutOptions {
/// Start a fluent builder for `PutOptions`.
#[must_use]
#[must_use]
pub fn builder() -> PutOptionsBuilder {
PutOptionsBuilder::default()
}
}
#[derive(Debug, Clone)]
#[derive(Default)]
#[derive(Debug, Clone, Default)]
pub struct PutOptionsBuilder {
inner: PutOptions,
}
impl PutOptionsBuilder {
#[must_use]
#[must_use]
pub fn timestamp(mut self, timestamp: i64) -> Self {
self.inner.timestamp = Some(timestamp);
self
@@ -152,7 +150,7 @@ impl PutOptionsBuilder {
self
}
#[must_use]
#[must_use]
pub fn metadata(mut self, metadata: DocMetadata) -> Self {
self.inner.metadata = Some(metadata);
self
@@ -183,44 +181,44 @@ impl PutOptionsBuilder {
self
}
#[must_use]
#[must_use]
pub fn enable_embedding(mut self, enable: bool) -> Self {
self.inner.enable_embedding = enable;
self
}
#[must_use]
#[must_use]
pub fn auto_tag(mut self, enabled: bool) -> Self {
self.inner.auto_tag = enabled;
self
}
#[must_use]
#[must_use]
pub fn extract_dates(mut self, enabled: bool) -> Self {
self.inner.extract_dates = enabled;
self
}
#[must_use]
#[must_use]
pub fn extract_triplets(mut self, enabled: bool) -> Self {
self.inner.extract_triplets = enabled;
self
}
#[must_use]
#[must_use]
pub fn parent_id(mut self, parent_id: FrameId) -> Self {
self.inner.parent_id = Some(parent_id);
self
}
#[must_use]
#[must_use]
pub fn role(mut self, role: FrameRole) -> Self {
self.inner.role = role;
self
}
/// Don't store raw binary content, only extracted text + SHA256 hash.
#[must_use]
#[must_use]
pub fn no_raw(mut self, enabled: bool) -> Self {
self.inner.no_raw = enabled;
self
@@ -233,7 +231,7 @@ impl PutOptionsBuilder {
}
/// Skip ingestion if a frame with matching BLAKE3 hash already exists.
#[must_use]
#[must_use]
pub fn dedup(mut self, enabled: bool) -> Self {
self.inner.dedup = enabled;
self
@@ -241,7 +239,7 @@ impl PutOptionsBuilder {
/// Enable instant indexing for immediate searchability.
/// When disabled, full commit is deferred (faster for batches).
#[must_use]
#[must_use]
pub fn instant_index(mut self, enabled: bool) -> Self {
self.inner.instant_index = enabled;
self
@@ -249,13 +247,13 @@ impl PutOptionsBuilder {
/// Set extraction time budget in milliseconds.
/// 0 means no budget (extract everything, slower but complete).
#[must_use]
#[must_use]
pub fn extraction_budget_ms(mut self, ms: u64) -> Self {
self.inner.extraction_budget_ms = ms;
self
}
#[must_use]
#[must_use]
pub fn build(self) -> PutOptions {
self.inner
}
+2 -2
View File
@@ -151,7 +151,7 @@ impl RerankerConfig {
/// ```
pub trait Reranker: Send + Sync {
/// Return the reranker kind identifier.
fn kind(&self) -> &str;
fn kind(&self) -> &'static str;
/// Rerank documents by relevance to the query.
///
@@ -235,7 +235,7 @@ mod tests {
struct MockReranker;
impl Reranker for MockReranker {
fn kind(&self) -> &str {
fn kind(&self) -> &'static str {
"mock"
}
+16 -14
View File
@@ -48,7 +48,7 @@ impl Default for ValueType {
impl ValueType {
/// Check if a value matches this type.
#[must_use]
#[must_use]
pub fn matches(&self, value: &str) -> bool {
match self {
Self::String | Self::Any => true,
@@ -67,7 +67,7 @@ impl ValueType {
}
/// Get a human-readable description of this type.
#[must_use]
#[must_use]
pub fn description(&self) -> String {
match self {
Self::String => "string".to_string(),
@@ -143,21 +143,21 @@ impl PredicateSchema {
}
/// Set the domain (entity kinds).
#[must_use]
#[must_use]
pub fn with_domain(mut self, kinds: Vec<EntityKind>) -> Self {
self.domain = kinds;
self
}
/// Set the range (value type).
#[must_use]
#[must_use]
pub fn with_range(mut self, range: ValueType) -> Self {
self.range = range;
self
}
/// Set cardinality to multiple.
#[must_use]
#[must_use]
pub fn multiple(mut self) -> Self {
self.cardinality = Cardinality::Multiple;
self
@@ -170,14 +170,14 @@ impl PredicateSchema {
}
/// Mark as built-in.
#[must_use]
#[must_use]
pub fn builtin(mut self) -> Self {
self.builtin = true;
self
}
/// Check if an entity kind is in the domain.
#[must_use]
#[must_use]
pub fn allows_entity(&self, kind: EntityKind) -> bool {
self.domain.is_empty() || self.domain.contains(&kind)
}
@@ -264,7 +264,7 @@ pub struct SchemaRegistry {
impl SchemaRegistry {
/// Create a new registry with built-in schemas.
#[must_use]
#[must_use]
pub fn new() -> Self {
let mut registry = Self {
schemas: HashMap::new(),
@@ -275,7 +275,7 @@ impl SchemaRegistry {
}
/// Create an empty registry without built-in schemas.
#[must_use]
#[must_use]
pub fn empty() -> Self {
Self {
schemas: HashMap::new(),
@@ -284,7 +284,7 @@ impl SchemaRegistry {
}
/// Enable strict validation (unknown predicates are rejected).
#[must_use]
#[must_use]
pub fn strict(mut self) -> Self {
self.strict = true;
self
@@ -420,13 +420,13 @@ impl SchemaRegistry {
}
/// Get a schema by predicate ID.
#[must_use]
#[must_use]
pub fn get(&self, predicate: &str) -> Option<&PredicateSchema> {
self.schemas.get(predicate)
}
/// Check if a predicate is known.
#[must_use]
#[must_use]
pub fn contains(&self, predicate: &str) -> bool {
self.schemas.contains_key(predicate)
}
@@ -443,7 +443,9 @@ impl SchemaRegistry {
value: &str,
entity_kind: Option<EntityKind>,
) -> Result<(), SchemaError> {
let schema = if let Some(s) = self.schemas.get(predicate) { s } else {
let schema = if let Some(s) = self.schemas.get(predicate) {
s
} else {
if self.strict {
return Err(SchemaError::UnknownPredicate(predicate.to_string()));
}
@@ -472,7 +474,7 @@ impl SchemaRegistry {
}
/// Infer a schema from existing memory cards.
#[must_use]
#[must_use]
pub fn infer_from_values(&self, predicate: &str, values: &[&str]) -> PredicateSchema {
let mut schema = PredicateSchema::new(predicate, predicate);
+28 -17
View File
@@ -116,7 +116,6 @@ pub enum SketchVariant {
Large = 2,
}
impl SketchVariant {
/// Get the entry size in bytes for this variant.
#[must_use]
@@ -591,7 +590,9 @@ pub fn hash_token(token: &str) -> u64 {
#[must_use]
pub fn hash_token_u32(token: &str) -> u32 {
let h = hash_token(token);
(h ^ (h >> 32)) as u32
#[allow(clippy::cast_possible_truncation)]
let res = (h ^ (h >> 32)) as u32;
res
}
// ============================================================================
@@ -609,9 +610,9 @@ pub fn build_term_filter(token_hashes: &[u64], filter_size_bytes: usize) -> Vec<
for &hash in token_hashes {
// Use 3 hash functions (simulated via rotation)
let h1 = hash as usize % filter_bits;
let h2 = (hash >> 16) as usize % filter_bits;
let h3 = (hash >> 32) as usize % filter_bits;
let h1 = usize::try_from(hash % (filter_bits as u64)).unwrap_or(0);
let h2 = usize::try_from((hash >> 16) % (filter_bits as u64)).unwrap_or(0);
let h3 = usize::try_from((hash >> 32) % (filter_bits as u64)).unwrap_or(0);
filter[h1 / 8] |= 1 << (h1 % 8);
filter[h2 / 8] |= 1 << (h2 % 8);
@@ -625,9 +626,9 @@ pub fn build_term_filter(token_hashes: &[u64], filter_size_bytes: usize) -> Vec<
#[must_use]
pub fn term_filter_maybe_contains(filter: &[u8], token_hash: u64) -> bool {
let filter_bits = filter.len() * 8;
let h1 = token_hash as usize % filter_bits;
let h2 = (token_hash >> 16) as usize % filter_bits;
let h3 = (token_hash >> 32) as usize % filter_bits;
let h1 = usize::try_from(token_hash % (filter_bits as u64)).unwrap_or(0);
let h2 = usize::try_from((token_hash >> 16) % (filter_bits as u64)).unwrap_or(0);
let h3 = usize::try_from((token_hash >> 32) % (filter_bits as u64)).unwrap_or(0);
(filter[h1 / 8] & (1 << (h1 % 8)) != 0)
&& (filter[h2 / 8] & (1 << (h2 % 8)) != 0)
@@ -684,6 +685,7 @@ pub fn compute_token_weights(
.copied()
.unwrap_or(1.0)
.max(0.1); // Default IDF = 1.0
#[allow(clippy::cast_possible_truncation)]
let weight = (capped_tf * idf * 100.0) as i32; // Scale to integer
(hash_token(token), weight.max(1))
})
@@ -701,7 +703,11 @@ pub fn extract_top_terms(weighted_tokens: &[(u64, i32)], k: usize) -> Vec<u32> {
weighted_tokens
.iter()
.take(k)
.map(|(h, _)| (*h ^ (*h >> 32)) as u32)
.map(|(h, _)| {
#[allow(clippy::cast_possible_truncation)]
let res = (*h ^ (*h >> 32)) as u32;
res
})
.collect()
}
@@ -746,6 +752,7 @@ pub fn generate_sketch(
.sum();
// Length hint: bucket token count (0-255 = 0-2550 tokens in steps of 10)
#[allow(clippy::cast_possible_truncation)]
let length_hint = ((token_count / 10).min(255)) as u16;
let mut flags = SketchFlags::all();
@@ -753,12 +760,15 @@ pub fn generate_sketch(
flags.set(SketchFlags::SHORT_TEXT);
}
#[allow(clippy::cast_possible_truncation)]
let term_weight_sum = term_weight_sum.min(u32::from(u16::MAX)) as u16;
SketchEntry {
frame_id,
simhash,
term_filter,
top_terms,
term_weight_sum: term_weight_sum.min(u32::from(u16::MAX)) as u16,
term_weight_sum,
flags,
length_hint,
}
@@ -1005,10 +1015,13 @@ impl SketchTrackHeader {
/// Create a new header.
#[must_use]
pub fn new(variant: SketchVariant, entry_count: u64) -> Self {
#[allow(clippy::cast_possible_truncation)]
let entry_size = variant.entry_size() as u16;
Self {
magic: SKETCH_TRACK_MAGIC,
version: SKETCH_TRACK_VERSION,
entry_size: variant.entry_size() as u16,
entry_size,
entry_count,
flags: 0,
reserved: 0,
@@ -1122,10 +1135,8 @@ pub fn read_sketch_track<R: Read + Seek>(
SketchTrackHeader::SIZE as u64 + header.entry_count * u64::from(header.entry_size);
if length < expected_length {
return Err(MemvidError::InvalidSketchTrack {
reason: format!(
"Sketch track length {length} less than expected {expected_length}"
)
.into(),
reason: format!("Sketch track length {length} less than expected {expected_length}")
.into(),
});
}
@@ -1200,7 +1211,7 @@ mod tests {
#[test]
fn test_term_filter() {
let tokens = vec!["hello", "world", "test"];
let tokens = ["hello", "world", "test"];
let hashes: Vec<u64> = tokens.iter().map(|t| hash_token(t)).collect();
let filter = build_term_filter(&hashes, 16);
@@ -1306,7 +1317,7 @@ mod tests {
// Should find at least some candidates with relaxed threshold
// The sketch is optimized for approximate matching, not exact
assert!(
!candidates.is_empty() || track.len() > 0,
!candidates.is_empty() || !track.is_empty(),
"Track should have entries"
);
}
+29 -29
View File
@@ -160,7 +160,7 @@ pub struct StructuredRow {
impl StructuredRow {
/// Create a new row with cells.
#[must_use]
#[must_use]
pub fn new(row: usize, cells: Vec<StructuredCell>) -> Self {
Self {
row,
@@ -170,14 +170,14 @@ impl StructuredRow {
}
/// Mark as header row.
#[must_use]
#[must_use]
pub fn as_header(mut self) -> Self {
self.is_header = true;
self
}
/// Get cell texts as a vector.
#[must_use]
#[must_use]
pub fn cell_texts(&self) -> Vec<&str> {
self.cells.iter().map(|c| c.text.as_str()).collect()
}
@@ -219,13 +219,13 @@ impl StructuredTable {
}
/// Get number of data rows.
#[must_use]
#[must_use]
pub fn data_row_count(&self) -> usize {
self.rows.iter().filter(|r| !r.is_header).count()
}
/// Format headers as markdown table header.
#[must_use]
#[must_use]
pub fn format_header(&self) -> String {
if self.headers.is_empty() {
return String::new();
@@ -243,14 +243,14 @@ impl StructuredTable {
}
/// Format a row as markdown.
#[must_use]
#[must_use]
pub fn format_row(&self, row: &StructuredRow) -> String {
let cells: Vec<&str> = row.cells.iter().map(|c| c.text.as_str()).collect();
format!("| {} |", cells.join(" | "))
}
/// Estimate character count.
#[must_use]
#[must_use]
pub fn char_count(&self) -> usize {
self.raw_text.chars().count()
}
@@ -285,7 +285,7 @@ impl StructuredCodeBlock {
}
/// Format as fenced code block.
#[must_use]
#[must_use]
pub fn format(&self) -> String {
let fence = "```";
let lang = self.language.as_deref().unwrap_or("");
@@ -293,7 +293,7 @@ impl StructuredCodeBlock {
}
/// Estimate character count.
#[must_use]
#[must_use]
pub fn char_count(&self) -> usize {
self.content.chars().count() + 10 // fences + language
}
@@ -317,7 +317,7 @@ fn default_start() -> usize {
impl StructuredList {
/// Create a new unordered list.
#[must_use]
#[must_use]
pub fn unordered(items: Vec<String>) -> Self {
Self {
ordered: false,
@@ -327,7 +327,7 @@ impl StructuredList {
}
/// Create a new ordered list.
#[must_use]
#[must_use]
pub fn ordered(items: Vec<String>) -> Self {
Self {
ordered: true,
@@ -337,7 +337,7 @@ impl StructuredList {
}
/// Format as markdown list.
#[must_use]
#[must_use]
pub fn format(&self) -> String {
self.items
.iter()
@@ -354,7 +354,7 @@ impl StructuredList {
}
/// Estimate character count.
#[must_use]
#[must_use]
pub fn char_count(&self) -> usize {
self.items.iter().map(|s| s.chars().count() + 3).sum()
}
@@ -379,7 +379,7 @@ impl StructuredHeading {
}
/// Format as markdown heading.
#[must_use]
#[must_use]
pub fn format(&self) -> String {
format!("{} {}", "#".repeat(self.level as usize), self.text)
}
@@ -433,7 +433,7 @@ impl DocumentElement {
}
/// Create a table element.
#[must_use]
#[must_use]
pub fn table(table: StructuredTable, char_start: usize, char_end: usize) -> Self {
Self {
element_type: ElementType::Table,
@@ -444,7 +444,7 @@ impl DocumentElement {
}
/// Create a code block element.
#[must_use]
#[must_use]
pub fn code_block(block: StructuredCodeBlock, char_start: usize, char_end: usize) -> Self {
Self {
element_type: ElementType::CodeBlock,
@@ -455,7 +455,7 @@ impl DocumentElement {
}
/// Create a list element.
#[must_use]
#[must_use]
pub fn list(list: StructuredList, char_start: usize, char_end: usize) -> Self {
Self {
element_type: ElementType::List,
@@ -466,7 +466,7 @@ impl DocumentElement {
}
/// Create a heading element.
#[must_use]
#[must_use]
pub fn heading(heading: StructuredHeading, char_start: usize, char_end: usize) -> Self {
Self {
element_type: ElementType::Heading,
@@ -477,7 +477,7 @@ impl DocumentElement {
}
/// Get element text content.
#[must_use]
#[must_use]
pub fn text(&self) -> String {
match &self.data {
ElementData::Paragraph { text } => text.clone(),
@@ -492,19 +492,19 @@ impl DocumentElement {
}
/// Get element character count.
#[must_use]
#[must_use]
pub fn char_count(&self) -> usize {
self.char_end.saturating_sub(self.char_start)
}
/// Check if this is a table element.
#[must_use]
#[must_use]
pub fn is_table(&self) -> bool {
self.element_type == ElementType::Table
}
/// Get table data if this is a table element.
#[must_use]
#[must_use]
pub fn as_table(&self) -> Option<&StructuredTable> {
match &self.data {
ElementData::Table(t) => Some(t),
@@ -540,7 +540,7 @@ pub struct StructuredDocument {
impl StructuredDocument {
/// Create a new empty structured document.
#[must_use]
#[must_use]
pub fn new() -> Self {
Self::default()
}
@@ -581,7 +581,7 @@ impl StructuredDocument {
}
/// Check if document has any structure (vs plain text).
#[must_use]
#[must_use]
pub fn has_structure(&self) -> bool {
self.table_count > 0 || self.code_block_count > 0
}
@@ -683,7 +683,7 @@ impl StructuredChunk {
}
/// Check if this is a table-related chunk.
#[must_use]
#[must_use]
pub fn is_table(&self) -> bool {
matches!(
self.chunk_type,
@@ -692,13 +692,13 @@ impl StructuredChunk {
}
/// Check if this is a continuation of a split element.
#[must_use]
#[must_use]
pub fn is_continuation(&self) -> bool {
self.part.is_some_and(|p| p > 1)
}
/// Get character count.
#[must_use]
#[must_use]
pub fn char_count(&self) -> usize {
self.text.chars().count()
}
@@ -775,13 +775,13 @@ pub struct ChunkingResult {
impl ChunkingResult {
/// Create empty result.
#[must_use]
#[must_use]
pub fn empty() -> Self {
Self::default()
}
/// Total chunks produced.
#[must_use]
#[must_use]
pub fn chunk_count(&self) -> usize {
self.chunks.len()
}
+6 -5
View File
@@ -9,6 +9,7 @@ fn vec_config() -> impl bincode::config::Config {
.with_little_endian()
}
#[allow(clippy::cast_possible_truncation)]
const VEC_DECODE_LIMIT: usize = crate::MAX_INDEX_BYTES as usize;
#[derive(Debug, Clone, Serialize, Deserialize)]
@@ -23,7 +24,7 @@ pub struct VecIndexBuilder {
}
impl VecIndexBuilder {
#[must_use]
#[must_use]
pub fn new() -> Self {
Self::default()
}
@@ -45,7 +46,7 @@ impl VecIndexBuilder {
let dimension = self
.documents
.first()
.map_or(0, |doc| doc.embedding.len() as u32);
.map_or(0, |doc| u32::try_from(doc.embedding.len()).unwrap_or(0));
#[cfg(feature = "parallel_segments")]
let bytes_uncompressed = self
.documents
@@ -149,7 +150,7 @@ impl VecIndex {
}
}
#[must_use]
#[must_use]
pub fn search(&self, query: &[f32], limit: usize) -> Vec<VecSearchHit> {
if query.is_empty() {
return Vec::new();
@@ -178,7 +179,7 @@ impl VecIndex {
}
}
#[must_use]
#[must_use]
pub fn entries(&self) -> Box<dyn Iterator<Item = (FrameId, &[f32])> + '_> {
match self {
VecIndex::Uncompressed { documents } => Box::new(
@@ -193,7 +194,7 @@ impl VecIndex {
}
}
#[must_use]
#[must_use]
pub fn embedding_for(&self, frame_id: FrameId) -> Option<&[f32]> {
match self {
VecIndex::Uncompressed { documents } => documents
+16 -12
View File
@@ -21,6 +21,7 @@ fn vec_config() -> impl bincode::config::Config {
.with_little_endian()
}
#[allow(clippy::cast_possible_truncation)]
const VEC_DECODE_LIMIT: usize = crate::MAX_INDEX_BYTES as usize;
/// Product Quantization parameters
@@ -62,11 +63,15 @@ impl SubspaceCodebook {
let mut best_dist = f32::INFINITY;
for i in 0..NUM_CENTROIDS {
#[allow(clippy::cast_possible_truncation)]
let centroid = self.get_centroid(i as u8);
let dist = l2_distance_squared(subspace, centroid);
if dist < best_dist {
best_dist = dist;
best_idx = i as u8;
#[allow(clippy::cast_possible_truncation)]
{
best_idx = i as u8;
}
}
}
@@ -87,9 +92,7 @@ impl ProductQuantizer {
pub fn new(dimension: u32) -> Result<Self> {
if dimension as usize != TOTAL_DIM {
return Err(MemvidError::InvalidQuery {
reason: format!(
"PQ only supports {TOTAL_DIM}-dim vectors, got {dimension}"
),
reason: format!("PQ only supports {TOTAL_DIM}-dim vectors, got {dimension}"),
});
}
@@ -136,6 +139,7 @@ impl ProductQuantizer {
// Store in codebook
for (i, centroid) in centroids.iter().enumerate() {
#[allow(clippy::cast_possible_truncation)]
self.codebooks[subspace_idx].set_centroid(i as u8, centroid);
}
}
@@ -193,7 +197,7 @@ impl ProductQuantizer {
/// Compute asymmetric distance between query vector and PQ-encoded vector
/// Uses precomputed lookup tables for efficiency
#[must_use]
#[must_use]
pub fn asymmetric_distance(&self, query: &[f32], codes: &[u8]) -> f32 {
if query.len() != TOTAL_DIM || codes.len() != NUM_SUBSPACES {
return f32::INFINITY;
@@ -232,7 +236,7 @@ pub struct QuantizedVecIndexBuilder {
}
impl QuantizedVecIndexBuilder {
#[must_use]
#[must_use]
pub fn new() -> Self {
Self::default()
}
@@ -339,7 +343,7 @@ impl QuantizedVecIndex {
// Convert old format to new format
let quantizer = ProductQuantizer {
codebooks: old_quantizer.codebooks,
dimension: (NUM_SUBSPACES * SUBSPACE_DIM) as u32,
dimension: u32::try_from(NUM_SUBSPACES * SUBSPACE_DIM).unwrap_or(u32::MAX),
};
Ok(Self {
@@ -349,7 +353,7 @@ impl QuantizedVecIndex {
}
/// Search using asymmetric distance computation
#[must_use]
#[must_use]
pub fn search(&self, query: &[f32], limit: usize) -> Vec<VecSearchHit> {
if query.is_empty() {
return Vec::new();
@@ -382,7 +386,7 @@ impl QuantizedVecIndex {
}
/// Get compression statistics
#[must_use]
#[must_use]
pub fn compression_stats(&self) -> CompressionStats {
let original_bytes = self.documents.len() * TOTAL_DIM * std::mem::size_of::<f32>();
let compressed_bytes = self.documents.len() * NUM_SUBSPACES; // 96 bytes per vector
@@ -556,7 +560,7 @@ mod tests {
}
// Train quantizer
let mut pq = ProductQuantizer::new(TOTAL_DIM as u32).unwrap();
let mut pq = ProductQuantizer::new(u32::try_from(TOTAL_DIM).unwrap()).unwrap();
pq.train(&training_vecs, 10).unwrap();
// Encode a vector
@@ -588,7 +592,7 @@ mod tests {
// Build index
let mut builder = QuantizedVecIndexBuilder::new();
builder
.train_quantizer(&training_vecs, TOTAL_DIM as u32)
.train_quantizer(&training_vecs, u32::try_from(TOTAL_DIM).unwrap())
.unwrap();
for (i, vec) in training_vecs.iter().take(10).enumerate() {
@@ -599,7 +603,7 @@ mod tests {
let artifact = builder.finish().unwrap();
assert_eq!(artifact.vector_count, 10);
assert_eq!(artifact.dimension, TOTAL_DIM as u32);
assert_eq!(artifact.dimension, u32::try_from(TOTAL_DIM).unwrap());
assert!(artifact.compression_ratio > 10.0);
// Decode and search
+5 -4
View File
@@ -60,7 +60,7 @@ fn doctor_rebuilds_tantivy_index() {
.unwrap();
assert!(
results.hits.len() > 0,
!results.hits.is_empty(),
"Search should return results before doctor"
);
assert!(results.total_hits >= 10, "Should have at least 10 hits");
@@ -106,7 +106,7 @@ fn doctor_rebuilds_tantivy_index() {
.unwrap();
assert!(
results.hits.len() > 0,
!results.hits.is_empty(),
"Search should return results after doctor rebuild"
);
assert_eq!(
@@ -250,7 +250,7 @@ fn open_file_with_tantivy_segments_enables_lex() {
);
let results = result.unwrap();
assert!(results.hits.len() > 0, "Should find the test document");
assert!(!results.hits.is_empty(), "Should find the test document");
}
}
@@ -389,8 +389,9 @@ fn doctor_recovers_corrupted_wal() {
let end = start + header.wal_size.min(100) as usize;
// corrupt some bytes
#[allow(clippy::needless_range_loop)]
for i in start..end {
bytes[i] = 0xFF;
bytes[i] = 0xFF; // Corrupt bytes
}
write(&mv2_path, &bytes).unwrap();
+2 -2
View File
@@ -52,9 +52,9 @@ fn create_handles_existing_file() {
// Create second time - this tests current behavior
// (Either it fails OR it creates a new file - both are valid implementations)
let result = Memvid::create(&path);
if result.is_ok() {
if let Ok(mut mem) = result {
// If create succeeds, the old data should be gone (new file)
let mut mem = result.unwrap();
// let mut mem = result.unwrap();
mem.commit().unwrap();
// Reopen and verify it's empty (new file was created)
let mem = Memvid::open_read_only(&path).unwrap();
+3 -2
View File
@@ -22,11 +22,12 @@ fn put_bytes_basic() {
..Default::default()
};
let frame_id = mem.put_bytes_with_options(b"Hello, World!", opts).unwrap();
let _frame_id = mem.put_bytes_with_options(b"Hello, World!", opts).unwrap();
mem.commit().unwrap();
// Verify frame was created
assert!(frame_id > 0 || frame_id == 0, "Frame ID should be valid");
// Verify frame was created (FrameId is u64 so >= 0 is implied)
// assert!(frame_id >= 0);
let mem = Memvid::open_read_only(&path).unwrap();
assert_eq!(mem.stats().unwrap().frame_count, 1, "Should have 1 frame");
+2 -2
View File
@@ -79,7 +79,7 @@ fn search_basic_query() {
})
.unwrap();
assert!(results.hits.len() > 0, "Should find quantum document");
assert!(!results.hits.is_empty(), "Should find quantum document");
assert!(
results.hits[0].uri.contains("quantum"),
"Top result should be quantum physics"
@@ -232,7 +232,7 @@ fn search_returns_snippets() {
})
.unwrap();
assert!(results.hits.len() > 0);
assert!(!results.hits.is_empty());
let hit = &results.hits[0];
// Snippet should contain matched content