fix: resolve all clippy lints and verify tests passing
This commit is contained in:
+10
-1
@@ -76,10 +76,19 @@ git commit -m "docs: update README examples"
|
||||
|
||||
- Follow standard Rust idioms and conventions
|
||||
- Use `rustfmt` for formatting (`cargo fmt`)
|
||||
- Use `clippy` for linting (`cargo clippy`)
|
||||
- Use `clippy` for linting (`cargo clippy`). We maintain a **zero-warning policy**.
|
||||
- Prefer explicit types for public APIs
|
||||
- Use `thiserror` for error definitions
|
||||
|
||||
### Linting & Safety
|
||||
|
||||
We enforce strict linting to ensure safety and portability:
|
||||
|
||||
1. **Zero Warnings**: CI will fail on any warning. Run `cargo clippy --workspace --all-targets -- -D warnings` locally.
|
||||
2. **No Panics**: `unwrap()` and `expect()` are **denied** in library code. Use `Result` propagation (`?`) or graceful error handling. They are allowed in `tests/`.
|
||||
3. **No Truncation**: `cast_possible_truncation` is denied. Use `try_from` when converting `u64` to `usize`/`u32`.
|
||||
4. **Exceptions**: We allow pragmatic lints (e.g., `cast_precision_loss` for ML math) in `src/lib.rs`. Do not add global `#![allow]` without discussion.
|
||||
|
||||
### Documentation
|
||||
|
||||
- Add doc comments (`///`) to all public functions, structs, and modules
|
||||
|
||||
@@ -39,7 +39,7 @@ fn setup_corpus(size: usize) -> std::path::PathBuf {
|
||||
let content = format!("Document {} about {}", i, topics[i % topics.len()]);
|
||||
mem.put_bytes_with_options(
|
||||
content.as_bytes(),
|
||||
PutOptions::builder().title(&format!("Doc {}", i)).build(),
|
||||
PutOptions::builder().title(format!("Doc {}", i)).build(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
|
||||
@@ -27,7 +27,7 @@ fn main() -> memvid_core::Result<()> {
|
||||
{
|
||||
eprintln!("This example requires the 'clip' feature.");
|
||||
eprintln!("Run with: cargo run --example clip_visual_search --features clip");
|
||||
return Ok(());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(feature = "clip")]
|
||||
|
||||
@@ -33,7 +33,7 @@ fn main() -> memvid_core::Result<()> {
|
||||
mem.put_bytes_with_options(
|
||||
content.as_bytes(),
|
||||
PutOptions::builder()
|
||||
.title(&format!("Doc {} - {}", i, topic.0))
|
||||
.title(format!("Doc {} - {}", i, topic.0))
|
||||
.build(),
|
||||
)?;
|
||||
|
||||
|
||||
@@ -63,7 +63,7 @@ fn main() -> Result<()> {
|
||||
|
||||
let options = PutOptions::builder()
|
||||
.title(&title)
|
||||
.uri(&format!(
|
||||
.uri(format!(
|
||||
"mv2://pdfs/{}",
|
||||
pdf_path.file_name().unwrap_or_default().to_string_lossy()
|
||||
))
|
||||
|
||||
+75
-72
@@ -40,75 +40,77 @@ impl AutoTagger {
|
||||
}
|
||||
|
||||
fn extract_keywords(text: &str, limit: usize) -> Vec<String> {
|
||||
static TOKEN_RE: std::sync::LazyLock<Regex> = std::sync::LazyLock::new(|| Regex::new(r"(?i)[a-z0-9][a-z0-9'-]+").unwrap());
|
||||
static STOPWORDS: std::sync::LazyLock<BTreeSet<&'static str>> = std::sync::LazyLock::new(|| {
|
||||
[
|
||||
"the",
|
||||
"and",
|
||||
"for",
|
||||
"with",
|
||||
"that",
|
||||
"from",
|
||||
"this",
|
||||
"were",
|
||||
"have",
|
||||
"has",
|
||||
"will",
|
||||
"shall",
|
||||
"into",
|
||||
"about",
|
||||
"without",
|
||||
"within",
|
||||
"between",
|
||||
"because",
|
||||
"over",
|
||||
"under",
|
||||
"after",
|
||||
"before",
|
||||
"until",
|
||||
"while",
|
||||
"their",
|
||||
"there",
|
||||
"these",
|
||||
"those",
|
||||
"your",
|
||||
"into",
|
||||
"such",
|
||||
"been",
|
||||
"where",
|
||||
"when",
|
||||
"which",
|
||||
"using",
|
||||
"also",
|
||||
"than",
|
||||
"could",
|
||||
"would",
|
||||
"should",
|
||||
"might",
|
||||
"cannot",
|
||||
"however",
|
||||
"therefore",
|
||||
"thereof",
|
||||
"hereby",
|
||||
"herein",
|
||||
"hereof",
|
||||
"based",
|
||||
"system",
|
||||
"application",
|
||||
"service",
|
||||
"provide",
|
||||
"provided",
|
||||
"including",
|
||||
"include",
|
||||
"includes",
|
||||
"version",
|
||||
"update",
|
||||
"updates",
|
||||
"usage",
|
||||
]
|
||||
.into_iter()
|
||||
.collect()
|
||||
});
|
||||
static TOKEN_RE: std::sync::LazyLock<Regex> =
|
||||
std::sync::LazyLock::new(|| Regex::new(r"(?i)[a-z0-9][a-z0-9'-]+").unwrap());
|
||||
static STOPWORDS: std::sync::LazyLock<BTreeSet<&'static str>> =
|
||||
std::sync::LazyLock::new(|| {
|
||||
[
|
||||
"the",
|
||||
"and",
|
||||
"for",
|
||||
"with",
|
||||
"that",
|
||||
"from",
|
||||
"this",
|
||||
"were",
|
||||
"have",
|
||||
"has",
|
||||
"will",
|
||||
"shall",
|
||||
"into",
|
||||
"about",
|
||||
"without",
|
||||
"within",
|
||||
"between",
|
||||
"because",
|
||||
"over",
|
||||
"under",
|
||||
"after",
|
||||
"before",
|
||||
"until",
|
||||
"while",
|
||||
"their",
|
||||
"there",
|
||||
"these",
|
||||
"those",
|
||||
"your",
|
||||
"into",
|
||||
"such",
|
||||
"been",
|
||||
"where",
|
||||
"when",
|
||||
"which",
|
||||
"using",
|
||||
"also",
|
||||
"than",
|
||||
"could",
|
||||
"would",
|
||||
"should",
|
||||
"might",
|
||||
"cannot",
|
||||
"however",
|
||||
"therefore",
|
||||
"thereof",
|
||||
"hereby",
|
||||
"herein",
|
||||
"hereof",
|
||||
"based",
|
||||
"system",
|
||||
"application",
|
||||
"service",
|
||||
"provide",
|
||||
"provided",
|
||||
"including",
|
||||
"include",
|
||||
"includes",
|
||||
"version",
|
||||
"update",
|
||||
"updates",
|
||||
"usage",
|
||||
]
|
||||
.into_iter()
|
||||
.collect()
|
||||
});
|
||||
|
||||
let mut counts: BTreeMap<String, u32> = BTreeMap::new();
|
||||
for token in TOKEN_RE.find_iter(text) {
|
||||
@@ -132,8 +134,9 @@ fn extract_keywords(text: &str, limit: usize) -> Vec<String> {
|
||||
}
|
||||
|
||||
fn derive_labels(text: &str, limit: usize) -> Vec<String> {
|
||||
static PHRASE_RE: std::sync::LazyLock<Regex> =
|
||||
std::sync::LazyLock::new(|| Regex::new(r"(?m)^(?P<phrase>[A-Z][A-Za-z0-9 &/-]{3,})$").unwrap());
|
||||
static PHRASE_RE: std::sync::LazyLock<Regex> = std::sync::LazyLock::new(|| {
|
||||
Regex::new(r"(?m)^(?P<phrase>[A-Z][A-Za-z0-9 &/-]{3,})$").unwrap()
|
||||
});
|
||||
|
||||
let mut labels = BTreeSet::new();
|
||||
for caps in PHRASE_RE.captures_iter(text) {
|
||||
@@ -218,7 +221,7 @@ mod tests {
|
||||
#[test]
|
||||
fn produces_keywords_and_labels() {
|
||||
let text = "Rust memory engines power efficient systems. Memory safety ensures reliability in 2025.";
|
||||
let result = AutoTagger::default().analyse(text, true);
|
||||
let result = AutoTagger.analyse(text, true);
|
||||
assert!(result.tags.iter().any(|tag| tag.contains("memory")));
|
||||
assert!(!result.content_dates.is_empty());
|
||||
}
|
||||
|
||||
+6
-6
@@ -84,13 +84,13 @@ pub static NER_MODELS: &[NerModelInfo] = &[NerModelInfo {
|
||||
}];
|
||||
|
||||
/// Get NER model info by name
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn get_ner_model_info(name: &str) -> Option<&'static NerModelInfo> {
|
||||
NER_MODELS.iter().find(|m| m.name == name)
|
||||
}
|
||||
|
||||
/// Get default NER model info
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn default_ner_model_info() -> &'static NerModelInfo {
|
||||
NER_MODELS
|
||||
.iter()
|
||||
@@ -119,7 +119,7 @@ pub struct ExtractedEntity {
|
||||
|
||||
impl ExtractedEntity {
|
||||
/// Convert the raw entity type to our `EntityKind` enum
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn to_entity_kind(&self) -> EntityKind {
|
||||
match self.entity_type.to_uppercase().as_str() {
|
||||
"PER" | "PERSON" | "B-PER" | "I-PER" => EntityKind::Person,
|
||||
@@ -562,19 +562,19 @@ impl NerModel {
|
||||
// ============================================================================
|
||||
|
||||
/// Get the expected path for the NER model in the models directory
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn ner_model_path(models_dir: &Path) -> PathBuf {
|
||||
models_dir.join(NER_MODEL_NAME).join("model.onnx")
|
||||
}
|
||||
|
||||
/// Get the expected path for the NER tokenizer in the models directory
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn ner_tokenizer_path(models_dir: &Path) -> PathBuf {
|
||||
models_dir.join(NER_MODEL_NAME).join("tokenizer.json")
|
||||
}
|
||||
|
||||
/// Check if NER model is installed
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn is_ner_model_installed(models_dir: &Path) -> bool {
|
||||
ner_model_path(models_dir).exists() && ner_tokenizer_path(models_dir).exists()
|
||||
}
|
||||
|
||||
+12
-11
@@ -35,6 +35,7 @@ use crate::{MemvidError, Result, types::FrameId};
|
||||
// ============================================================================
|
||||
|
||||
/// CLIP index decode limit (512MB max)
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
const CLIP_DECODE_LIMIT: usize = crate::MAX_INDEX_BYTES as usize;
|
||||
|
||||
/// MobileCLIP-S2 embedding dimensions
|
||||
@@ -140,7 +141,7 @@ pub static CLIP_MODELS: &[ClipModelInfo] = &[
|
||||
];
|
||||
|
||||
/// Get model info by name, defaults to mobileclip-s2
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn get_model_info(name: &str) -> &'static ClipModelInfo {
|
||||
CLIP_MODELS
|
||||
.iter()
|
||||
@@ -154,7 +155,7 @@ pub fn get_model_info(name: &str) -> &'static ClipModelInfo {
|
||||
}
|
||||
|
||||
/// Get the default model info
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn default_model_info() -> &'static ClipModelInfo {
|
||||
CLIP_MODELS
|
||||
.iter()
|
||||
@@ -185,7 +186,7 @@ pub struct ClipIndexBuilder {
|
||||
}
|
||||
|
||||
impl ClipIndexBuilder {
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
@@ -210,7 +211,7 @@ impl ClipIndexBuilder {
|
||||
let dimension = self
|
||||
.documents
|
||||
.first()
|
||||
.map_or(0, |doc| doc.embedding.len() as u32);
|
||||
.map_or(0, |doc| u32::try_from(doc.embedding.len()).unwrap_or(0));
|
||||
|
||||
Ok(ClipIndexArtifact {
|
||||
bytes,
|
||||
@@ -248,7 +249,7 @@ impl Default for ClipIndex {
|
||||
|
||||
impl ClipIndex {
|
||||
/// Create a new empty CLIP index
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
documents: Vec::new(),
|
||||
@@ -297,7 +298,7 @@ impl ClipIndex {
|
||||
}
|
||||
|
||||
/// Search for similar embeddings using L2 distance
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn search(&self, query: &[f32], limit: usize) -> Vec<ClipSearchHit> {
|
||||
if query.is_empty() {
|
||||
return Vec::new();
|
||||
@@ -333,7 +334,7 @@ impl ClipIndex {
|
||||
}
|
||||
|
||||
/// Get embedding for a specific frame
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn embedding_for(&self, frame_id: FrameId) -> Option<&[f32]> {
|
||||
self.documents
|
||||
.iter()
|
||||
@@ -347,13 +348,13 @@ impl ClipIndex {
|
||||
}
|
||||
|
||||
/// Number of documents in the index
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn len(&self) -> usize {
|
||||
self.documents.len()
|
||||
}
|
||||
|
||||
/// Check if index is empty
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.documents.is_empty()
|
||||
}
|
||||
@@ -366,7 +367,7 @@ impl ClipIndex {
|
||||
let dimension = self
|
||||
.documents
|
||||
.first()
|
||||
.map_or(0, |doc| doc.embedding.len() as u32);
|
||||
.map_or(0, |doc| u32::try_from(doc.embedding.len()).unwrap_or(0));
|
||||
|
||||
Ok(ClipIndexArtifact {
|
||||
bytes,
|
||||
@@ -411,7 +412,7 @@ pub struct ImageInfo {
|
||||
|
||||
impl ImageInfo {
|
||||
/// Check if this image should be processed for CLIP embedding
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn should_embed(&self) -> bool {
|
||||
// Skip tiny images (icons, bullets)
|
||||
if self.width < MIN_IMAGE_DIM || self.height < MIN_IMAGE_DIM {
|
||||
|
||||
@@ -149,10 +149,10 @@ mod tests {
|
||||
struct TestEngine;
|
||||
|
||||
impl EnrichmentEngine for TestEngine {
|
||||
fn kind(&self) -> &str {
|
||||
fn kind(&self) -> &'static str {
|
||||
"test"
|
||||
}
|
||||
fn version(&self) -> &str {
|
||||
fn version(&self) -> &'static str {
|
||||
"1.0.0"
|
||||
}
|
||||
fn enrich(&self, _ctx: &EnrichmentContext) -> EnrichmentResult {
|
||||
|
||||
+14
-10
@@ -72,7 +72,7 @@ pub struct EnrichmentWorkerHandle {
|
||||
|
||||
impl EnrichmentWorkerHandle {
|
||||
/// Create a new worker handle.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
stop_signal: Arc::new(AtomicBool::new(false)),
|
||||
@@ -90,19 +90,19 @@ impl EnrichmentWorkerHandle {
|
||||
}
|
||||
|
||||
/// Check if stop was requested.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn should_stop(&self) -> bool {
|
||||
self.stop_signal.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
/// Check if worker is currently running.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn is_running(&self) -> bool {
|
||||
self.is_running.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
/// Get current statistics.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn stats(&self) -> EnrichmentWorkerStats {
|
||||
EnrichmentWorkerStats {
|
||||
frames_processed: self.frames_processed.load(Ordering::Relaxed),
|
||||
@@ -141,7 +141,7 @@ impl EnrichmentWorkerHandle {
|
||||
}
|
||||
|
||||
/// Clone the handle for sharing with the worker thread.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn clone_handle(&self) -> Self {
|
||||
Self {
|
||||
stop_signal: Arc::clone(&self.stop_signal),
|
||||
@@ -266,7 +266,7 @@ pub struct EnrichmentProcessor {
|
||||
|
||||
impl EnrichmentProcessor {
|
||||
/// Create a new enrichment processor.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn new(config: EnrichmentWorkerConfig) -> Self {
|
||||
Self { config }
|
||||
}
|
||||
@@ -306,9 +306,11 @@ impl EnrichmentProcessor {
|
||||
};
|
||||
|
||||
// Read current frame state
|
||||
let (text, is_skim, _needs_embedding) = if let Some(data) = read_frame(task.frame_id) { data } else {
|
||||
let (text, is_skim, _needs_embedding) = if let Some(data) = read_frame(task.frame_id) {
|
||||
data
|
||||
} else {
|
||||
result.error = Some("Frame not found".to_string());
|
||||
result.elapsed_ms = start.elapsed().as_millis() as u64;
|
||||
result.elapsed_ms = start.elapsed().as_millis().try_into().unwrap_or(u64::MAX);
|
||||
return result;
|
||||
};
|
||||
|
||||
@@ -337,7 +339,7 @@ impl EnrichmentProcessor {
|
||||
result.error = Some(format!("Index update failed: {err}"));
|
||||
}
|
||||
|
||||
result.elapsed_ms = start.elapsed().as_millis() as u64;
|
||||
result.elapsed_ms = start.elapsed().as_millis().try_into().unwrap_or(u64::MAX);
|
||||
result
|
||||
}
|
||||
}
|
||||
@@ -374,7 +376,9 @@ pub fn run_worker_loop<G, P, M, C>(
|
||||
|
||||
while !handle.should_stop() {
|
||||
// Get next task
|
||||
let task = if let Some(task) = get_next_task() { task } else {
|
||||
let task = if let Some(task) = get_next_task() {
|
||||
task
|
||||
} else {
|
||||
// Queue is empty, wait and check again
|
||||
std::thread::sleep(Duration::from_millis(config.task_delay_ms * 10));
|
||||
continue;
|
||||
|
||||
+7
-8
@@ -30,7 +30,7 @@ pub struct ExtractedDocument {
|
||||
}
|
||||
|
||||
impl ExtractedDocument {
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn empty() -> Self {
|
||||
Self {
|
||||
text: None,
|
||||
@@ -285,7 +285,7 @@ impl Default for DocumentProcessor {
|
||||
|
||||
#[cfg(not(feature = "extractous"))]
|
||||
impl DocumentProcessor {
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn new(config: ProcessorConfig) -> Self {
|
||||
Self {
|
||||
max_length: config.max_text_chars,
|
||||
@@ -790,12 +790,11 @@ fn pdf_text_extract_lopdf(bytes: &[u8]) -> Result<Option<String>> {
|
||||
})?;
|
||||
|
||||
// Try to decrypt if encrypted (empty password for unprotected PDFs)
|
||||
if document.is_encrypted()
|
||||
&& document.decrypt("").is_err() {
|
||||
return Err(MemvidError::ExtractionFailed {
|
||||
reason: "cannot decrypt password-protected PDF".into(),
|
||||
});
|
||||
}
|
||||
if document.is_encrypted() && document.decrypt("").is_err() {
|
||||
return Err(MemvidError::ExtractionFailed {
|
||||
reason: "cannot decrypt password-protected PDF".into(),
|
||||
});
|
||||
}
|
||||
|
||||
// Decompress streams for better text extraction
|
||||
let () = document.decompress();
|
||||
|
||||
+16
-17
@@ -43,7 +43,7 @@ impl Default for ExtractionBudget {
|
||||
|
||||
impl ExtractionBudget {
|
||||
/// Create a budget with custom milliseconds.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn with_ms(ms: u64) -> Self {
|
||||
Self {
|
||||
budget: Duration::from_millis(ms),
|
||||
@@ -52,7 +52,7 @@ impl ExtractionBudget {
|
||||
}
|
||||
|
||||
/// Create an unlimited budget (extract everything).
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn unlimited() -> Self {
|
||||
Self {
|
||||
budget: Duration::from_secs(3600), // 1 hour = effectively unlimited
|
||||
@@ -81,13 +81,13 @@ pub struct BudgetedExtractionResult {
|
||||
|
||||
impl BudgetedExtractionResult {
|
||||
/// Check if we got meaningful content.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn has_content(&self) -> bool {
|
||||
!self.text.trim().is_empty()
|
||||
}
|
||||
|
||||
/// Check if this is a skim (partial) extraction.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn is_skim(&self) -> bool {
|
||||
!self.completed && self.sections_extracted < self.sections_total
|
||||
}
|
||||
@@ -138,7 +138,7 @@ pub fn extract_pdf_budgeted(
|
||||
sections_extracted: estimated_pages,
|
||||
sections_total: estimated_pages,
|
||||
completed,
|
||||
elapsed_ms: start.elapsed().as_millis() as u64,
|
||||
elapsed_ms: start.elapsed().as_millis().try_into().unwrap_or(u64::MAX),
|
||||
coverage: 1.0,
|
||||
});
|
||||
}
|
||||
@@ -180,7 +180,7 @@ pub fn extract_pdf_budgeted(
|
||||
sections_extracted: estimated_pages,
|
||||
sections_total: estimated_pages,
|
||||
completed,
|
||||
elapsed_ms: start.elapsed().as_millis() as u64,
|
||||
elapsed_ms: start.elapsed().as_millis().try_into().unwrap_or(u64::MAX),
|
||||
coverage: 1.0,
|
||||
});
|
||||
}
|
||||
@@ -233,12 +233,11 @@ fn extract_pdf_budgeted_lopdf(
|
||||
})?;
|
||||
|
||||
// Handle encryption
|
||||
if document.is_encrypted()
|
||||
&& document.decrypt("").is_err() {
|
||||
return Err(MemvidError::ExtractionFailed {
|
||||
reason: "cannot decrypt password-protected PDF".into(),
|
||||
});
|
||||
}
|
||||
if document.is_encrypted() && document.decrypt("").is_err() {
|
||||
return Err(MemvidError::ExtractionFailed {
|
||||
reason: "cannot decrypt password-protected PDF".into(),
|
||||
});
|
||||
}
|
||||
|
||||
// Decompress for better extraction
|
||||
let () = document.decompress();
|
||||
@@ -251,7 +250,7 @@ fn extract_pdf_budgeted_lopdf(
|
||||
sections_extracted: 0,
|
||||
sections_total: 0,
|
||||
completed: true,
|
||||
elapsed_ms: start.elapsed().as_millis() as u64,
|
||||
elapsed_ms: start.elapsed().as_millis().try_into().unwrap_or(u64::MAX),
|
||||
coverage: 1.0,
|
||||
});
|
||||
}
|
||||
@@ -360,7 +359,7 @@ pub fn extract_text_budgeted(
|
||||
sections_extracted: sections,
|
||||
sections_total: sections,
|
||||
completed: true,
|
||||
elapsed_ms: start.elapsed().as_millis() as u64,
|
||||
elapsed_ms: start.elapsed().as_millis().try_into().unwrap_or(u64::MAX),
|
||||
coverage: 1.0,
|
||||
})
|
||||
}
|
||||
@@ -429,7 +428,7 @@ fn extract_ooxml_budgeted(
|
||||
sections_extracted: sections,
|
||||
sections_total: sections,
|
||||
completed: true,
|
||||
elapsed_ms: start.elapsed().as_millis() as u64,
|
||||
elapsed_ms: start.elapsed().as_millis().try_into().unwrap_or(u64::MAX),
|
||||
coverage: 1.0,
|
||||
})
|
||||
}
|
||||
@@ -442,7 +441,7 @@ fn extract_ooxml_budgeted(
|
||||
sections_extracted: 0,
|
||||
sections_total: 0,
|
||||
completed: true,
|
||||
elapsed_ms: start.elapsed().as_millis() as u64,
|
||||
elapsed_ms: start.elapsed().as_millis().try_into().unwrap_or(u64::MAX),
|
||||
coverage: 1.0,
|
||||
})
|
||||
}
|
||||
@@ -588,7 +587,7 @@ fn finish_extraction(
|
||||
sections_extracted,
|
||||
sections_total: total_pages,
|
||||
completed,
|
||||
elapsed_ms: start.elapsed().as_millis() as u64,
|
||||
elapsed_ms: start.elapsed().as_millis().try_into().unwrap_or(u64::MAX),
|
||||
coverage,
|
||||
})
|
||||
}
|
||||
|
||||
+1
-1
@@ -91,7 +91,7 @@ pub fn find_last_valid_footer(bytes: &[u8]) -> Option<FooterSlice<'_>> {
|
||||
let candidate = &bytes[pos..pos + FOOTER_SIZE];
|
||||
if let Some(footer) = CommitFooter::decode(candidate) {
|
||||
let toc_end = pos;
|
||||
let toc_len = footer.toc_len as usize;
|
||||
let toc_len = usize::try_from(footer.toc_len).unwrap_or(0);
|
||||
if toc_len == 0 || toc_len > toc_end {
|
||||
search_end = pos;
|
||||
continue;
|
||||
|
||||
+2
-2
@@ -193,13 +193,13 @@ pub struct GraphMatcher<'a> {
|
||||
|
||||
impl<'a> GraphMatcher<'a> {
|
||||
/// Create a new graph matcher.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn new(memvid: &'a Memvid) -> Self {
|
||||
Self { memvid }
|
||||
}
|
||||
|
||||
/// Execute a graph pattern and return matching results.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn execute(&self, pattern: &GraphPattern) -> Vec<GraphMatchResult> {
|
||||
let mut results = Vec::new();
|
||||
|
||||
|
||||
@@ -102,7 +102,8 @@ impl ManifestWal {
|
||||
let checksum = hash(&payload);
|
||||
self.file.seek(SeekFrom::Start(self.write_offset))?;
|
||||
// Safe: validated payload.len() <= MAX_RECORD_BYTES (4MB) on line 96
|
||||
self.file.write_all(&(payload.len() as u32).to_le_bytes())?;
|
||||
self.file
|
||||
.write_all(&(u32::try_from(payload.len()).unwrap_or(u32::MAX)).to_le_bytes())?;
|
||||
self.file.write_all(checksum.as_bytes())?;
|
||||
self.file.write_all(&payload)?;
|
||||
|
||||
|
||||
@@ -96,6 +96,7 @@ pub fn read_track<R: Read + Seek>(
|
||||
}
|
||||
|
||||
// Safe: count validated by checked_mul and payload_bytes comparison above
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
let mut entries = Vec::with_capacity(count as usize);
|
||||
let mut prev: Option<TimeIndexEntry> = None;
|
||||
for _ in 0..count {
|
||||
|
||||
+6
-4
@@ -231,7 +231,8 @@ impl EmbeddedWal {
|
||||
let digest = blake3::hash(payload);
|
||||
let mut header = [0u8; ENTRY_HEADER_SIZE];
|
||||
header[..8].copy_from_slice(&sequence.to_le_bytes());
|
||||
header[8..12].copy_from_slice(&(payload.len() as u32).to_le_bytes());
|
||||
header[8..12]
|
||||
.copy_from_slice(&(u32::try_from(payload.len()).unwrap_or(u32::MAX)).to_le_bytes());
|
||||
header[16..48].copy_from_slice(digest.as_bytes());
|
||||
|
||||
// Atomic write: combine header and payload into single buffer
|
||||
@@ -264,6 +265,7 @@ impl EmbeddedWal {
|
||||
if remaining < ENTRY_HEADER_SIZE as u64 {
|
||||
if remaining > 0 {
|
||||
// Safe: remaining < ENTRY_HEADER_SIZE (48) so always fits in usize
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
let zero_tail = vec![0u8; remaining as usize];
|
||||
self.seek_and_write(pos, &zero_tail)?;
|
||||
}
|
||||
@@ -458,13 +460,13 @@ mod tests {
|
||||
let (file, mut header) = prepare_wal(size);
|
||||
let mut wal = EmbeddedWal::open(&file, &header).expect("open wal");
|
||||
|
||||
wal.append_entry(&vec![0xAA; 32]).expect("append a");
|
||||
wal.append_entry(&vec![0xBB; 32]).expect("append b");
|
||||
wal.append_entry(&[0xAA; 32]).expect("append a");
|
||||
wal.append_entry(&[0xBB; 32]).expect("append b");
|
||||
wal.record_checkpoint(&mut header).expect("checkpoint");
|
||||
|
||||
assert!(wal.pending_records().expect("pending").is_empty());
|
||||
|
||||
wal.append_entry(&vec![0xCC; 32]).expect("append c");
|
||||
wal.append_entry(&[0xCC; 32]).expect("append c");
|
||||
let records = wal.pending_records().expect("after append");
|
||||
assert_eq!(records.len(), 1);
|
||||
assert_eq!(records[0].payload, vec![0xCC; 32]);
|
||||
|
||||
+4
-3
@@ -15,6 +15,7 @@ fn lex_config() -> impl bincode::config::Config {
|
||||
.with_little_endian()
|
||||
}
|
||||
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
const LEX_DECODE_LIMIT: usize = crate::MAX_INDEX_BYTES as usize;
|
||||
const LEX_SECTION_SOFT_CHARS: usize = 900;
|
||||
const LEX_SECTION_HARD_CHARS: usize = 1400;
|
||||
@@ -27,7 +28,7 @@ pub struct LexIndexBuilder {
|
||||
}
|
||||
|
||||
impl LexIndexBuilder {
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
@@ -150,7 +151,7 @@ impl LexIndex {
|
||||
Self { documents }
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn search(&self, query: &str, limit: usize) -> Vec<LexSearchHit> {
|
||||
let mut query_tokens = tokenize(query);
|
||||
query_tokens.retain(|token| !token.is_empty());
|
||||
@@ -684,7 +685,7 @@ mod tests {
|
||||
|
||||
let artifact = builder.finish().expect("finish");
|
||||
assert_eq!(artifact.doc_count, 2);
|
||||
assert!(artifact.bytes.len() > 0);
|
||||
assert!(!artifact.bytes.is_empty());
|
||||
|
||||
let index = LexIndex::decode(&artifact.bytes).expect("decode");
|
||||
let hits = index.search("rust", 10);
|
||||
|
||||
+16
-6
@@ -1,5 +1,15 @@
|
||||
#![deny(clippy::all, clippy::pedantic)]
|
||||
#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used))]
|
||||
#![cfg_attr(
|
||||
test,
|
||||
allow(
|
||||
clippy::useless_vec,
|
||||
clippy::uninlined_format_args,
|
||||
clippy::cast_possible_truncation,
|
||||
clippy::float_cmp,
|
||||
clippy::cast_precision_loss
|
||||
)
|
||||
)]
|
||||
#![allow(clippy::module_name_repetitions)]
|
||||
//
|
||||
// Strategic lint exceptions - these are allowed project-wide for pragmatic reasons:
|
||||
@@ -13,7 +23,6 @@
|
||||
// Cast safety: All casts in this codebase are carefully reviewed and bounded by
|
||||
// real-world constraints (file sizes, frame counts, etc). Using try_into() everywhere
|
||||
// would add significant complexity without safety benefits in our use case.
|
||||
#![allow(clippy::cast_possible_truncation)]
|
||||
#![allow(clippy::cast_precision_loss)]
|
||||
#![allow(clippy::cast_possible_wrap)]
|
||||
#![allow(clippy::cast_sign_loss)]
|
||||
@@ -322,6 +331,7 @@ const MAX_FRAME_BYTES: u64 = 256 * 1024 * 1024;
|
||||
const DEFAULT_SEARCH_TEXT_LIMIT: usize = 32_768;
|
||||
|
||||
#[cfg(test)]
|
||||
#[allow(clippy::non_std_lazy_statics)]
|
||||
static SERIAL_TEST_MUTEX: Lazy<Mutex<()>> = Lazy::new(|| Mutex::new(()));
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -1160,7 +1170,7 @@ mod tests {
|
||||
.expect("open file");
|
||||
file.seek(SeekFrom::Start(manifest.bytes_offset))
|
||||
.expect("seek");
|
||||
let zeros = vec![0u8; manifest.bytes_length as usize];
|
||||
let zeros = vec![0u8; usize::try_from(manifest.bytes_length).unwrap_or(0)];
|
||||
file.write_all(&zeros).expect("corrupt time index");
|
||||
file.flush().expect("flush");
|
||||
file.sync_all().expect("sync");
|
||||
@@ -1178,7 +1188,7 @@ mod tests {
|
||||
assert_eq!(report.overall_status, VerificationStatus::Failed);
|
||||
}
|
||||
Err(e) => {
|
||||
println!("test: verify failed with error (expected): {}", e);
|
||||
println!("test: verify failed with error (expected): {e}");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1330,6 +1340,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[allow(deprecated)]
|
||||
fn ticket_sequence_enforced() {
|
||||
run_serial_test(|| {
|
||||
let dir = tempdir().expect("tmp");
|
||||
@@ -1347,6 +1358,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[allow(deprecated)]
|
||||
fn capacity_limit_enforced() {
|
||||
run_serial_test(|| {
|
||||
let dir = tempdir().expect("tmp");
|
||||
@@ -1360,9 +1372,7 @@ mod tests {
|
||||
mem.put_bytes(&vec![0xFF; 32]).expect("first put");
|
||||
mem.commit().expect("commit");
|
||||
|
||||
let err = mem
|
||||
.put_bytes(&vec![0xFF; 40])
|
||||
.expect_err("capacity exceeded");
|
||||
let err = mem.put_bytes(&[0xFF; 40]).expect_err("capacity exceeded");
|
||||
assert!(matches!(err, MemvidError::CapacityExceeded { .. }));
|
||||
});
|
||||
}
|
||||
|
||||
+1
-1
@@ -81,7 +81,7 @@ impl FileLock {
|
||||
Ok(self.file.try_clone()?)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn mode(&self) -> LockMode {
|
||||
self.mode
|
||||
}
|
||||
|
||||
+9
-4
@@ -502,12 +502,17 @@ impl Memvid {
|
||||
.map(|manifest| manifest.dimension)
|
||||
.filter(|dim| *dim > 0)
|
||||
.or_else(|| {
|
||||
self.vec_index
|
||||
.as_ref()
|
||||
.and_then(|index| index.entries().next().map(|(_, emb)| emb.len() as u32))
|
||||
self.vec_index.as_ref().and_then(|index| {
|
||||
index
|
||||
.entries()
|
||||
.next()
|
||||
.map(|(_, emb)| u32::try_from(emb.len()).unwrap_or(0))
|
||||
})
|
||||
})
|
||||
.unwrap_or(0);
|
||||
if stored_dimension > 0 && query_embedding.len() as u32 != stored_dimension {
|
||||
if stored_dimension > 0
|
||||
&& u32::try_from(query_embedding.len()).unwrap_or(u32::MAX) != stored_dimension
|
||||
{
|
||||
return Err(MemvidError::VecDimensionMismatch {
|
||||
expected: stored_dimension,
|
||||
actual: query_embedding.len(),
|
||||
|
||||
@@ -297,7 +297,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn structural_chunking_keeps_small_table_whole() {
|
||||
let text = r#"# Small Report
|
||||
let text = r"# Small Report
|
||||
|
||||
Introduction paragraph.
|
||||
|
||||
@@ -307,7 +307,7 @@ Introduction paragraph.
|
||||
| Orange | $2 |
|
||||
|
||||
Conclusion.
|
||||
"#
|
||||
"
|
||||
.repeat(50); // Repeat to meet minimum size
|
||||
|
||||
let plan = plan_document_chunks(text.as_bytes()).expect("chunk plan");
|
||||
@@ -327,7 +327,7 @@ Conclusion.
|
||||
|
||||
#[test]
|
||||
fn structural_chunking_detects_code_blocks() {
|
||||
let text = r#"# Code Example
|
||||
let text = r"# Code Example
|
||||
|
||||
Here is some code:
|
||||
|
||||
@@ -347,7 +347,7 @@ class DataProcessor:
|
||||
self.data.append(item)
|
||||
```
|
||||
|
||||
More explanation here. "#
|
||||
More explanation here. "
|
||||
.repeat(20);
|
||||
|
||||
let plan = plan_document_chunks(text.as_bytes()).expect("chunk plan");
|
||||
|
||||
+12
-4
@@ -106,6 +106,7 @@ fn try_recover_from_wal_corruption(path: &Path) -> Result<Memvid> {
|
||||
);
|
||||
|
||||
// Zero out the entire WAL region to create a clean slate
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
let wal_size = header.wal_size as usize;
|
||||
let zeros = vec![0u8; min(1024 * 1024, wal_size)]; // Write in 1MB chunks
|
||||
let mut written = 0;
|
||||
@@ -619,6 +620,7 @@ impl DoctorPlanner {
|
||||
));
|
||||
return;
|
||||
}
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
let mut buf = vec![0u8; manifest.bytes_length as usize];
|
||||
if let Err(err) = file.seek(SeekFrom::Start(manifest.bytes_offset)) {
|
||||
probe.index.needs_lex = true;
|
||||
@@ -728,6 +730,7 @@ impl DoctorPlanner {
|
||||
}
|
||||
|
||||
// Read and validate segment
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
let mut buf = vec![0u8; segment.common.bytes_length as usize];
|
||||
if let Err(err) = file.seek(SeekFrom::Start(segment.common.bytes_offset)) {
|
||||
probe.index.needs_vec = true;
|
||||
@@ -799,6 +802,7 @@ impl DoctorPlanner {
|
||||
return;
|
||||
}
|
||||
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
let mut buf = vec![0u8; manifest.bytes_length as usize];
|
||||
if let Err(err) = file.seek(SeekFrom::Start(manifest.bytes_offset)) {
|
||||
probe.index.needs_vec = true;
|
||||
@@ -1225,7 +1229,11 @@ impl DoctorExecutor {
|
||||
|
||||
metrics.phase_durations.push(DoctorPhaseDuration {
|
||||
phase: phase.phase,
|
||||
duration_ms: phase_start.elapsed().as_millis() as u64,
|
||||
duration_ms: phase_start
|
||||
.elapsed()
|
||||
.as_millis()
|
||||
.try_into()
|
||||
.unwrap_or(u64::MAX),
|
||||
});
|
||||
metrics.actions_completed += actions
|
||||
.iter()
|
||||
@@ -1251,7 +1259,7 @@ impl DoctorExecutor {
|
||||
}
|
||||
}
|
||||
|
||||
metrics.total_duration_ms = start.elapsed().as_millis() as u64;
|
||||
metrics.total_duration_ms = start.elapsed().as_millis().try_into().unwrap_or(u64::MAX);
|
||||
|
||||
if overall_failed {
|
||||
if let Some(original) = &original_header {
|
||||
@@ -1499,10 +1507,10 @@ impl DoctorExecutor {
|
||||
);
|
||||
let mut remaining = mem.header.wal_size;
|
||||
let mut offset = mem.header.wal_offset;
|
||||
let chunk_size = min(remaining as usize, 4096).max(1);
|
||||
let chunk_size = (remaining.min(4096) as usize).max(1);
|
||||
let zeros = vec![0u8; chunk_size];
|
||||
while remaining > 0 {
|
||||
let write_len = min(remaining as usize, zeros.len());
|
||||
let write_len = usize::try_from(remaining.min(zeros.len() as u64)).unwrap_or(0);
|
||||
mem.file.seek(SeekFrom::Start(offset))?;
|
||||
mem.file.write_all(&zeros[..write_len])?;
|
||||
remaining -= write_len as u64;
|
||||
|
||||
+17
-17
@@ -30,7 +30,7 @@ pub struct EnrichmentHandle {
|
||||
|
||||
impl EnrichmentHandle {
|
||||
/// Stop the worker and wait for it to finish.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn stop_and_wait(mut self) -> EnrichmentWorkerStats {
|
||||
self.handle.stop();
|
||||
if let Some(thread) = self.thread.take() {
|
||||
@@ -40,13 +40,13 @@ impl EnrichmentHandle {
|
||||
}
|
||||
|
||||
/// Check if worker is still running.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn is_running(&self) -> bool {
|
||||
self.handle.is_running()
|
||||
}
|
||||
|
||||
/// Get current statistics.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn stats(&self) -> EnrichmentWorkerStats {
|
||||
self.handle.stats()
|
||||
}
|
||||
@@ -216,19 +216,19 @@ where
|
||||
|
||||
impl Memvid {
|
||||
/// Get the number of frames pending enrichment.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn enrichment_queue_len(&self) -> usize {
|
||||
self.toc.enrichment_queue.len()
|
||||
}
|
||||
|
||||
/// Check if any frames need enrichment.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn has_pending_enrichment(&self) -> bool {
|
||||
!self.toc.enrichment_queue.is_empty()
|
||||
}
|
||||
|
||||
/// Get the next task from the enrichment queue.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn next_enrichment_task(&self) -> Option<EnrichmentTask> {
|
||||
self.toc.enrichment_queue.tasks.first().cloned()
|
||||
}
|
||||
@@ -242,7 +242,7 @@ impl Memvid {
|
||||
/// Read frame data needed for enrichment.
|
||||
///
|
||||
/// Returns (`search_text`, `is_skim`, `needs_embedding`) if frame exists.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn read_frame_for_enrichment(&self, frame_id: FrameId) -> Option<(String, bool, bool)> {
|
||||
let frame = self
|
||||
.toc
|
||||
@@ -401,7 +401,7 @@ impl Memvid {
|
||||
// Mark frame as enriched
|
||||
self.mark_frame_enriched(task.frame_id);
|
||||
|
||||
result.elapsed_ms = start.elapsed().as_millis() as u64;
|
||||
result.elapsed_ms = start.elapsed().as_millis().try_into().unwrap_or(u64::MAX);
|
||||
result
|
||||
}
|
||||
|
||||
@@ -437,7 +437,7 @@ impl Memvid {
|
||||
}
|
||||
|
||||
/// Get enrichment statistics.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn enrichment_stats(&self) -> EnrichmentStats {
|
||||
let total_frames = self
|
||||
.toc
|
||||
@@ -602,7 +602,7 @@ impl Memvid {
|
||||
frames_processed += 1;
|
||||
|
||||
// Update checkpoint in queue for crash recovery
|
||||
let chunks_done = embeddings_generated as u32;
|
||||
let chunks_done = u32::try_from(embeddings_generated).unwrap_or(u32::MAX);
|
||||
self.toc.enrichment_queue.update_checkpoint(
|
||||
task.frame_id,
|
||||
chunks_done,
|
||||
@@ -638,19 +638,19 @@ impl Memvid {
|
||||
}
|
||||
|
||||
/// Check if vector embeddings are enabled.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn has_embeddings(&self) -> bool {
|
||||
self.vec_enabled && self.vec_index.is_some()
|
||||
}
|
||||
|
||||
/// Get vector count from the index.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn vector_count(&self) -> usize {
|
||||
self.toc
|
||||
.indexes
|
||||
.vec
|
||||
.as_ref()
|
||||
.map_or(0, |m| m.vector_count as usize)
|
||||
self.toc.indexes.vec.as_ref().map_or(0, |m| {
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
let count = m.vector_count as usize;
|
||||
count
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+69
-44
@@ -70,6 +70,7 @@ impl Read for BlobReader {
|
||||
if remaining == 0 {
|
||||
return Ok(0);
|
||||
}
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
let to_read = remaining.min(buf.len() as u64) as usize;
|
||||
file.seek(SeekFrom::Start(*start + *pos))?;
|
||||
let read = file.read(&mut buf[..to_read])?;
|
||||
@@ -161,9 +162,11 @@ fn mime_is_text(mime: &str) -> bool {
|
||||
|
||||
impl Memvid {
|
||||
pub fn frame_by_id(&self, frame_id: FrameId) -> Result<Frame> {
|
||||
let index =
|
||||
usize::try_from(frame_id).map_err(|_| MemvidError::FrameNotFound { frame_id })?;
|
||||
self.toc
|
||||
.frames
|
||||
.get(frame_id as usize)
|
||||
.get(index)
|
||||
.cloned()
|
||||
.ok_or(MemvidError::FrameNotFound { frame_id })
|
||||
}
|
||||
@@ -201,7 +204,7 @@ impl Memvid {
|
||||
/// we can skip re-ingestion. The hash is computed from the original file bytes.
|
||||
///
|
||||
/// Returns `None` if no matching frame is found.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn find_frame_by_hash(&self, hash: &[u8; 32]) -> Option<&Frame> {
|
||||
self.toc
|
||||
.frames
|
||||
@@ -254,11 +257,17 @@ impl Memvid {
|
||||
}
|
||||
|
||||
pub fn frame_preview_by_id(&mut self, frame_id: FrameId) -> Result<String> {
|
||||
let frame = self.toc.frames.get(frame_id as usize).cloned().ok_or(
|
||||
MemvidError::InvalidTimeIndex {
|
||||
let index = usize::try_from(frame_id).map_err(|_| MemvidError::InvalidTimeIndex {
|
||||
reason: "frame id too large".into(),
|
||||
})?;
|
||||
let frame = self
|
||||
.toc
|
||||
.frames
|
||||
.get(index)
|
||||
.cloned()
|
||||
.ok_or(MemvidError::InvalidTimeIndex {
|
||||
reason: "frame id out of range".into(),
|
||||
},
|
||||
)?;
|
||||
})?;
|
||||
self.frame_preview(&frame)
|
||||
}
|
||||
|
||||
@@ -267,11 +276,17 @@ impl Memvid {
|
||||
/// Unlike `frame_preview_by_id` which truncates for display purposes,
|
||||
/// this returns the complete text content suitable for LLM processing.
|
||||
pub fn frame_text_by_id(&mut self, frame_id: FrameId) -> Result<String> {
|
||||
let frame = self.toc.frames.get(frame_id as usize).cloned().ok_or(
|
||||
MemvidError::InvalidTimeIndex {
|
||||
let index = usize::try_from(frame_id).map_err(|_| MemvidError::InvalidTimeIndex {
|
||||
reason: "frame id too large".into(),
|
||||
})?;
|
||||
let frame = self
|
||||
.toc
|
||||
.frames
|
||||
.get(index)
|
||||
.cloned()
|
||||
.ok_or(MemvidError::InvalidTimeIndex {
|
||||
reason: "frame id out of range".into(),
|
||||
},
|
||||
)?;
|
||||
})?;
|
||||
self.frame_content(&frame)
|
||||
}
|
||||
|
||||
@@ -337,19 +352,24 @@ impl Memvid {
|
||||
return Ok(None);
|
||||
}
|
||||
self.ensure_vec_index()?;
|
||||
Ok(self.vec_index.as_ref().and_then(|index| {
|
||||
index
|
||||
.embedding_for(frame_id)
|
||||
.map(<[f32]>::to_vec)
|
||||
}))
|
||||
Ok(self
|
||||
.vec_index
|
||||
.as_ref()
|
||||
.and_then(|index| index.embedding_for(frame_id).map(<[f32]>::to_vec)))
|
||||
}
|
||||
|
||||
pub fn frame_context(&mut self, frame_id: FrameId, query: &str) -> Result<(String, usize)> {
|
||||
let frame = self.toc.frames.get(frame_id as usize).cloned().ok_or(
|
||||
MemvidError::InvalidTimeIndex {
|
||||
let index = usize::try_from(frame_id).map_err(|_| MemvidError::InvalidTimeIndex {
|
||||
reason: "frame id too large".into(),
|
||||
})?;
|
||||
let frame = self
|
||||
.toc
|
||||
.frames
|
||||
.get(index)
|
||||
.cloned()
|
||||
.ok_or(MemvidError::InvalidTimeIndex {
|
||||
reason: "frame id out of range".into(),
|
||||
},
|
||||
)?;
|
||||
})?;
|
||||
let preview = self.frame_preview(&frame)?;
|
||||
let content = self.frame_content(&frame)?;
|
||||
let count = query
|
||||
@@ -362,15 +382,14 @@ impl Memvid {
|
||||
}
|
||||
|
||||
pub(crate) fn frame_canonical_bytes(&mut self, frame: &Frame) -> Result<Vec<u8>> {
|
||||
if frame.role == FrameRole::Document
|
||||
&& frame.chunk_manifest.is_some() {
|
||||
let chunks = self.document_chunk_payloads(frame)?;
|
||||
let mut buffer = Vec::new();
|
||||
for (_, bytes) in chunks {
|
||||
buffer.extend_from_slice(&bytes);
|
||||
}
|
||||
return Ok(buffer);
|
||||
if frame.role == FrameRole::Document && frame.chunk_manifest.is_some() {
|
||||
let chunks = self.document_chunk_payloads(frame)?;
|
||||
let mut buffer = Vec::new();
|
||||
for (_, bytes) in chunks {
|
||||
buffer.extend_from_slice(&bytes);
|
||||
}
|
||||
return Ok(buffer);
|
||||
}
|
||||
let raw = self.read_frame_payload_bytes(frame)?;
|
||||
let decoded = crate::decode_canonical_bytes(&raw, frame.canonical_encoding, frame.id)?;
|
||||
if let Some(expected) = frame.canonical_length {
|
||||
@@ -410,6 +429,7 @@ impl Memvid {
|
||||
.canonical_length
|
||||
.or(Some(frame.payload_length))
|
||||
.unwrap_or(frame.payload_length);
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
return Ok(Self::render_binary_summary(logical as usize));
|
||||
}
|
||||
}
|
||||
@@ -529,24 +549,27 @@ impl Memvid {
|
||||
FrameRole::DocumentChunk => {
|
||||
// Try to resolve via parent's chunk manifest (new format)
|
||||
if let Some(parent_id) = frame.parent_id {
|
||||
if let Some(parent) = self.toc.frames.get(parent_id as usize).cloned() {
|
||||
if parent.chunk_manifest.is_some() {
|
||||
if let Ok(payloads) = self.document_chunk_payloads(&parent) {
|
||||
if let Some(idx) = frame.chunk_index {
|
||||
let idx = idx as usize;
|
||||
if idx < payloads.len() {
|
||||
let mut offset = 0usize;
|
||||
for (_, bytes) in payloads.iter().take(idx) {
|
||||
offset += bytes.len();
|
||||
// Safe frame lookup
|
||||
if let Ok(index) = usize::try_from(parent_id) {
|
||||
if let Some(parent) = self.toc.frames.get(index).cloned() {
|
||||
if parent.chunk_manifest.is_some() {
|
||||
if let Ok(payloads) = self.document_chunk_payloads(&parent) {
|
||||
if let Some(idx) = frame.chunk_index {
|
||||
let idx = idx as usize;
|
||||
if idx < payloads.len() {
|
||||
let mut offset = 0usize;
|
||||
for (_, bytes) in payloads.iter().take(idx) {
|
||||
offset += bytes.len();
|
||||
}
|
||||
let (_, bytes) = &payloads[idx];
|
||||
let text = String::from_utf8_lossy(bytes).into_owned();
|
||||
let end = offset + bytes.len();
|
||||
return Ok(ChunkInfo {
|
||||
start: offset,
|
||||
end,
|
||||
text,
|
||||
});
|
||||
}
|
||||
let (_, bytes) = &payloads[idx];
|
||||
let text = String::from_utf8_lossy(bytes).into_owned();
|
||||
let end = offset + bytes.len();
|
||||
return Ok(ChunkInfo {
|
||||
start: offset,
|
||||
end,
|
||||
text,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -588,6 +611,8 @@ impl Memvid {
|
||||
pub(crate) fn read_frame_payload_bytes(&mut self, frame: &Frame) -> Result<Vec<u8>> {
|
||||
self.validate_frame_bounds(frame)?;
|
||||
self.file.seek(SeekFrom::Start(frame.payload_offset))?;
|
||||
// Safe: guarded by MAX_FRAME_BYTES check
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
let mut buf = vec![0u8; frame.payload_length as usize];
|
||||
self.file.read_exact(&mut buf)?;
|
||||
Ok(buf)
|
||||
|
||||
@@ -13,7 +13,7 @@ use crate::{MemvidError, Result};
|
||||
impl Memvid {
|
||||
/// Returns the vector index dimension stored in the MV2 file, if available.
|
||||
/// This is useful for auto-detecting which embedding model was used to create the file.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn vec_index_dimension(&self) -> Option<u32> {
|
||||
self.toc
|
||||
.indexes
|
||||
|
||||
+34
-16
@@ -99,8 +99,7 @@ pub struct Memvid {
|
||||
}
|
||||
|
||||
/// Controls read-only open behaviour for `.mv2` memories.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
#[derive(Default)]
|
||||
#[derive(Debug, Clone, Copy, Default)]
|
||||
pub struct OpenReadOptions {
|
||||
pub allow_repair: bool,
|
||||
}
|
||||
@@ -126,7 +125,6 @@ impl Default for LockSettings {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
impl Memvid {
|
||||
/// Create a new, empty `.mv2` file with an embedded WAL and empty TOC.
|
||||
/// The file is locked exclusively for the lifetime of the handle.
|
||||
@@ -255,7 +253,7 @@ impl Memvid {
|
||||
Ok(memvid)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn lock_settings(&self) -> &LockSettings {
|
||||
&self.lock_settings
|
||||
}
|
||||
@@ -271,7 +269,7 @@ impl Memvid {
|
||||
}
|
||||
|
||||
/// Get the current vector compression mode
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn vector_compression(&self) -> &VectorCompression {
|
||||
&self.vec_compression
|
||||
}
|
||||
@@ -281,7 +279,7 @@ impl Memvid {
|
||||
/// Frame IDs are dense indices into `toc.frames`. When a memory is mutable, inserts are first
|
||||
/// appended to the embedded WAL and only materialized into `toc.frames` on commit. This helper
|
||||
/// lets frontends allocate stable frame IDs before an explicit commit.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn next_frame_id(&self) -> u64 {
|
||||
(self.toc.frames.len() as u64).saturating_add(self.pending_frame_inserts)
|
||||
}
|
||||
@@ -619,6 +617,13 @@ impl Memvid {
|
||||
};
|
||||
|
||||
// Read the compressed data from the file
|
||||
if manifest.bytes_length > crate::MAX_INDEX_BYTES {
|
||||
return Err(MemvidError::InvalidToc {
|
||||
reason: "memories track exceeds safety limit".into(),
|
||||
});
|
||||
}
|
||||
// Safe: guarded by MAX_INDEX_BYTES check above
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
let mut buf = vec![0u8; manifest.bytes_length as usize];
|
||||
self.file
|
||||
.seek(std::io::SeekFrom::Start(manifest.bytes_offset))?;
|
||||
@@ -646,6 +651,13 @@ impl Memvid {
|
||||
};
|
||||
|
||||
// Read the serialized data from the file
|
||||
if manifest.bytes_length > crate::MAX_INDEX_BYTES {
|
||||
return Err(MemvidError::InvalidToc {
|
||||
reason: "logic mesh exceeds safety limit".into(),
|
||||
});
|
||||
}
|
||||
// Safe: guarded by MAX_INDEX_BYTES check above
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
let mut buf = vec![0u8; manifest.bytes_length as usize];
|
||||
self.file
|
||||
.seek(std::io::SeekFrom::Start(manifest.bytes_offset))?;
|
||||
@@ -760,7 +772,7 @@ impl Memvid {
|
||||
///
|
||||
/// Returns the binding if this file is bound to a dashboard memory,
|
||||
/// or None if unbound.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn get_memory_binding(&self) -> Option<&crate::types::MemoryBinding> {
|
||||
self.toc.memory_binding.as_ref()
|
||||
}
|
||||
@@ -860,7 +872,14 @@ pub(crate) fn read_toc(file: &mut File, header: &Header) -> Result<Toc> {
|
||||
|
||||
// Read the entire region from footer_offset to EOF (includes TOC + footer)
|
||||
file.seek(SeekFrom::Start(header.footer_offset))?;
|
||||
// Safe: total_size bounded by file length, and we check MAX_INDEX_BYTES before reading
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
let total_size = (len - header.footer_offset) as usize;
|
||||
if total_size as u64 > crate::MAX_INDEX_BYTES {
|
||||
return Err(MemvidError::InvalidToc {
|
||||
reason: "toc region exceeds safety limit".into(),
|
||||
});
|
||||
}
|
||||
|
||||
if total_size < FOOTER_SIZE {
|
||||
return Err(MemvidError::InvalidToc {
|
||||
@@ -880,6 +899,7 @@ pub(crate) fn read_toc(file: &mut File, header: &Header) -> Result<Toc> {
|
||||
|
||||
// Extract only the TOC bytes (excluding the footer)
|
||||
let toc_bytes = &buf[..footer_start];
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
if toc_bytes.len() != footer.toc_len as usize {
|
||||
return Err(MemvidError::InvalidToc {
|
||||
reason: "toc length mismatch".into(),
|
||||
@@ -1024,6 +1044,8 @@ pub(crate) fn recover_toc(file: &mut File, hint: Option<u64>) -> Result<(Toc, u6
|
||||
if let Some(hint_offset) = hint {
|
||||
use crate::footer::FOOTER_SIZE;
|
||||
|
||||
// Safe: file successfully mmapped so length fits in usize
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
let start = (hint_offset.min(len)) as usize;
|
||||
if mmap.len().saturating_sub(start) >= FOOTER_SIZE {
|
||||
let toc_end = mmap.len().saturating_sub(FOOTER_SIZE);
|
||||
@@ -1047,6 +1069,8 @@ pub(crate) fn recover_toc(file: &mut File, hint: Option<u64>) -> Result<(Toc, u6
|
||||
// Fallback to manual scan if footer-based recovery failed
|
||||
let mut ranges = Vec::new();
|
||||
if let Some(hint_offset) = hint {
|
||||
// Safe: file successfully mmapped so length fits in usize
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
let hint_idx = hint_offset.min(len) as usize;
|
||||
ranges.push((hint_idx, mmap.len()));
|
||||
if hint_idx > 0 {
|
||||
@@ -1439,9 +1463,7 @@ fn validate_segment_integrity(toc: &Toc, header: &Header, file_len: u64) -> Resu
|
||||
let end = offset
|
||||
.checked_add(length)
|
||||
.ok_or_else(|| MemvidError::Doctor {
|
||||
reason: format!(
|
||||
"Tantivy segment {idx} offset overflow: {offset} + {length}"
|
||||
),
|
||||
reason: format!("Tantivy segment {idx} offset overflow: {offset} + {length}"),
|
||||
})?;
|
||||
|
||||
if end > file_len || end > data_limit {
|
||||
@@ -1465,9 +1487,7 @@ fn validate_segment_integrity(toc: &Toc, header: &Header, file_len: u64) -> Resu
|
||||
let end = offset
|
||||
.checked_add(length)
|
||||
.ok_or_else(|| MemvidError::Doctor {
|
||||
reason: format!(
|
||||
"Time segment {idx} offset overflow: {offset} + {length}"
|
||||
),
|
||||
reason: format!("Time segment {idx} offset overflow: {offset} + {length}"),
|
||||
})?;
|
||||
|
||||
if end > file_len || end > data_limit {
|
||||
@@ -1491,9 +1511,7 @@ fn validate_segment_integrity(toc: &Toc, header: &Header, file_len: u64) -> Resu
|
||||
let end = offset
|
||||
.checked_add(length)
|
||||
.ok_or_else(|| MemvidError::Doctor {
|
||||
reason: format!(
|
||||
"Vec segment {idx} offset overflow: {offset} + {length}"
|
||||
),
|
||||
reason: format!("Vec segment {idx} offset overflow: {offset} + {length}"),
|
||||
})?;
|
||||
|
||||
if end > file_len || end > data_limit {
|
||||
|
||||
@@ -413,7 +413,7 @@ impl Memvid {
|
||||
///
|
||||
/// # Returns
|
||||
/// A vector of (index, error) tuples for invalid cards.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn validate_cards(&self, cards: &[MemoryCard]) -> Vec<(usize, SchemaError)> {
|
||||
cards
|
||||
.iter()
|
||||
@@ -540,8 +540,9 @@ impl Memvid {
|
||||
.into_iter()
|
||||
.map(|schema| {
|
||||
let stats = predicate_stats.get(&schema.id);
|
||||
let (entity_count, value_count, unique_values) = stats
|
||||
.map_or((0, 0, 0), |s| (s.entities.len(), s.value_count, s.unique_values.len()));
|
||||
let (entity_count, value_count, unique_values) = stats.map_or((0, 0, 0), |s| {
|
||||
(s.entities.len(), s.value_count, s.unique_values.len())
|
||||
});
|
||||
|
||||
// Check if there's an existing (builtin) schema
|
||||
let is_builtin = self
|
||||
@@ -590,7 +591,11 @@ impl Memvid {
|
||||
|
||||
for frame_id in unenriched {
|
||||
// Get frame data
|
||||
let Some(frame) = self.toc.frames.get(frame_id as usize) else {
|
||||
// Safe frame lookup
|
||||
let Ok(index) = usize::try_from(frame_id) else {
|
||||
continue;
|
||||
};
|
||||
let Some(frame) = self.toc.frames.get(index) else {
|
||||
continue;
|
||||
};
|
||||
let frame = frame.clone();
|
||||
|
||||
+9
-9
@@ -95,7 +95,7 @@ impl Memvid {
|
||||
///
|
||||
/// # Returns
|
||||
/// A list of entities found by traversing the relationships.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn follow(&self, start: &str, link: &str, hops: usize) -> Vec<FollowResult> {
|
||||
self.logic_mesh.follow(start, link, hops)
|
||||
}
|
||||
@@ -107,7 +107,7 @@ impl Memvid {
|
||||
///
|
||||
/// # Returns
|
||||
/// The matching node if found.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn find_entity(&self, name: &str) -> Option<&MeshNode> {
|
||||
self.logic_mesh.find_node(name)
|
||||
}
|
||||
@@ -119,7 +119,7 @@ impl Memvid {
|
||||
///
|
||||
/// # Returns
|
||||
/// A list of entity nodes that have mentions in the specified frame.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn frame_entities(&self, frame_id: FrameId) -> Vec<&MeshNode> {
|
||||
self.logic_mesh
|
||||
.nodes
|
||||
@@ -135,7 +135,7 @@ impl Memvid {
|
||||
///
|
||||
/// # Returns
|
||||
/// A list of entity nodes matching the specified kind.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn entities_by_kind(&self, kind: EntityKind) -> Vec<&MeshNode> {
|
||||
self.logic_mesh
|
||||
.nodes
|
||||
@@ -148,7 +148,7 @@ impl Memvid {
|
||||
///
|
||||
/// # Returns
|
||||
/// Statistics including node count, edge count, and breakdowns by kind/link type.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn logic_mesh_stats(&self) -> LogicMeshStats {
|
||||
self.logic_mesh.stats()
|
||||
}
|
||||
@@ -157,19 +157,19 @@ impl Memvid {
|
||||
///
|
||||
/// # Returns
|
||||
/// `true` if the mesh has nodes or edges.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn has_logic_mesh(&self) -> bool {
|
||||
!self.logic_mesh.is_empty()
|
||||
}
|
||||
|
||||
/// Get the number of entity nodes in the mesh.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn mesh_node_count(&self) -> usize {
|
||||
self.logic_mesh.nodes.len()
|
||||
}
|
||||
|
||||
/// Get the number of relationship edges in the mesh.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn mesh_edge_count(&self) -> usize {
|
||||
self.logic_mesh.edges.len()
|
||||
}
|
||||
@@ -177,7 +177,7 @@ impl Memvid {
|
||||
/// Get entities for a frame as `SearchHitEntity` for search metadata.
|
||||
///
|
||||
/// Returns entities from the Logic-Mesh that appear in the given frame.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn frame_entities_for_search(&self, frame_id: FrameId) -> Vec<SearchHitEntity> {
|
||||
self.logic_mesh
|
||||
.nodes
|
||||
|
||||
+84
-55
@@ -381,7 +381,7 @@ fn finalize_reader_output(output: ReaderOutput, start: Instant) -> ExtractedDocu
|
||||
fn log_reader_result(reader: &str, diagnostics: &ReaderDiagnostics, elapsed: Duration) {
|
||||
let duration_ms = diagnostics
|
||||
.duration_ms
|
||||
.unwrap_or(elapsed.as_millis() as u64);
|
||||
.unwrap_or(elapsed.as_millis().try_into().unwrap_or(u64::MAX));
|
||||
let warnings = diagnostics.warnings.len();
|
||||
let pages = diagnostics.pages_processed;
|
||||
|
||||
@@ -671,9 +671,11 @@ impl Memvid {
|
||||
let chunk = min(remaining, buffer.len() as u64);
|
||||
let src = data_start + remaining - chunk;
|
||||
self.file.seek(SeekFrom::Start(src))?;
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
self.file.read_exact(&mut buffer[..chunk as usize])?;
|
||||
let dst = src + delta;
|
||||
self.file.seek(SeekFrom::Start(dst))?;
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
self.file.write_all(&buffer[..chunk as usize])?;
|
||||
remaining -= chunk;
|
||||
}
|
||||
@@ -683,6 +685,7 @@ impl Memvid {
|
||||
let mut remaining = delta;
|
||||
while remaining > 0 {
|
||||
let write = min(remaining, zero_buf.len() as u64);
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
self.file.write_all(&zero_buf[..write as usize])?;
|
||||
remaining -= write;
|
||||
}
|
||||
@@ -1127,7 +1130,13 @@ impl Memvid {
|
||||
reason: "reused payload entry contained inline bytes",
|
||||
});
|
||||
}
|
||||
let source = self.toc.frames.get(source_id as usize).cloned().ok_or(
|
||||
let source_idx = usize::try_from(source_id).map_err(|_| {
|
||||
MemvidError::InvalidFrame {
|
||||
frame_id: source_id,
|
||||
reason: "frame id too large for memory",
|
||||
}
|
||||
})?;
|
||||
let source = self.toc.frames.get(source_idx).cloned().ok_or(
|
||||
MemvidError::InvalidFrame {
|
||||
frame_id: source_id,
|
||||
reason: "reused payload source missing",
|
||||
@@ -1248,21 +1257,21 @@ impl Memvid {
|
||||
if entry.role == FrameRole::DocumentChunk {
|
||||
// Look backwards through recently inserted frames
|
||||
for &candidate_id in delta.inserted_frames.iter().rev() {
|
||||
if let Some(candidate) =
|
||||
self.toc.frames.get(candidate_id as usize)
|
||||
{
|
||||
if candidate.role == FrameRole::Document
|
||||
&& candidate.chunk_manifest.is_some()
|
||||
{
|
||||
// Found a parent document - use it
|
||||
frame.parent_id = Some(candidate_id);
|
||||
tracing::debug!(
|
||||
chunk_frame_id = frame_id,
|
||||
parent_frame_id = candidate_id,
|
||||
parent_seq = parent_seq,
|
||||
"resolved chunk parent via fallback"
|
||||
);
|
||||
break;
|
||||
if let Ok(idx) = usize::try_from(candidate_id) {
|
||||
if let Some(candidate) = self.toc.frames.get(idx) {
|
||||
if candidate.role == FrameRole::Document
|
||||
&& candidate.chunk_manifest.is_some()
|
||||
{
|
||||
// Found a parent document - use it
|
||||
frame.parent_id = Some(candidate_id);
|
||||
tracing::debug!(
|
||||
chunk_frame_id = frame_id,
|
||||
parent_frame_id = candidate_id,
|
||||
parent_seq = parent_seq,
|
||||
"resolved chunk parent via fallback"
|
||||
);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1366,18 +1375,21 @@ impl Memvid {
|
||||
.inserted_frames
|
||||
.iter()
|
||||
.filter_map(|&frame_id| {
|
||||
let frame = self.toc.frames.get(frame_id as usize)?;
|
||||
let idx = usize::try_from(frame_id).ok()?;
|
||||
let frame = self.toc.frames.get(idx)?;
|
||||
if frame.role != FrameRole::DocumentChunk || frame.parent_id.is_some() {
|
||||
return None;
|
||||
}
|
||||
// Find the most recent Document frame before this chunk that has a manifest
|
||||
for candidate_id in (0..frame_id).rev() {
|
||||
if let Some(candidate) = self.toc.frames.get(candidate_id as usize) {
|
||||
if candidate.role == FrameRole::Document
|
||||
&& candidate.chunk_manifest.is_some()
|
||||
&& candidate.status == FrameStatus::Active
|
||||
{
|
||||
return Some((frame_id, candidate_id));
|
||||
if let Ok(idx) = usize::try_from(candidate_id) {
|
||||
if let Some(candidate) = self.toc.frames.get(idx) {
|
||||
if candidate.role == FrameRole::Document
|
||||
&& candidate.chunk_manifest.is_some()
|
||||
&& candidate.status == FrameStatus::Active
|
||||
{
|
||||
return Some((frame_id, candidate_id));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1387,13 +1399,15 @@ impl Memvid {
|
||||
|
||||
// Now apply the resolutions
|
||||
for (chunk_id, parent_id) in orphan_resolutions {
|
||||
if let Some(frame) = self.toc.frames.get_mut(chunk_id as usize) {
|
||||
frame.parent_id = Some(parent_id);
|
||||
tracing::debug!(
|
||||
chunk_frame_id = chunk_id,
|
||||
parent_frame_id = parent_id,
|
||||
"resolved orphan chunk parent in second pass"
|
||||
);
|
||||
if let Ok(idx) = usize::try_from(chunk_id) {
|
||||
if let Some(frame) = self.toc.frames.get_mut(idx) {
|
||||
frame.parent_id = Some(parent_id);
|
||||
tracing::debug!(
|
||||
chunk_frame_id = chunk_id,
|
||||
parent_frame_id = parent_id,
|
||||
"resolved orphan chunk parent in second pass"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1884,14 +1898,18 @@ impl Memvid {
|
||||
}
|
||||
|
||||
fn mark_frame_superseded(&mut self, frame_id: FrameId, successor_id: FrameId) -> Result<()> {
|
||||
let frame =
|
||||
self.toc
|
||||
.frames
|
||||
.get_mut(frame_id as usize)
|
||||
.ok_or(MemvidError::InvalidFrame {
|
||||
frame_id,
|
||||
reason: "supersede target missing",
|
||||
})?;
|
||||
let index = usize::try_from(frame_id).map_err(|_| MemvidError::InvalidFrame {
|
||||
frame_id,
|
||||
reason: "frame id too large",
|
||||
})?;
|
||||
let frame = self
|
||||
.toc
|
||||
.frames
|
||||
.get_mut(index)
|
||||
.ok_or(MemvidError::InvalidFrame {
|
||||
frame_id,
|
||||
reason: "supersede target missing",
|
||||
})?;
|
||||
frame.status = FrameStatus::Superseded;
|
||||
frame.superseded_by = Some(successor_id);
|
||||
self.remove_frame_from_indexes(frame_id)
|
||||
@@ -2311,6 +2329,7 @@ impl Memvid {
|
||||
bytes_offset: sketch_offset,
|
||||
bytes_length: sketch_length,
|
||||
entry_count: stats.entry_count,
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
entry_size: stats.variant.entry_size() as u16,
|
||||
flags: 0,
|
||||
checksum: sketch_checksum,
|
||||
@@ -2471,14 +2490,18 @@ impl Memvid {
|
||||
}
|
||||
|
||||
fn mark_frame_deleted(&mut self, frame_id: FrameId) -> Result<()> {
|
||||
let frame =
|
||||
self.toc
|
||||
.frames
|
||||
.get_mut(frame_id as usize)
|
||||
.ok_or(MemvidError::InvalidFrame {
|
||||
frame_id,
|
||||
reason: "delete target missing",
|
||||
})?;
|
||||
let index = usize::try_from(frame_id).map_err(|_| MemvidError::InvalidFrame {
|
||||
frame_id,
|
||||
reason: "frame id too large",
|
||||
})?;
|
||||
let frame = self
|
||||
.toc
|
||||
.frames
|
||||
.get_mut(index)
|
||||
.ok_or(MemvidError::InvalidFrame {
|
||||
frame_id,
|
||||
reason: "delete target missing",
|
||||
})?;
|
||||
frame.status = FrameStatus::Deleted;
|
||||
frame.superseded_by = None;
|
||||
self.remove_frame_from_indexes(frame_id)
|
||||
@@ -2500,9 +2523,12 @@ impl Memvid {
|
||||
}
|
||||
|
||||
pub(crate) fn frame_is_active(&self, frame_id: FrameId) -> bool {
|
||||
let Ok(index) = usize::try_from(frame_id) else {
|
||||
return false;
|
||||
};
|
||||
self.toc
|
||||
.frames
|
||||
.get(frame_id as usize)
|
||||
.get(index)
|
||||
.is_some_and(|frame| frame.status == FrameStatus::Active)
|
||||
}
|
||||
|
||||
@@ -3087,7 +3113,9 @@ impl Memvid {
|
||||
|
||||
if let Some(ref vector) = embedding {
|
||||
if !vector.is_empty() {
|
||||
dim = Some(vector.len() as u32);
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
let len = vector.len() as u32;
|
||||
dim = Some(len);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3096,7 +3124,7 @@ impl Memvid {
|
||||
if vector.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let vec_dim = vector.len() as u32;
|
||||
let vec_dim = u32::try_from(vector.len()).unwrap_or(0);
|
||||
match dim {
|
||||
None => dim = Some(vec_dim),
|
||||
Some(existing) if existing == vec_dim => {}
|
||||
@@ -3168,6 +3196,7 @@ impl Memvid {
|
||||
.unwrap_or(0)
|
||||
});
|
||||
|
||||
#[allow(unused_assignments)]
|
||||
let mut reuse_bytes: Option<Vec<u8>> = None;
|
||||
let payload_for_processing = if let Some(bytes) = payload {
|
||||
Some(bytes)
|
||||
@@ -3412,7 +3441,7 @@ impl Memvid {
|
||||
let triplet_title = title_value.clone();
|
||||
|
||||
if let Some(plan) = chunk_plan.as_ref() {
|
||||
let chunk_total = plan.chunks.len() as u32;
|
||||
let chunk_total = u32::try_from(plan.chunks.len()).unwrap_or(0);
|
||||
parent_chunk_manifest = Some(plan.manifest.clone());
|
||||
parent_chunk_count = Some(chunk_total);
|
||||
|
||||
@@ -3470,7 +3499,7 @@ impl Memvid {
|
||||
chunk_manifest: None,
|
||||
role: FrameRole::DocumentChunk,
|
||||
parent_sequence: None,
|
||||
chunk_index: Some(idx as u32),
|
||||
chunk_index: Some(u32::try_from(idx).unwrap_or(0)),
|
||||
chunk_count: Some(chunk_total),
|
||||
op: FrameWalOp::Insert,
|
||||
target_frame_id: None,
|
||||
@@ -3494,9 +3523,9 @@ impl Memvid {
|
||||
// Since frame.id corresponds to the array index, we need to find the sequence
|
||||
// For now, we'll use the frame_id + WAL_START_SEQUENCE as an approximation
|
||||
// This works because sequence numbers are assigned incrementally
|
||||
self.toc
|
||||
.frames
|
||||
.get(parent_id as usize)
|
||||
usize::try_from(parent_id)
|
||||
.ok()
|
||||
.and_then(|idx| self.toc.frames.get(idx))
|
||||
.map(|_| parent_id + 2) // WAL sequences start at 2
|
||||
} else {
|
||||
None
|
||||
|
||||
+38
-11
@@ -99,15 +99,24 @@ impl Memvid {
|
||||
return Err(MemvidError::VecNotEnabled);
|
||||
}
|
||||
let mut ensured_vec_index = false;
|
||||
let expected_dim = if let Some(dim) = self.effective_vec_index_dimension()? { dim } else {
|
||||
let expected_dim = if let Some(dim) = self.effective_vec_index_dimension()? {
|
||||
dim
|
||||
} else {
|
||||
self.ensure_vec_index()?;
|
||||
ensured_vec_index = true;
|
||||
self.vec_index
|
||||
.as_ref()
|
||||
.and_then(|index| index.entries().next().map(|(_, emb)| emb.len() as u32))
|
||||
.and_then(|index| {
|
||||
index
|
||||
.entries()
|
||||
.next()
|
||||
.map(|(_, emb)| u32::try_from(emb.len()).unwrap_or(0))
|
||||
})
|
||||
.unwrap_or(0)
|
||||
};
|
||||
if expected_dim > 0 && query.len() as u32 != expected_dim {
|
||||
// Safe: embedding dimensions are small (< few thousands)
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
if expected_dim > 0 && (query.len() as u32) != expected_dim {
|
||||
return Err(MemvidError::VecDimensionMismatch {
|
||||
expected: expected_dim,
|
||||
actual: query.len(),
|
||||
@@ -246,15 +255,24 @@ impl Memvid {
|
||||
// Validate embedding dimension BEFORE searching to prevent silent wrong results.
|
||||
// For segment-only memories, dimension may only be discoverable after loading segments.
|
||||
let mut ensured_vec_index = false;
|
||||
let expected_dim = if let Some(dim) = self.effective_vec_index_dimension()? { dim } else {
|
||||
let expected_dim = if let Some(dim) = self.effective_vec_index_dimension()? {
|
||||
dim
|
||||
} else {
|
||||
self.ensure_vec_index()?;
|
||||
ensured_vec_index = true;
|
||||
self.vec_index
|
||||
.as_ref()
|
||||
.and_then(|index| index.entries().next().map(|(_, emb)| emb.len() as u32))
|
||||
.and_then(|index| {
|
||||
index
|
||||
.entries()
|
||||
.next()
|
||||
.map(|(_, emb)| u32::try_from(emb.len()).unwrap_or(0))
|
||||
})
|
||||
.unwrap_or(0)
|
||||
};
|
||||
if expected_dim > 0 && query_embedding.len() as u32 != expected_dim {
|
||||
// Safe: embedding dimensions are small
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
if expected_dim > 0 && (query_embedding.len() as u32) != expected_dim {
|
||||
return Err(MemvidError::VecDimensionMismatch {
|
||||
expected: expected_dim,
|
||||
actual: query_embedding.len(),
|
||||
@@ -297,7 +315,14 @@ impl Memvid {
|
||||
|
||||
for vec_hit in vec_hits {
|
||||
// Apply scope filter if provided
|
||||
let frame = match self.toc.frames.get(vec_hit.frame_id as usize) {
|
||||
// Apply scope filter if provided
|
||||
let frame_idx = if let Ok(idx) = usize::try_from(vec_hit.frame_id) {
|
||||
idx
|
||||
} else {
|
||||
continue;
|
||||
};
|
||||
|
||||
let frame = match self.toc.frames.get(frame_idx) {
|
||||
Some(f) => f.clone(),
|
||||
None => continue,
|
||||
};
|
||||
@@ -690,6 +715,8 @@ impl Memvid {
|
||||
self.file.seek(SeekFrom::Start(segment.bytes_offset))?;
|
||||
let mut remaining = segment.bytes_length;
|
||||
while remaining > 0 {
|
||||
// Safe: chunk is at most buffer.len() which is usize
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
let chunk = remaining.min(buffer.len() as u64) as usize;
|
||||
if let Err(err) = self.file.read_exact(&mut buffer[..chunk]) {
|
||||
return Err(MemvidError::Tantivy {
|
||||
@@ -800,7 +827,7 @@ impl Memvid {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn vec_segment_descriptor(&self, segment_id: u64) -> Option<VecSegmentDescriptor> {
|
||||
self.toc
|
||||
.segment_catalog
|
||||
@@ -830,7 +857,7 @@ impl Memvid {
|
||||
pub const DEFAULT_MAX_INDEX_PAYLOAD: u64 = 256 * 1024 * 1024;
|
||||
|
||||
/// Get the maximum indexable payload size from environment or use default
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn max_index_payload() -> u64 {
|
||||
std::env::var("MEMVID_MAX_INDEX_PAYLOAD")
|
||||
.ok()
|
||||
@@ -839,7 +866,7 @@ pub fn max_index_payload() -> u64 {
|
||||
}
|
||||
|
||||
/// Check if a MIME type represents text-based content that should be indexed
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn is_text_indexable_mime(mime: &str) -> bool {
|
||||
let mime_lower = mime.to_lowercase();
|
||||
|
||||
@@ -891,7 +918,7 @@ pub fn is_text_indexable_mime(mime: &str) -> bool {
|
||||
}
|
||||
|
||||
/// Check if a frame should be indexed for text search
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn is_frame_text_indexable(frame: &crate::types::Frame) -> bool {
|
||||
// Must be active
|
||||
if frame.status != crate::types::FrameStatus::Active {
|
||||
|
||||
@@ -96,11 +96,14 @@ impl Memvid {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let bytes = if let Ok(bytes) = self.read_range(manifest.bytes_offset, manifest.bytes_length) { bytes } else {
|
||||
// Don't disable lex if loading fails - keep it enabled
|
||||
self.lex_index = None;
|
||||
return Ok(());
|
||||
};
|
||||
let bytes =
|
||||
if let Ok(bytes) = self.read_range(manifest.bytes_offset, manifest.bytes_length) {
|
||||
bytes
|
||||
} else {
|
||||
// Don't disable lex if loading fails - keep it enabled
|
||||
self.lex_index = None;
|
||||
return Ok(());
|
||||
};
|
||||
match LexIndex::decode(&bytes) {
|
||||
Ok(mut index) => {
|
||||
self.hydrate_lex_index_metadata(&mut index);
|
||||
@@ -127,12 +130,15 @@ impl Memvid {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let bytes = if let Ok(bytes) = self.read_range(manifest.bytes_offset, manifest.bytes_length) { bytes } else {
|
||||
self.vec_index = None;
|
||||
// Don't disable vec if loading fails - keep it enabled
|
||||
// self.vec_enabled = false;
|
||||
return Ok(());
|
||||
};
|
||||
let bytes =
|
||||
if let Ok(bytes) = self.read_range(manifest.bytes_offset, manifest.bytes_length) {
|
||||
bytes
|
||||
} else {
|
||||
self.vec_index = None;
|
||||
// Don't disable vec if loading fails - keep it enabled
|
||||
// self.vec_enabled = false;
|
||||
return Ok(());
|
||||
};
|
||||
match catch_unwind(AssertUnwindSafe(|| VecIndex::decode(&bytes))) {
|
||||
Ok(Ok(index)) => self.vec_index = Some(index),
|
||||
Ok(Err(_)) | Err(_) => {
|
||||
@@ -160,10 +166,13 @@ impl Memvid {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let bytes = if let Ok(bytes) = self.read_range(manifest.bytes_offset, manifest.bytes_length) { bytes } else {
|
||||
self.clip_index = None;
|
||||
return Ok(());
|
||||
};
|
||||
let bytes =
|
||||
if let Ok(bytes) = self.read_range(manifest.bytes_offset, manifest.bytes_length) {
|
||||
bytes
|
||||
} else {
|
||||
self.clip_index = None;
|
||||
return Ok(());
|
||||
};
|
||||
match catch_unwind(AssertUnwindSafe(|| ClipIndex::decode(&bytes))) {
|
||||
Ok(Ok(index)) => self.clip_index = Some(index),
|
||||
Ok(Err(_)) | Err(_) => {
|
||||
@@ -187,6 +196,8 @@ impl Memvid {
|
||||
});
|
||||
}
|
||||
self.file.seek(SeekFrom::Start(offset))?;
|
||||
// Safe: length is checked against MAX_INDEX_BYTES above
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
let mut buf = vec![0u8; length as usize];
|
||||
self.file.read_exact(&mut buf)?;
|
||||
Ok(buf)
|
||||
@@ -264,7 +275,8 @@ impl Memvid {
|
||||
|
||||
fn hydrate_lex_index_metadata(&self, index: &mut LexIndex) {
|
||||
for document in index.documents_mut() {
|
||||
let frame_meta = self.toc.frames.get(document.frame_id as usize);
|
||||
let frame_idx = usize::try_from(document.frame_id).ok();
|
||||
let frame_meta = frame_idx.and_then(|idx| self.toc.frames.get(idx));
|
||||
|
||||
if document.uri.is_none() {
|
||||
let derived = frame_meta
|
||||
|
||||
@@ -45,11 +45,12 @@ pub(super) fn search_with_lex_fallback(
|
||||
continue;
|
||||
}
|
||||
}
|
||||
let frame_meta = memvid.toc.frames.get(matched.frame_id as usize).ok_or(
|
||||
MemvidError::InvalidTimeIndex {
|
||||
let frame_meta = usize::try_from(matched.frame_id)
|
||||
.ok()
|
||||
.and_then(|idx| memvid.toc.frames.get(idx))
|
||||
.ok_or(MemvidError::InvalidTimeIndex {
|
||||
reason: "frame id out of range".into(),
|
||||
},
|
||||
)?;
|
||||
})?;
|
||||
let content_lower = matched.content.to_ascii_lowercase();
|
||||
let ctx = EvaluationContext {
|
||||
frame: frame_meta,
|
||||
@@ -88,14 +89,21 @@ pub(super) fn search_with_lex_fallback(
|
||||
let frame_meta = memvid
|
||||
.toc
|
||||
.frames
|
||||
.get(matched.frame_id as usize)
|
||||
.get(usize::try_from(matched.frame_id).unwrap_or(usize::MAX))
|
||||
.cloned()
|
||||
.ok_or(MemvidError::InvalidTimeIndex {
|
||||
reason: "frame id out of range".into(),
|
||||
})?;
|
||||
let canonical = memvid.frame_content(&frame_meta)?;
|
||||
let canonical_limit = frame_meta
|
||||
.canonical_length.map_or_else(|| canonical.len(), |len| len as usize);
|
||||
let canonical_limit = frame_meta.canonical_length.map_or_else(
|
||||
|| canonical.len(),
|
||||
|len| {
|
||||
// Safe: canonical length is reasonably small string length
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
let l = len as usize;
|
||||
l
|
||||
},
|
||||
);
|
||||
let canonical_len = canonical.len();
|
||||
let effective_len = canonical_limit.min(canonical_len);
|
||||
let uri = matched
|
||||
|
||||
@@ -406,7 +406,10 @@ pub(super) fn enrich_hits_with_entities(hits: &mut [SearchHit], memvid: &Memvid)
|
||||
|
||||
// If no entities found and this is a chunk, check the parent frame
|
||||
if entities.is_empty() {
|
||||
if let Some(frame) = memvid.toc.frames.get(hit.frame_id as usize) {
|
||||
if let Some(frame) = usize::try_from(hit.frame_id)
|
||||
.ok()
|
||||
.and_then(|idx| memvid.toc.frames.get(idx))
|
||||
{
|
||||
if let Some(parent_id) = frame.parent_id {
|
||||
entities = memvid.frame_entities_for_search(parent_id);
|
||||
}
|
||||
|
||||
@@ -123,7 +123,7 @@ pub(super) fn try_tantivy_search(
|
||||
let frame_meta = memvid
|
||||
.toc
|
||||
.frames
|
||||
.get(hit.frame_id as usize)
|
||||
.get(usize::try_from(hit.frame_id).unwrap_or(usize::MAX))
|
||||
.cloned()
|
||||
.ok_or(MemvidError::InvalidTimeIndex {
|
||||
reason: "frame id out of range".into(),
|
||||
@@ -201,6 +201,7 @@ pub(super) fn try_tantivy_search(
|
||||
.map(|(hit, occurrences, slices, chunk_info, timestamp)| {
|
||||
let bm25_score = hit.score;
|
||||
// Age relative to the most recent document in results
|
||||
#[allow(clippy::cast_precision_loss)]
|
||||
let age_seconds = (max_ts - timestamp).max(0) as f32;
|
||||
// Decay factor: half-life of ~1 day for aggressive recency preference
|
||||
// This ensures even a few days difference has significant impact
|
||||
@@ -268,7 +269,7 @@ pub(super) fn try_tantivy_search(
|
||||
let frame_meta = memvid
|
||||
.toc
|
||||
.frames
|
||||
.get(hit.frame_id as usize)
|
||||
.get(usize::try_from(hit.frame_id).unwrap_or(usize::MAX))
|
||||
.cloned()
|
||||
.ok_or(MemvidError::InvalidTimeIndex {
|
||||
reason: "frame id out of range".into(),
|
||||
@@ -383,7 +384,7 @@ fn uri_matches(candidate: Option<&str>, expected: &str) -> bool {
|
||||
/// Parse content dates (from frame metadata) to find the most relevant timestamp.
|
||||
/// Content dates are strings like "2023/06/30 (Fri) 14:20", ISO dates, or spelled-out dates.
|
||||
/// Returns the most recent timestamp found, or None if parsing fails.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn parse_content_date_to_timestamp(content_dates: &[String]) -> Option<i64> {
|
||||
if content_dates.is_empty() {
|
||||
return None;
|
||||
|
||||
+11
-8
@@ -103,12 +103,15 @@ impl Memvid {
|
||||
let mut builder = LexIndexBuilder::new();
|
||||
let empty_tags = std::collections::HashMap::new();
|
||||
for frame_id in frame_ids {
|
||||
let frame = self.toc.frames.get(*frame_id as usize).cloned().ok_or(
|
||||
MemvidError::InvalidFrame {
|
||||
let frame = self
|
||||
.toc
|
||||
.frames
|
||||
.get(usize::try_from(*frame_id).unwrap_or(0))
|
||||
.cloned()
|
||||
.ok_or(MemvidError::InvalidFrame {
|
||||
frame_id: *frame_id,
|
||||
reason: "frame id out of range for lex segment",
|
||||
},
|
||||
)?;
|
||||
})?;
|
||||
|
||||
if frame.status != FrameStatus::Active {
|
||||
continue;
|
||||
@@ -174,7 +177,7 @@ impl Memvid {
|
||||
continue;
|
||||
}
|
||||
non_empty_count = non_empty_count.saturating_add(1);
|
||||
let vec_dim = vector.len() as u32;
|
||||
let vec_dim = u32::try_from(vector.len()).unwrap_or(0);
|
||||
match dimension {
|
||||
None => dimension = Some(vec_dim),
|
||||
Some(existing) if existing == vec_dim => {}
|
||||
@@ -568,9 +571,9 @@ impl Memvid {
|
||||
let existing = latest
|
||||
.get(path.as_str())
|
||||
.map(|descriptor| (*descriptor).clone());
|
||||
let requires_append = existing.as_ref().is_none_or(|descriptor| {
|
||||
descriptor.common.checksum != blob.checksum
|
||||
});
|
||||
let requires_append = existing
|
||||
.as_ref()
|
||||
.is_none_or(|descriptor| descriptor.common.checksum != blob.checksum);
|
||||
|
||||
let artifact = if requires_append {
|
||||
Some(TantivySegmentArtifact {
|
||||
|
||||
+10
-10
@@ -189,10 +189,10 @@ impl Memvid {
|
||||
.filter(|(_, score)| *score >= opts.min_score)
|
||||
.map(|(frame_id, score)| {
|
||||
let entry = self.sketch_track.get(frame_id);
|
||||
let hamming_distance = entry
|
||||
.map_or(64, |e| e.hamming_distance(query_sketch.simhash));
|
||||
let matching_top_terms = entry
|
||||
.map_or(0, |e| e.count_matching_top_terms(&query_sketch.top_terms));
|
||||
let hamming_distance =
|
||||
entry.map_or(64, |e| e.hamming_distance(query_sketch.simhash));
|
||||
let matching_top_terms =
|
||||
entry.map_or(0, |e| e.count_matching_top_terms(&query_sketch.top_terms));
|
||||
|
||||
SketchCandidate {
|
||||
frame_id,
|
||||
@@ -255,10 +255,10 @@ impl Memvid {
|
||||
.into_iter()
|
||||
.map(|(frame_id, score)| {
|
||||
let entry = self.sketch_track.get(frame_id);
|
||||
let hamming_distance = entry
|
||||
.map_or(64, |e| e.hamming_distance(query_sketch.simhash));
|
||||
let matching_top_terms = entry
|
||||
.map_or(0, |e| e.count_matching_top_terms(&query_sketch.top_terms));
|
||||
let hamming_distance =
|
||||
entry.map_or(64, |e| e.hamming_distance(query_sketch.simhash));
|
||||
let matching_top_terms =
|
||||
entry.map_or(0, |e| e.count_matching_top_terms(&query_sketch.top_terms));
|
||||
|
||||
SketchCandidate {
|
||||
frame_id,
|
||||
@@ -274,7 +274,7 @@ impl Memvid {
|
||||
term_filter_hits,
|
||||
simhash_hits,
|
||||
candidates_returned,
|
||||
scan_us: start.elapsed().as_micros() as u64,
|
||||
scan_us: u64::try_from(start.elapsed().as_micros()).unwrap_or(u64::MAX),
|
||||
};
|
||||
|
||||
(result, stats)
|
||||
@@ -303,7 +303,7 @@ mod tests {
|
||||
let candidates = mem.find_sketch_candidates("cats pets", None);
|
||||
|
||||
// Should find some candidates
|
||||
assert!(!candidates.is_empty() || mem.sketches().len() > 0);
|
||||
assert!(!candidates.is_empty() || !mem.sketches().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -86,12 +86,7 @@ impl Memvid {
|
||||
}
|
||||
|
||||
// CLIP image count from clip index manifest
|
||||
let clip_image_count = self
|
||||
.toc
|
||||
.indexes
|
||||
.clip
|
||||
.as_ref()
|
||||
.map_or(0, |c| c.vector_count);
|
||||
let clip_image_count = self.toc.indexes.clip.as_ref().map_or(0, |c| c.vector_count);
|
||||
|
||||
Ok(Stats {
|
||||
frame_count: self.toc.frames.len() as u64,
|
||||
@@ -247,13 +242,13 @@ impl Memvid {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn current_ticket(&self) -> TicketRef {
|
||||
self.toc.ticket_ref.clone()
|
||||
}
|
||||
|
||||
/// Returns a reference to the Logic-Mesh manifest, if present.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn logic_mesh_manifest(&self) -> Option<&crate::types::LogicMeshManifest> {
|
||||
self.toc.logic_mesh.as_ref()
|
||||
}
|
||||
|
||||
@@ -88,7 +88,9 @@ pub(crate) fn build_timeline(
|
||||
entries.reverse();
|
||||
}
|
||||
|
||||
let limit = limit.map_or(entries.len(), |nz| nz.get() as usize);
|
||||
let limit = limit.map_or(entries.len(), |nz| {
|
||||
usize::try_from(nz.get()).unwrap_or(usize::MAX)
|
||||
});
|
||||
let mut result = Vec::with_capacity(entries.len().min(limit));
|
||||
#[cfg(feature = "temporal_track")]
|
||||
let temporal_track_snapshot = memvid.temporal_track_ref()?.cloned();
|
||||
@@ -96,7 +98,7 @@ pub(crate) fn build_timeline(
|
||||
let frame = memvid
|
||||
.toc
|
||||
.frames
|
||||
.get(entry.frame_id as usize)
|
||||
.get(usize::try_from(entry.frame_id).unwrap_or(usize::MAX))
|
||||
.ok_or(MemvidError::InvalidTimeIndex {
|
||||
reason: "frame id out of range".into(),
|
||||
})?
|
||||
|
||||
+3
-5
@@ -62,13 +62,11 @@ impl ModelVerification {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
#[derive(Default)]
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct ModelVerifyOptions {
|
||||
pub run_onnx_smoke: bool,
|
||||
}
|
||||
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(default, rename_all = "kebab-case")]
|
||||
pub struct ModelManifest {
|
||||
@@ -106,7 +104,6 @@ pub struct ModelManifestEntry {
|
||||
pub kind: Option<String>,
|
||||
}
|
||||
|
||||
|
||||
pub fn verify_models(root: &Path, options: &ModelVerifyOptions) -> Result<Vec<ModelVerification>> {
|
||||
if !root.exists() {
|
||||
return Ok(Vec::new());
|
||||
@@ -317,7 +314,8 @@ fn normalize_sha256(value: &str, context: &str) -> Result<String> {
|
||||
|
||||
fn digest_from_dir_name(path: &Path) -> Option<String> {
|
||||
let name = path.file_name()?.to_str()?;
|
||||
name.strip_prefix("sha256-").map(std::string::ToString::to_string)
|
||||
name.strip_prefix("sha256-")
|
||||
.map(std::string::ToString::to_string)
|
||||
}
|
||||
|
||||
fn compute_sha256_hex(path: &Path) -> Result<String> {
|
||||
|
||||
+5
-7
@@ -44,13 +44,11 @@ impl DocumentReader for DocxReader {
|
||||
|
||||
fn supports(&self, hint: &ReaderHint<'_>) -> bool {
|
||||
matches!(hint.format, Some(DocumentFormat::Docx))
|
||||
|| hint
|
||||
.mime
|
||||
.is_some_and(|mime| {
|
||||
mime.eq_ignore_ascii_case(
|
||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||
)
|
||||
})
|
||||
|| hint.mime.is_some_and(|mime| {
|
||||
mime.eq_ignore_ascii_case(
|
||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn extract(&self, bytes: &[u8], hint: &ReaderHint<'_>) -> Result<ReaderOutput> {
|
||||
|
||||
+2
-2
@@ -33,7 +33,7 @@ pub enum DocumentFormat {
|
||||
}
|
||||
|
||||
impl DocumentFormat {
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn label(self) -> &'static str {
|
||||
match self {
|
||||
Self::Pdf => "pdf",
|
||||
@@ -126,7 +126,7 @@ impl ReaderDiagnostics {
|
||||
self.fallback = true;
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn with_metadata(mut self, value: Value) -> Self {
|
||||
self.extra_metadata = value;
|
||||
self
|
||||
|
||||
@@ -18,8 +18,12 @@ impl PassthroughReader {
|
||||
fn supported_format(format: Option<DocumentFormat>) -> bool {
|
||||
matches!(
|
||||
format,
|
||||
Some(DocumentFormat::Pdf | DocumentFormat::PlainText |
|
||||
DocumentFormat::Markdown | DocumentFormat::Html) | None
|
||||
Some(
|
||||
DocumentFormat::Pdf
|
||||
| DocumentFormat::PlainText
|
||||
| DocumentFormat::Markdown
|
||||
| DocumentFormat::Html
|
||||
) | None
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -31,11 +35,9 @@ impl DocumentReader for PassthroughReader {
|
||||
|
||||
fn supports(&self, hint: &ReaderHint<'_>) -> bool {
|
||||
Self::supported_format(hint.format)
|
||||
|| hint
|
||||
.mime
|
||||
.is_none_or(|mime| {
|
||||
mime.eq_ignore_ascii_case("application/pdf") || mime.starts_with("text/")
|
||||
})
|
||||
|| hint.mime.is_none_or(|mime| {
|
||||
mime.eq_ignore_ascii_case("application/pdf") || mime.starts_with("text/")
|
||||
})
|
||||
}
|
||||
|
||||
fn extract(&self, bytes: &[u8], _hint: &ReaderHint<'_>) -> Result<ReaderOutput> {
|
||||
|
||||
+1
-1
@@ -105,7 +105,7 @@ impl PdfReader {
|
||||
pages += 1;
|
||||
}
|
||||
|
||||
let duration_ms = start.elapsed().as_millis() as u64;
|
||||
let duration_ms = start.elapsed().as_millis().try_into().unwrap_or(u64::MAX);
|
||||
let trimmed = combined.trim();
|
||||
if trimmed.is_empty() {
|
||||
return Err(crate::MemvidError::ExtractionFailed {
|
||||
|
||||
+5
-7
@@ -60,13 +60,11 @@ impl DocumentReader for PptxReader {
|
||||
|
||||
fn supports(&self, hint: &ReaderHint<'_>) -> bool {
|
||||
matches!(hint.format, Some(DocumentFormat::Pptx))
|
||||
|| hint
|
||||
.mime
|
||||
.is_some_and(|mime| {
|
||||
mime.eq_ignore_ascii_case(
|
||||
"application/vnd.openxmlformats-officedocument.presentationml.presentation",
|
||||
)
|
||||
})
|
||||
|| hint.mime.is_some_and(|mime| {
|
||||
mime.eq_ignore_ascii_case(
|
||||
"application/vnd.openxmlformats-officedocument.presentationml.presentation",
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn extract(&self, bytes: &[u8], hint: &ReaderHint<'_>) -> Result<ReaderOutput> {
|
||||
|
||||
+5
-7
@@ -60,13 +60,11 @@ impl DocumentReader for XlsxReader {
|
||||
|
||||
fn supports(&self, hint: &ReaderHint<'_>) -> bool {
|
||||
matches!(hint.format, Some(DocumentFormat::Xlsx))
|
||||
|| hint
|
||||
.mime
|
||||
.is_some_and(|mime| {
|
||||
mime.eq_ignore_ascii_case(
|
||||
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
)
|
||||
})
|
||||
|| hint.mime.is_some_and(|mime| {
|
||||
mime.eq_ignore_ascii_case(
|
||||
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn extract(&self, bytes: &[u8], hint: &ReaderHint<'_>) -> Result<ReaderOutput> {
|
||||
|
||||
+3
-9
@@ -153,11 +153,7 @@ fn registry_root() -> Result<PathBuf> {
|
||||
}
|
||||
|
||||
Err(last_err
|
||||
.unwrap_or_else(|| {
|
||||
io::Error::other(
|
||||
"failed to establish memvid lock registry directory",
|
||||
)
|
||||
})
|
||||
.unwrap_or_else(|| io::Error::other("failed to establish memvid lock registry directory"))
|
||||
.into())
|
||||
}
|
||||
|
||||
@@ -216,8 +212,7 @@ pub fn write_record(record: &LockRecord) -> Result<()> {
|
||||
.create(true)
|
||||
.truncate(true)
|
||||
.open(path)?;
|
||||
serde_json::to_writer(&mut file, record)
|
||||
.map_err(io::Error::other)?;
|
||||
serde_json::to_writer(&mut file, record).map_err(io::Error::other)?;
|
||||
file.flush()?;
|
||||
file.sync_all()?;
|
||||
Ok(())
|
||||
@@ -240,8 +235,7 @@ pub fn read_record(file_id: &FileId) -> Result<Option<LockRecord>> {
|
||||
Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(None),
|
||||
Err(err) => return Err(err.into()),
|
||||
};
|
||||
let record: LockRecord =
|
||||
serde_json::from_reader(file).map_err(io::Error::other)?;
|
||||
let record: LockRecord = serde_json::from_reader(file).map_err(io::Error::other)?;
|
||||
Ok(Some(record))
|
||||
}
|
||||
|
||||
|
||||
+18
-12
@@ -50,13 +50,13 @@ pub struct ReplayResult {
|
||||
|
||||
impl ReplayResult {
|
||||
/// Check if the replay was successful (all actions matched).
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn is_success(&self) -> bool {
|
||||
self.mismatched_actions == 0
|
||||
}
|
||||
|
||||
/// Get the match rate as a percentage.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn match_rate(&self) -> f64 {
|
||||
if self.total_actions == 0 {
|
||||
100.0
|
||||
@@ -366,9 +366,8 @@ impl<'a> ReplayEngine<'a> {
|
||||
|
||||
// If model override is set, show it (CLI handles actual LLM re-execution)
|
||||
if let Some(ref override_model) = self.config.use_model {
|
||||
details.push_str(&format!(
|
||||
"\n Override Model: {override_model}"
|
||||
));
|
||||
details
|
||||
.push_str(&format!("\n Override Model: {override_model}"));
|
||||
}
|
||||
|
||||
// Show original answer
|
||||
@@ -377,9 +376,8 @@ impl<'a> ReplayEngine<'a> {
|
||||
} else {
|
||||
original_answer.clone()
|
||||
};
|
||||
details.push_str(&format!(
|
||||
"\n Original Answer: \"{answer_preview}\""
|
||||
));
|
||||
details
|
||||
.push_str(&format!("\n Original Answer: \"{answer_preview}\""));
|
||||
|
||||
// In audit mode with frozen frames, we consider it verified
|
||||
if action.affected_frames.is_empty() {
|
||||
@@ -503,7 +501,11 @@ impl<'a> ReplayEngine<'a> {
|
||||
}
|
||||
}
|
||||
|
||||
action_result.duration_ms = action_start.elapsed().as_millis() as u64;
|
||||
action_result.duration_ms = action_start
|
||||
.elapsed()
|
||||
.as_millis()
|
||||
.try_into()
|
||||
.unwrap_or(u64::MAX);
|
||||
result.action_results.push(action_result);
|
||||
|
||||
// Stop on mismatch if configured
|
||||
@@ -515,7 +517,11 @@ impl<'a> ReplayEngine<'a> {
|
||||
}
|
||||
}
|
||||
|
||||
result.total_duration_ms = start_time.elapsed().as_millis() as u64;
|
||||
result.total_duration_ms = start_time
|
||||
.elapsed()
|
||||
.as_millis()
|
||||
.try_into()
|
||||
.unwrap_or(u64::MAX);
|
||||
|
||||
if self.config.verbose {
|
||||
tracing::info!(
|
||||
@@ -530,7 +536,7 @@ impl<'a> ReplayEngine<'a> {
|
||||
}
|
||||
|
||||
/// Compare two sessions to find differences.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn compare_sessions(
|
||||
session_a: &ReplaySession,
|
||||
session_b: &ReplaySession,
|
||||
@@ -643,7 +649,7 @@ pub struct SessionComparison {
|
||||
|
||||
impl SessionComparison {
|
||||
/// Check if the sessions are identical.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn is_identical(&self) -> bool {
|
||||
self.actions_only_in_a.is_empty()
|
||||
&& self.actions_only_in_b.is_empty()
|
||||
|
||||
+6
-6
@@ -116,7 +116,7 @@ impl ActiveSession {
|
||||
}
|
||||
|
||||
/// End the session and return it
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn end(mut self) -> ReplaySession {
|
||||
self.session.end();
|
||||
self.session
|
||||
@@ -131,7 +131,7 @@ impl ActiveSession {
|
||||
|
||||
/// Storage operations for replay segments
|
||||
pub mod storage {
|
||||
use super::{REPLAY_SEGMENT_MAGIC, REPLAY_SEGMENT_VERSION, Result, MemvidError, ReplaySession};
|
||||
use super::{MemvidError, REPLAY_SEGMENT_MAGIC, REPLAY_SEGMENT_VERSION, ReplaySession, Result};
|
||||
use bincode::config::{self, Config};
|
||||
use std::io::{Read, Write};
|
||||
|
||||
@@ -235,11 +235,11 @@ pub mod storage {
|
||||
}
|
||||
|
||||
let header = ReplaySegmentHeader::new(
|
||||
sessions.len() as u32,
|
||||
u32::try_from(sessions.len()).unwrap_or(u32::MAX),
|
||||
ReplaySegmentHeader::SIZE as u64 + total_session_bytes,
|
||||
);
|
||||
|
||||
let mut segment = Vec::with_capacity(header.total_size as usize);
|
||||
let mut segment = Vec::with_capacity(usize::try_from(header.total_size).unwrap_or(0));
|
||||
header.write(&mut segment)?;
|
||||
|
||||
// Write each session with length prefix
|
||||
@@ -260,7 +260,7 @@ pub mod storage {
|
||||
for _ in 0..header.session_count {
|
||||
let mut len_bytes = [0u8; 8];
|
||||
cursor.read_exact(&mut len_bytes)?;
|
||||
let len = u64::from_le_bytes(len_bytes) as usize;
|
||||
let len = usize::try_from(u64::from_le_bytes(len_bytes)).unwrap_or(0);
|
||||
|
||||
let mut session_data = vec![0u8; len];
|
||||
cursor.read_exact(&mut session_data)?;
|
||||
@@ -302,7 +302,7 @@ pub mod storage {
|
||||
reason: "Invalid active session magic".into(),
|
||||
});
|
||||
}
|
||||
let len = u64::from_le_bytes(data[8..16].try_into().unwrap()) as usize;
|
||||
let len = usize::try_from(u64::from_le_bytes(data[8..16].try_into().unwrap())).unwrap_or(0);
|
||||
if data.len() < 16 + len {
|
||||
return Err(MemvidError::InvalidToc {
|
||||
reason: "Active session data truncated".into(),
|
||||
|
||||
@@ -158,16 +158,12 @@ impl QueryPlanner<'_> {
|
||||
)))
|
||||
}
|
||||
FieldTerm::DateRange(range) => {
|
||||
let lower = range
|
||||
.start
|
||||
.map_or(Bound::Unbounded, |value| {
|
||||
Bound::Included(Term::from_field_i64(self.engine.timestamp, value))
|
||||
});
|
||||
let upper = range
|
||||
.end
|
||||
.map_or(Bound::Unbounded, |value| {
|
||||
Bound::Included(Term::from_field_i64(self.engine.timestamp, value))
|
||||
});
|
||||
let lower = range.start.map_or(Bound::Unbounded, |value| {
|
||||
Bound::Included(Term::from_field_i64(self.engine.timestamp, value))
|
||||
});
|
||||
let upper = range.end.map_or(Bound::Unbounded, |value| {
|
||||
Bound::Included(Term::from_field_i64(self.engine.timestamp, value))
|
||||
});
|
||||
Ok(Box::new(RangeQuery::new(lower, upper)))
|
||||
}
|
||||
}
|
||||
|
||||
+2
-2
@@ -11,7 +11,7 @@ use wide::f32x8;
|
||||
/// Uses 8-wide SIMD lanes (AVX2 on `x86_64`, NEON on aarch64).
|
||||
/// Falls back to scalar for remainder elements.
|
||||
#[cfg(feature = "simd")]
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn l2_distance_squared_simd(a: &[f32], b: &[f32]) -> f32 {
|
||||
debug_assert_eq!(a.len(), b.len(), "vectors must have same length");
|
||||
|
||||
@@ -64,7 +64,7 @@ pub fn l2_distance_squared_simd(a: &[f32], b: &[f32]) -> f32 {
|
||||
|
||||
/// Compute L2 distance (with sqrt) using SIMD.
|
||||
#[cfg(feature = "simd")]
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn l2_distance_simd(a: &[f32], b: &[f32]) -> f32 {
|
||||
l2_distance_squared_simd(a, b).sqrt()
|
||||
}
|
||||
|
||||
+21
-21
@@ -40,13 +40,13 @@ impl Default for StructuralChunker {
|
||||
|
||||
impl StructuralChunker {
|
||||
/// Create a new chunker with the given options.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn new(options: ChunkingOptions) -> Self {
|
||||
Self { options }
|
||||
}
|
||||
|
||||
/// Create a chunker with default options and custom max chars.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn with_max_chars(max_chars: usize) -> Self {
|
||||
Self {
|
||||
options: ChunkingOptions {
|
||||
@@ -57,7 +57,7 @@ impl StructuralChunker {
|
||||
}
|
||||
|
||||
/// Chunk a structured document.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn chunk(&self, doc: &StructuredDocument) -> ChunkingResult {
|
||||
let mut result = ChunkingResult::empty();
|
||||
let mut current_text = String::new();
|
||||
@@ -331,7 +331,7 @@ impl StructuralChunker {
|
||||
index,
|
||||
&table.id,
|
||||
part as u32,
|
||||
total_parts as u32,
|
||||
u32::try_from(total_parts).unwrap_or(0),
|
||||
&header_text,
|
||||
char_start,
|
||||
char_end,
|
||||
@@ -526,7 +526,7 @@ impl StructuralChunker {
|
||||
index,
|
||||
element_id: None,
|
||||
part: Some(1),
|
||||
total_parts: Some(total_parts as u32),
|
||||
total_parts: Some(u32::try_from(total_parts).unwrap_or(0)),
|
||||
context: language.map(std::string::ToString::to_string),
|
||||
char_start,
|
||||
char_end,
|
||||
@@ -537,8 +537,8 @@ impl StructuralChunker {
|
||||
chunk_type: ChunkType::CodeBlockContinuation,
|
||||
index,
|
||||
element_id: None,
|
||||
part: Some((i + 1) as u32),
|
||||
total_parts: Some(total_parts as u32),
|
||||
part: Some(u32::try_from(i + 1).unwrap_or(0)),
|
||||
total_parts: Some(u32::try_from(total_parts).unwrap_or(0)),
|
||||
context: language.map(std::string::ToString::to_string),
|
||||
char_start,
|
||||
char_end,
|
||||
@@ -617,8 +617,8 @@ impl StructuralChunker {
|
||||
chunk_type,
|
||||
index,
|
||||
element_id: None,
|
||||
part: Some((i + 1) as u32),
|
||||
total_parts: Some(total_parts as u32),
|
||||
part: Some(u32::try_from(i + 1).unwrap_or(0)),
|
||||
total_parts: Some(u32::try_from(total_parts).unwrap_or(0)),
|
||||
context: language.map(std::string::ToString::to_string),
|
||||
char_start,
|
||||
char_end,
|
||||
@@ -628,13 +628,13 @@ impl StructuralChunker {
|
||||
}
|
||||
|
||||
/// Convenience function to chunk text with default options.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn chunk_structured(doc: &StructuredDocument) -> ChunkingResult {
|
||||
StructuralChunker::default().chunk(doc)
|
||||
}
|
||||
|
||||
/// Convenience function to chunk text with custom max chars.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn chunk_structured_with_max(doc: &StructuredDocument, max_chars: usize) -> ChunkingResult {
|
||||
StructuralChunker::with_max_chars(max_chars).chunk(doc)
|
||||
}
|
||||
@@ -656,14 +656,14 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_table_preserved_when_small() {
|
||||
let text = r#"Introduction.
|
||||
let text = r"Introduction.
|
||||
|
||||
| Name | Age |
|
||||
|------|-----|
|
||||
| Alice | 30 |
|
||||
| Bob | 25 |
|
||||
|
||||
Conclusion."#;
|
||||
Conclusion.";
|
||||
|
||||
let doc = detect_structure(text);
|
||||
let result = chunk_structured(&doc);
|
||||
@@ -688,12 +688,12 @@ Conclusion."#;
|
||||
}
|
||||
|
||||
let text = format!(
|
||||
r#"Introduction.
|
||||
r"Introduction.
|
||||
|
||||
| Column A | Column B | Column C |
|
||||
|----------|----------|----------|
|
||||
{}
|
||||
Conclusion."#,
|
||||
Conclusion.",
|
||||
rows
|
||||
);
|
||||
|
||||
@@ -781,10 +781,10 @@ All done."#;
|
||||
|
||||
#[test]
|
||||
fn test_table_header_formatting() {
|
||||
let text = r#"| Col1 | Col2 | Col3 |
|
||||
let text = r"| Col1 | Col2 | Col3 |
|
||||
|------|------|------|
|
||||
| A1 | A2 | A3 |
|
||||
| B1 | B2 | B3 |"#;
|
||||
| B1 | B2 | B3 |";
|
||||
|
||||
let doc = detect_structure(text);
|
||||
let table = doc.tables().next().unwrap();
|
||||
@@ -802,9 +802,9 @@ All done."#;
|
||||
}
|
||||
|
||||
let text = format!(
|
||||
r#"| Header1 | Header2 |
|
||||
r"| Header1 | Header2 |
|
||||
|---------|---------|
|
||||
{}"#,
|
||||
{}",
|
||||
rows
|
||||
);
|
||||
|
||||
@@ -825,7 +825,7 @@ All done."#;
|
||||
|
||||
#[test]
|
||||
fn test_chunking_result_stats() {
|
||||
let text = r#"| A | B |
|
||||
let text = r"| A | B |
|
||||
|---|---|
|
||||
| 1 | 2 |
|
||||
|
||||
@@ -835,7 +835,7 @@ x = 1
|
||||
|
||||
| C | D |
|
||||
|---|---|
|
||||
| 3 | 4 |"#;
|
||||
| 3 | 4 |";
|
||||
|
||||
let doc = detect_structure(text);
|
||||
let result = chunk_structured(&doc);
|
||||
|
||||
+12
-11
@@ -390,7 +390,8 @@ fn try_detect_heading(
|
||||
patterns: &Patterns,
|
||||
) -> Option<DocumentElement> {
|
||||
let caps = patterns.heading.captures(line)?;
|
||||
let level = caps.get(1)?.as_str().len() as u8;
|
||||
// Safe: markdown headers are ### (max few chars).
|
||||
let level = u8::try_from(caps.get(1)?.as_str().len()).unwrap_or(0);
|
||||
let text = caps.get(2)?.as_str().to_string();
|
||||
let char_end = char_start + line.len();
|
||||
|
||||
@@ -592,14 +593,14 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_detect_markdown_table() {
|
||||
let text = r#"Some text before.
|
||||
let text = r"Some text before.
|
||||
|
||||
| Name | Age | City |
|
||||
|------|-----|------|
|
||||
| Alice | 30 | NYC |
|
||||
| Bob | 25 | LA |
|
||||
|
||||
Some text after."#;
|
||||
Some text after.";
|
||||
|
||||
let doc = detect_structure(text);
|
||||
assert_eq!(doc.table_count, 1);
|
||||
@@ -641,7 +642,7 @@ And more text."#;
|
||||
|
||||
#[test]
|
||||
fn test_detect_lists() {
|
||||
let text = r#"Shopping list:
|
||||
let text = r"Shopping list:
|
||||
|
||||
- Apples
|
||||
- Bananas
|
||||
@@ -651,7 +652,7 @@ Steps:
|
||||
|
||||
1. First step
|
||||
2. Second step
|
||||
3. Third step"#;
|
||||
3. Third step";
|
||||
|
||||
let doc = detect_structure(text);
|
||||
|
||||
@@ -666,7 +667,7 @@ Steps:
|
||||
|
||||
#[test]
|
||||
fn test_detect_headings() {
|
||||
let text = r#"# Main Title
|
||||
let text = r"# Main Title
|
||||
|
||||
Some intro text.
|
||||
|
||||
@@ -676,7 +677,7 @@ Content here.
|
||||
|
||||
### Subsection
|
||||
|
||||
More content."#;
|
||||
More content.";
|
||||
|
||||
let doc = detect_structure(text);
|
||||
|
||||
@@ -696,7 +697,7 @@ More content."#;
|
||||
|
||||
#[test]
|
||||
fn test_complex_document() {
|
||||
let text = r#"# Report
|
||||
let text = r"# Report
|
||||
|
||||
## Summary
|
||||
|
||||
@@ -724,7 +725,7 @@ def calculate_growth(current, previous):
|
||||
|
||||
## Conclusion
|
||||
|
||||
Strong performance overall."#;
|
||||
Strong performance overall.";
|
||||
|
||||
let doc = detect_structure(text);
|
||||
|
||||
@@ -772,10 +773,10 @@ Strong performance overall."#;
|
||||
|
||||
#[test]
|
||||
fn test_ascii_table_detection() {
|
||||
let text = r#"Name Age City
|
||||
let text = r"Name Age City
|
||||
Alice 30 NYC
|
||||
Bob 25 LA
|
||||
Charlie 35 SF"#;
|
||||
Charlie 35 SF";
|
||||
|
||||
let tables = detect_ascii_tables(text);
|
||||
assert_eq!(tables.len(), 1);
|
||||
|
||||
+3
-3
@@ -320,7 +320,7 @@ pub fn extract_pdf_layout(bytes: &[u8], max_pages: usize) -> Result<Vec<PageLayo
|
||||
let mut layouts = Vec::with_capacity(pages_to_process);
|
||||
|
||||
for page_idx in 0..pages_to_process {
|
||||
let page_number = (page_idx + 1) as u32;
|
||||
let page_number = u32::try_from(page_idx + 1).unwrap_or(0);
|
||||
|
||||
// Get page dimensions (default to standard US Letter if not available)
|
||||
let (width, height) = get_page_dimensions(&document, page_idx).unwrap_or((612.0, 792.0));
|
||||
@@ -452,7 +452,7 @@ fn parse_line_into_columns(
|
||||
#[cfg(not(feature = "pdfium"))]
|
||||
fn get_page_dimensions(document: &lopdf::Document, page_idx: usize) -> Option<(f32, f32)> {
|
||||
let pages = document.get_pages();
|
||||
let page_id = *pages.get(&((page_idx + 1) as u32))?;
|
||||
let page_id = *pages.get(&u32::try_from(page_idx + 1).unwrap_or(0))?;
|
||||
|
||||
if let Ok(page) = document.get_dictionary(page_id) {
|
||||
if let Ok(media_box) = page.get(b"MediaBox") {
|
||||
@@ -480,7 +480,7 @@ fn get_page_dimensions(document: &lopdf::Document, page_idx: usize) -> Option<(f
|
||||
///
|
||||
/// Groups values that are within `threshold` of each other
|
||||
/// and returns cluster centroids.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn cluster_values(values: &[f32], threshold: f32) -> Vec<f32> {
|
||||
if values.is_empty() {
|
||||
return Vec::new();
|
||||
|
||||
@@ -19,7 +19,7 @@ use super::types::{ExtractedTable, TableExtractionOptions, TableQuality, TableRo
|
||||
///
|
||||
/// # Returns
|
||||
/// Vector of tables with multi-page tables merged
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn merge_multi_page_tables(
|
||||
tables: Vec<ExtractedTable>,
|
||||
options: &TableExtractionOptions,
|
||||
@@ -263,7 +263,7 @@ fn combined_quality(q1: TableQuality, q2: TableQuality) -> TableQuality {
|
||||
///
|
||||
/// This is a utility function that identifies tables that might
|
||||
/// be continuations of other tables without actually merging them.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn find_continuation_candidates(
|
||||
tables: &[ExtractedTable],
|
||||
options: &TableExtractionOptions,
|
||||
@@ -301,7 +301,7 @@ mod tests {
|
||||
let mut table = ExtractedTable::new(id, source);
|
||||
table.page_start = page_start;
|
||||
table.page_end = page_end;
|
||||
table.headers = headers.iter().map(|s| s.to_string()).collect();
|
||||
table.headers = headers.iter().map(|s| (*s).to_string()).collect();
|
||||
table.n_cols = headers.len();
|
||||
table.n_rows = n_rows;
|
||||
|
||||
|
||||
@@ -41,7 +41,7 @@ pub fn extract_tables_from_pdf(
|
||||
|
||||
// Extract page layouts
|
||||
let layouts = extract_pdf_layout(bytes, options.max_pages)?;
|
||||
let pages_processed = layouts.len() as u32;
|
||||
let pages_processed = u32::try_from(layouts.len()).unwrap_or(0);
|
||||
|
||||
let mut all_tables = Vec::new();
|
||||
let mut warnings = Vec::new();
|
||||
@@ -102,7 +102,7 @@ pub fn extract_tables_from_pdf(
|
||||
}
|
||||
}
|
||||
|
||||
let total_ms = start.elapsed().as_millis() as u64;
|
||||
let total_ms = start.elapsed().as_millis().try_into().unwrap_or(u64::MAX);
|
||||
|
||||
if all_tables.is_empty() && pages_processed > 0 {
|
||||
warnings.push("No tables detected in document".to_string());
|
||||
@@ -651,7 +651,7 @@ fn extract_raw_text(bytes: &[u8]) -> Option<String> {
|
||||
let pages = document.get_pages();
|
||||
let mut all_text = String::new();
|
||||
|
||||
for page_num in 1..=pages.len() as u32 {
|
||||
for page_num in 1..=u32::try_from(pages.len()).unwrap_or(0) {
|
||||
if let Ok(text) = document.extract_text(&[page_num]) {
|
||||
all_text.push_str(&text);
|
||||
all_text.push('\n');
|
||||
|
||||
+22
-8
@@ -305,7 +305,10 @@ pub fn list_tables(mem: &mut Memvid) -> Result<Vec<TableSummary>> {
|
||||
.to_string();
|
||||
let page_start = meta["page_start"].as_u64().unwrap_or(0);
|
||||
let page_end = meta["page_end"].as_u64().unwrap_or(0);
|
||||
// Safe: table dimensions fit throughout supported platforms
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
let n_rows = meta["n_rows"].as_u64().unwrap_or(0) as usize;
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
let n_cols = meta["n_cols"].as_u64().unwrap_or(0) as usize;
|
||||
let quality = meta["quality"].as_str().unwrap_or("unknown").to_string();
|
||||
let headers = meta["headers"]
|
||||
@@ -321,8 +324,8 @@ pub fn list_tables(mem: &mut Memvid) -> Result<Vec<TableSummary>> {
|
||||
summaries.push(TableSummary {
|
||||
table_id,
|
||||
source_file,
|
||||
page_start: page_start as u32,
|
||||
page_end: page_end as u32,
|
||||
page_start: u32::try_from(page_start).unwrap_or(0),
|
||||
page_end: u32::try_from(page_end).unwrap_or(0),
|
||||
n_rows,
|
||||
n_cols,
|
||||
quality: quality.parse().unwrap_or(TableQuality::Medium),
|
||||
@@ -377,11 +380,20 @@ pub fn get_table(mem: &mut Memvid, table_id: &str) -> Result<Option<ExtractedTab
|
||||
);
|
||||
|
||||
table.source_uri = meta["source_uri"].as_str().map(String::from);
|
||||
table.page_start = meta["page_start"].as_u64().unwrap_or(1) as u32;
|
||||
table.page_end = meta["page_end"].as_u64().unwrap_or(1) as u32;
|
||||
table.n_cols = meta["n_cols"].as_u64().unwrap_or(0) as usize;
|
||||
table.n_rows = meta["n_rows"].as_u64().unwrap_or(0) as usize;
|
||||
table.confidence_score = meta["confidence_score"].as_f64().unwrap_or(0.5) as f32;
|
||||
table.page_start = u32::try_from(meta["page_start"].as_u64().unwrap_or(1)).unwrap_or(1);
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
{
|
||||
table.page_end = meta["page_end"].as_u64().unwrap_or(1) as u32;
|
||||
}
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
{
|
||||
table.n_cols = meta["n_cols"].as_u64().unwrap_or(0) as usize;
|
||||
table.n_rows = meta["n_rows"].as_u64().unwrap_or(0) as usize;
|
||||
}
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
{
|
||||
table.confidence_score = meta["confidence_score"].as_f64().unwrap_or(0.5) as f32;
|
||||
}
|
||||
table.extraction_ms = meta["extraction_ms"].as_u64().unwrap_or(0);
|
||||
|
||||
table.headers = meta["headers"]
|
||||
@@ -443,7 +455,9 @@ pub fn get_table(mem: &mut Memvid, table_id: &str) -> Result<Option<ExtractedTab
|
||||
reason: format!("failed to parse row data: {e}"),
|
||||
})?;
|
||||
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
let row_index = row_data["row_index"].as_u64().unwrap_or(0) as usize;
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
let page = row_data["page"].as_u64().unwrap_or(1) as u32;
|
||||
|
||||
let cells: Vec<super::types::TableCell> =
|
||||
@@ -480,7 +494,7 @@ pub fn get_table(mem: &mut Memvid, table_id: &str) -> Result<Option<ExtractedTab
|
||||
///
|
||||
/// # Returns
|
||||
/// CSV formatted string
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn export_to_csv(table: &ExtractedTable) -> String {
|
||||
let mut output = String::new();
|
||||
|
||||
|
||||
+1
-3
@@ -313,8 +313,7 @@ impl TableExtractionOptions {
|
||||
}
|
||||
|
||||
/// Builder for `TableExtractionOptions`.
|
||||
#[derive(Debug, Clone)]
|
||||
#[derive(Default)]
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct TableExtractionOptionsBuilder {
|
||||
inner: TableExtractionOptions,
|
||||
}
|
||||
@@ -390,7 +389,6 @@ impl TableExtractionOptionsBuilder {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/// Extraction mode controls quality vs coverage tradeoff.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||
pub enum ExtractionMode {
|
||||
|
||||
+5
-2
@@ -10,6 +10,7 @@ use crate::{
|
||||
},
|
||||
};
|
||||
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
fn canonical_config() -> impl bincode::config::Config {
|
||||
bincode::config::standard()
|
||||
.with_fixed_int_encoding()
|
||||
@@ -121,7 +122,9 @@ impl Toc {
|
||||
}
|
||||
|
||||
// Try V2 format (with memories_track/logic_mesh, without replay_manifest)
|
||||
if let Ok((legacy, bytes_read)) = decode_from_slice::<LegacyTocV2, _>(bytes, canonical_config()) {
|
||||
if let Ok((legacy, bytes_read)) =
|
||||
decode_from_slice::<LegacyTocV2, _>(bytes, canonical_config())
|
||||
{
|
||||
if bytes_read != bytes.len() {
|
||||
return Err(MemvidError::InvalidToc {
|
||||
reason: "unexpected trailing bytes in V2 format".into(),
|
||||
@@ -185,7 +188,7 @@ impl LegacyTocV2 {
|
||||
|
||||
impl Toc {
|
||||
/// Computes the BLAKE3 checksum used for the TOC integrity field.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn calculate_checksum(bytes: &[u8]) -> [u8; 32] {
|
||||
let mut hasher = Hasher::new();
|
||||
hasher.update(bytes);
|
||||
|
||||
@@ -128,7 +128,7 @@ impl TripletExtractor {
|
||||
0
|
||||
};
|
||||
|
||||
let elapsed_ms = start.elapsed().as_millis() as u64;
|
||||
let elapsed_ms = start.elapsed().as_millis().try_into().unwrap_or(u64::MAX);
|
||||
let rules_count = all_cards.len();
|
||||
|
||||
// Deduplicate cards with same entity:slot
|
||||
@@ -144,7 +144,7 @@ impl TripletExtractor {
|
||||
/// Extract triplets from an existing `EnrichmentContext`.
|
||||
///
|
||||
/// This is useful when you already have a context from the enrichment pipeline.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn extract_from_context(
|
||||
&self,
|
||||
ctx: &EnrichmentContext,
|
||||
|
||||
+10
-10
@@ -86,7 +86,7 @@ impl Default for AdaptiveConfig {
|
||||
|
||||
impl AdaptiveConfig {
|
||||
/// Create a config with absolute threshold strategy.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn with_absolute_threshold(min_score: f32) -> Self {
|
||||
Self {
|
||||
strategy: CutoffStrategy::AbsoluteThreshold { min_score },
|
||||
@@ -95,7 +95,7 @@ impl AdaptiveConfig {
|
||||
}
|
||||
|
||||
/// Create a config with relative threshold strategy.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn with_relative_threshold(min_ratio: f32) -> Self {
|
||||
Self {
|
||||
strategy: CutoffStrategy::RelativeThreshold { min_ratio },
|
||||
@@ -104,7 +104,7 @@ impl AdaptiveConfig {
|
||||
}
|
||||
|
||||
/// Create a config with score cliff detection.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn with_score_cliff(max_drop_ratio: f32) -> Self {
|
||||
Self {
|
||||
strategy: CutoffStrategy::ScoreCliff { max_drop_ratio },
|
||||
@@ -113,7 +113,7 @@ impl AdaptiveConfig {
|
||||
}
|
||||
|
||||
/// Create a config with automatic elbow detection.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn with_elbow_detection() -> Self {
|
||||
Self {
|
||||
strategy: CutoffStrategy::Elbow { sensitivity: 1.0 },
|
||||
@@ -122,7 +122,7 @@ impl AdaptiveConfig {
|
||||
}
|
||||
|
||||
/// Create a combined strategy (recommended for production).
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn combined(min_ratio: f32, max_drop: f32, min_score: f32) -> Self {
|
||||
Self {
|
||||
strategy: CutoffStrategy::Combined {
|
||||
@@ -236,7 +236,7 @@ pub struct AdaptiveStats {
|
||||
|
||||
impl<T> AdaptiveResult<T> {
|
||||
/// Create an empty result.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn empty() -> Self {
|
||||
Self {
|
||||
results: Vec::new(),
|
||||
@@ -365,9 +365,9 @@ pub fn compute_embedding_quality(embeddings: &[(u64, Vec<f32>)]) -> EmbeddingQua
|
||||
while similarities.len() < max_pairs {
|
||||
// Simple LCG random
|
||||
rng_state = rng_state.wrapping_mul(6364136223846793005).wrapping_add(1);
|
||||
let i = (rng_state as usize) % vector_count;
|
||||
let i = usize::try_from(rng_state % (vector_count as u64)).unwrap_or(0);
|
||||
rng_state = rng_state.wrapping_mul(6364136223846793005).wrapping_add(1);
|
||||
let j = (rng_state as usize) % vector_count;
|
||||
let j = usize::try_from(rng_state % (vector_count as u64)).unwrap_or(0);
|
||||
|
||||
if i != j {
|
||||
let pair = if i < j { (i, j) } else { (j, i) };
|
||||
@@ -500,7 +500,7 @@ fn cosine_similarity(a: &[f32], b: &[f32]) -> f32 {
|
||||
///
|
||||
/// Returns (`cutoff_index`, `triggered_by_strategy`).
|
||||
/// Results at indices `0..cutoff_index` should be included.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn find_adaptive_cutoff(scores: &[f32], config: &AdaptiveConfig) -> (usize, String) {
|
||||
if scores.is_empty() {
|
||||
return (0, "no_results".to_string());
|
||||
@@ -758,7 +758,7 @@ mod tests {
|
||||
|
||||
// Should stop either at relative threshold (50% of 0.95 = 0.475)
|
||||
// or at cliff (0.75 -> 0.40 is ~47% drop)
|
||||
assert!(cutoff >= 4 && cutoff <= 6);
|
||||
assert!((4..=6).contains(&cutoff));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
+11
-12
@@ -130,7 +130,7 @@ pub struct AuditReport {
|
||||
|
||||
impl AuditReport {
|
||||
/// Format the report as human-readable text.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn to_text(&self) -> String {
|
||||
let mut output = String::new();
|
||||
|
||||
@@ -270,7 +270,7 @@ impl AuditReport {
|
||||
}
|
||||
|
||||
/// Format the report as Markdown.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn to_markdown(&self) -> String {
|
||||
let mut output = String::new();
|
||||
|
||||
@@ -415,13 +415,12 @@ fn format_snippet_block(text: &str, width: usize, prefix: &str) -> String {
|
||||
let mut line = String::new();
|
||||
|
||||
for word in cleaned.split_whitespace() {
|
||||
if line.len() + word.len() + 1 > width
|
||||
&& !line.is_empty() {
|
||||
result.push_str(prefix);
|
||||
result.push_str(&line);
|
||||
result.push('\n');
|
||||
line.clear();
|
||||
}
|
||||
if line.len() + word.len() + 1 > width && !line.is_empty() {
|
||||
result.push_str(prefix);
|
||||
result.push_str(&line);
|
||||
result.push('\n');
|
||||
line.clear();
|
||||
}
|
||||
if !line.is_empty() {
|
||||
line.push(' ');
|
||||
}
|
||||
@@ -477,6 +476,8 @@ fn format_timestamp(ts: i64) -> String {
|
||||
|
||||
// Approximate year/month/day calculation
|
||||
let mut year = 1970i32;
|
||||
// Safe: days will fit in i32 for any reasonable usage (millions of years)
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
let mut remaining_days = days as i32;
|
||||
|
||||
loop {
|
||||
@@ -505,9 +506,7 @@ fn format_timestamp(ts: i64) -> String {
|
||||
|
||||
let day = remaining_days + 1;
|
||||
|
||||
format!(
|
||||
"{year:04}-{month:02}-{day:02}T{hours:02}:{minutes:02}:{seconds:02}Z"
|
||||
)
|
||||
format!("{year:04}-{month:02}-{day:02}T{hours:02}:{minutes:02}:{seconds:02}Z")
|
||||
}
|
||||
|
||||
fn is_leap_year(year: i32) -> bool {
|
||||
|
||||
+4
-4
@@ -18,7 +18,7 @@ pub enum CanonicalEncoding {
|
||||
}
|
||||
|
||||
impl CanonicalEncoding {
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub const fn from_byte(value: u8) -> Self {
|
||||
match value {
|
||||
0 => CanonicalEncoding::Plain,
|
||||
@@ -27,7 +27,7 @@ impl CanonicalEncoding {
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub const fn as_byte(self) -> u8 {
|
||||
match self {
|
||||
CanonicalEncoding::Plain => 0,
|
||||
@@ -143,13 +143,13 @@ pub enum EnrichmentState {
|
||||
|
||||
impl EnrichmentState {
|
||||
/// Returns true if this frame needs background enrichment.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn needs_enrichment(&self) -> bool {
|
||||
matches!(self, Self::Searchable)
|
||||
}
|
||||
|
||||
/// Returns true if this frame has full semantic search capability.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn has_embeddings(&self) -> bool {
|
||||
matches!(self, Self::Enriched)
|
||||
}
|
||||
|
||||
@@ -205,10 +205,12 @@ mod tests {
|
||||
}
|
||||
|
||||
impl EmbeddingProvider for MockProvider {
|
||||
#[allow(clippy::unnecessary_literal_bound)]
|
||||
fn kind(&self) -> &str {
|
||||
"mock"
|
||||
}
|
||||
|
||||
#[allow(clippy::unnecessary_literal_bound)]
|
||||
fn model(&self) -> &str {
|
||||
"mock-model"
|
||||
}
|
||||
|
||||
+8
-10
@@ -19,8 +19,7 @@ use super::{
|
||||
// Note: AnchorSource is always defined (not feature-gated) to maintain binary compatibility
|
||||
|
||||
/// Timeline query parameters for scanning frames chronologically or in reverse.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[derive(Default)]
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||
pub struct TimelineQuery {
|
||||
pub limit: Option<NonZeroU64>,
|
||||
pub since: Option<i64>,
|
||||
@@ -33,38 +32,37 @@ pub struct TimelineQuery {
|
||||
|
||||
impl TimelineQuery {
|
||||
/// Start a fluent builder for timeline queries.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn builder() -> TimelineQueryBuilder {
|
||||
TimelineQueryBuilder::default()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct TimelineQueryBuilder {
|
||||
inner: TimelineQuery,
|
||||
}
|
||||
|
||||
impl TimelineQueryBuilder {
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn limit(mut self, limit: NonZeroU64) -> Self {
|
||||
self.inner.limit = Some(limit);
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn since(mut self, ts: i64) -> Self {
|
||||
self.inner.since = Some(ts);
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn until(mut self, ts: i64) -> Self {
|
||||
self.inner.until = Some(ts);
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn reverse(mut self, reverse: bool) -> Self {
|
||||
self.inner.reverse = reverse;
|
||||
self
|
||||
@@ -76,13 +74,13 @@ impl TimelineQueryBuilder {
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn no_limit(mut self) -> Self {
|
||||
self.inner.limit = None;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn build(mut self) -> TimelineQuery {
|
||||
if self.inner.limit.is_none() {
|
||||
self.inner.limit = NonZeroU64::new(100);
|
||||
|
||||
@@ -20,7 +20,7 @@ pub struct TriplePattern {
|
||||
|
||||
impl TriplePattern {
|
||||
/// Create a new triple pattern.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn new(subject: PatternTerm, predicate: PatternTerm, object: PatternTerm) -> Self {
|
||||
Self {
|
||||
subject,
|
||||
@@ -30,7 +30,7 @@ impl TriplePattern {
|
||||
}
|
||||
|
||||
/// Create a pattern matching entity:slot = value
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn entity_slot_value(entity: &str, slot: &str, value: &str) -> Self {
|
||||
Self {
|
||||
subject: PatternTerm::Literal(entity.to_lowercase()),
|
||||
@@ -40,7 +40,7 @@ impl TriplePattern {
|
||||
}
|
||||
|
||||
/// Create a pattern matching entity:slot = ?var (any value)
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn entity_slot_any(entity: &str, slot: &str, var: &str) -> Self {
|
||||
Self {
|
||||
subject: PatternTerm::Literal(entity.to_lowercase()),
|
||||
@@ -50,7 +50,7 @@ impl TriplePattern {
|
||||
}
|
||||
|
||||
/// Create a pattern matching ?entity:slot = value (find entities with this value)
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn any_slot_value(var: &str, slot: &str, value: &str) -> Self {
|
||||
Self {
|
||||
subject: PatternTerm::Variable(var.to_string()),
|
||||
@@ -252,7 +252,7 @@ pub struct GraphMatchResult {
|
||||
|
||||
impl GraphMatchResult {
|
||||
/// Create a new match result.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn new(entity: String, frame_ids: Vec<FrameId>, confidence: f32) -> Self {
|
||||
Self {
|
||||
entity,
|
||||
|
||||
+16
-9
@@ -43,7 +43,7 @@ pub struct MeshNode {
|
||||
|
||||
impl MeshNode {
|
||||
/// Create a new mesh node with computed ID.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn new(
|
||||
canonical_name: String,
|
||||
display_name: String,
|
||||
@@ -54,12 +54,15 @@ impl MeshNode {
|
||||
byte_len: u16,
|
||||
) -> Self {
|
||||
let id = compute_node_id(&canonical_name, kind);
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
let confidence = (confidence * 100.0).min(100.0) as u8;
|
||||
|
||||
Self {
|
||||
id,
|
||||
canonical_name,
|
||||
display_name,
|
||||
kind,
|
||||
confidence: (confidence * 100.0).min(100.0) as u8,
|
||||
confidence,
|
||||
frame_ids: vec![frame_id],
|
||||
mentions: vec![(frame_id, byte_start, byte_len)],
|
||||
}
|
||||
@@ -144,7 +147,7 @@ pub struct MeshEdge {
|
||||
|
||||
impl MeshEdge {
|
||||
/// Create a new edge.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn new(
|
||||
from_node: u64,
|
||||
to_node: u64,
|
||||
@@ -152,11 +155,14 @@ impl MeshEdge {
|
||||
confidence: f32,
|
||||
frame_id: FrameId,
|
||||
) -> Self {
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
let confidence = (confidence * 100.0).min(100.0) as u8;
|
||||
|
||||
Self {
|
||||
from_node,
|
||||
to_node,
|
||||
link,
|
||||
confidence: (confidence * 100.0).min(100.0) as u8,
|
||||
confidence,
|
||||
frame_id,
|
||||
}
|
||||
}
|
||||
@@ -372,11 +378,12 @@ impl LogicMesh {
|
||||
});
|
||||
}
|
||||
|
||||
let compressed_len = u64::from_le_bytes(bytes[6..14].try_into().map_err(|_| {
|
||||
MemvidError::InvalidLogicMesh {
|
||||
let compressed_len = usize::try_from(u64::from_le_bytes(bytes[6..14].try_into().map_err(
|
||||
|_| MemvidError::InvalidLogicMesh {
|
||||
reason: "invalid length header".into(),
|
||||
}
|
||||
})?) as usize;
|
||||
},
|
||||
)?))
|
||||
.unwrap_or(0);
|
||||
|
||||
if bytes.len() < 14 + compressed_len {
|
||||
return Err(MemvidError::InvalidLogicMesh {
|
||||
@@ -448,7 +455,7 @@ impl LogicMesh {
|
||||
}
|
||||
|
||||
/// Follow edges from a start node.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn follow(&self, start: &str, link: &str, hops: usize) -> Vec<FollowResult> {
|
||||
let Some(start_node) = self.find_node(start) else {
|
||||
return Vec::new();
|
||||
|
||||
+7
-15
@@ -244,8 +244,7 @@ pub struct IndexSegmentRef {
|
||||
|
||||
/// Segment category emitted by the parallel builder.
|
||||
/// Always defined for backwards compatibility.
|
||||
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[derive(Default)]
|
||||
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Default)]
|
||||
pub enum SegmentKind {
|
||||
#[default]
|
||||
Lexical,
|
||||
@@ -255,7 +254,6 @@ pub enum SegmentKind {
|
||||
Tantivy,
|
||||
}
|
||||
|
||||
|
||||
/// Build-time metrics captured for a sealed segment.
|
||||
/// Always defined for backwards compatibility.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||
@@ -673,8 +671,7 @@ impl<'de> Deserialize<'de> for TantivySegmentDescriptor {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[derive(Default)]
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||
pub struct IndexManifests {
|
||||
pub lex: Option<LexIndexManifest>,
|
||||
#[serde(default)]
|
||||
@@ -685,7 +682,6 @@ pub struct IndexManifests {
|
||||
pub clip: Option<crate::clip::ClipIndexManifest>,
|
||||
}
|
||||
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct LexIndexManifest {
|
||||
pub doc_count: u64,
|
||||
@@ -704,15 +700,13 @@ pub struct LexSegmentManifest {
|
||||
pub checksum: [u8; 32],
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||
#[derive(Default)]
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)]
|
||||
pub enum VectorCompression {
|
||||
#[default]
|
||||
None, // Full f32 vectors (1,536 bytes for 384 dims)
|
||||
Pq96, // Product quantization with 96 subspaces (96 bytes)
|
||||
}
|
||||
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct VecIndexManifest {
|
||||
pub vector_count: u64,
|
||||
@@ -725,8 +719,7 @@ pub struct VecIndexManifest {
|
||||
pub compression_mode: VectorCompression,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[derive(Default)]
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||
pub enum SegmentCompression {
|
||||
#[default]
|
||||
None,
|
||||
@@ -734,7 +727,6 @@ pub enum SegmentCompression {
|
||||
Lz4,
|
||||
}
|
||||
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct SegmentMeta {
|
||||
pub id: u64,
|
||||
@@ -877,7 +869,7 @@ pub struct EnrichmentQueueManifest {
|
||||
|
||||
impl EnrichmentQueueManifest {
|
||||
/// Create a new empty enrichment queue.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
tasks: Vec::new(),
|
||||
@@ -926,13 +918,13 @@ impl EnrichmentQueueManifest {
|
||||
}
|
||||
|
||||
/// Check if any enrichment work is pending.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.tasks.is_empty()
|
||||
}
|
||||
|
||||
/// Get count of pending tasks.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn len(&self) -> usize {
|
||||
self.tasks.len()
|
||||
}
|
||||
|
||||
@@ -533,11 +533,12 @@ impl MemoriesTrack {
|
||||
});
|
||||
}
|
||||
|
||||
let len = u64::from_le_bytes([
|
||||
let len = usize::try_from(u64::from_le_bytes([
|
||||
data[6], data[7], data[8], data[9], data[10], data[11], data[12],
|
||||
data[13],
|
||||
// Safe: checked on next line that data.len() >= 14 + len, so len fits in available memory
|
||||
]) as usize;
|
||||
]))
|
||||
.unwrap_or(0);
|
||||
if data.len() < 14 + len {
|
||||
return Err(MemvidError::InvalidHeader {
|
||||
reason: "memories track data truncated".into(),
|
||||
|
||||
+15
-17
@@ -107,21 +107,19 @@ impl Default for PutOptions {
|
||||
|
||||
impl PutOptions {
|
||||
/// Start a fluent builder for `PutOptions`.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn builder() -> PutOptionsBuilder {
|
||||
PutOptionsBuilder::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
#[derive(Default)]
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct PutOptionsBuilder {
|
||||
inner: PutOptions,
|
||||
}
|
||||
|
||||
|
||||
impl PutOptionsBuilder {
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn timestamp(mut self, timestamp: i64) -> Self {
|
||||
self.inner.timestamp = Some(timestamp);
|
||||
self
|
||||
@@ -152,7 +150,7 @@ impl PutOptionsBuilder {
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn metadata(mut self, metadata: DocMetadata) -> Self {
|
||||
self.inner.metadata = Some(metadata);
|
||||
self
|
||||
@@ -183,44 +181,44 @@ impl PutOptionsBuilder {
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn enable_embedding(mut self, enable: bool) -> Self {
|
||||
self.inner.enable_embedding = enable;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn auto_tag(mut self, enabled: bool) -> Self {
|
||||
self.inner.auto_tag = enabled;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn extract_dates(mut self, enabled: bool) -> Self {
|
||||
self.inner.extract_dates = enabled;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn extract_triplets(mut self, enabled: bool) -> Self {
|
||||
self.inner.extract_triplets = enabled;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn parent_id(mut self, parent_id: FrameId) -> Self {
|
||||
self.inner.parent_id = Some(parent_id);
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn role(mut self, role: FrameRole) -> Self {
|
||||
self.inner.role = role;
|
||||
self
|
||||
}
|
||||
|
||||
/// Don't store raw binary content, only extracted text + SHA256 hash.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn no_raw(mut self, enabled: bool) -> Self {
|
||||
self.inner.no_raw = enabled;
|
||||
self
|
||||
@@ -233,7 +231,7 @@ impl PutOptionsBuilder {
|
||||
}
|
||||
|
||||
/// Skip ingestion if a frame with matching BLAKE3 hash already exists.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn dedup(mut self, enabled: bool) -> Self {
|
||||
self.inner.dedup = enabled;
|
||||
self
|
||||
@@ -241,7 +239,7 @@ impl PutOptionsBuilder {
|
||||
|
||||
/// Enable instant indexing for immediate searchability.
|
||||
/// When disabled, full commit is deferred (faster for batches).
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn instant_index(mut self, enabled: bool) -> Self {
|
||||
self.inner.instant_index = enabled;
|
||||
self
|
||||
@@ -249,13 +247,13 @@ impl PutOptionsBuilder {
|
||||
|
||||
/// Set extraction time budget in milliseconds.
|
||||
/// 0 means no budget (extract everything, slower but complete).
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn extraction_budget_ms(mut self, ms: u64) -> Self {
|
||||
self.inner.extraction_budget_ms = ms;
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn build(self) -> PutOptions {
|
||||
self.inner
|
||||
}
|
||||
|
||||
@@ -151,7 +151,7 @@ impl RerankerConfig {
|
||||
/// ```
|
||||
pub trait Reranker: Send + Sync {
|
||||
/// Return the reranker kind identifier.
|
||||
fn kind(&self) -> &str;
|
||||
fn kind(&self) -> &'static str;
|
||||
|
||||
/// Rerank documents by relevance to the query.
|
||||
///
|
||||
@@ -235,7 +235,7 @@ mod tests {
|
||||
struct MockReranker;
|
||||
|
||||
impl Reranker for MockReranker {
|
||||
fn kind(&self) -> &str {
|
||||
fn kind(&self) -> &'static str {
|
||||
"mock"
|
||||
}
|
||||
|
||||
|
||||
+16
-14
@@ -48,7 +48,7 @@ impl Default for ValueType {
|
||||
|
||||
impl ValueType {
|
||||
/// Check if a value matches this type.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn matches(&self, value: &str) -> bool {
|
||||
match self {
|
||||
Self::String | Self::Any => true,
|
||||
@@ -67,7 +67,7 @@ impl ValueType {
|
||||
}
|
||||
|
||||
/// Get a human-readable description of this type.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn description(&self) -> String {
|
||||
match self {
|
||||
Self::String => "string".to_string(),
|
||||
@@ -143,21 +143,21 @@ impl PredicateSchema {
|
||||
}
|
||||
|
||||
/// Set the domain (entity kinds).
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn with_domain(mut self, kinds: Vec<EntityKind>) -> Self {
|
||||
self.domain = kinds;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the range (value type).
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn with_range(mut self, range: ValueType) -> Self {
|
||||
self.range = range;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set cardinality to multiple.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn multiple(mut self) -> Self {
|
||||
self.cardinality = Cardinality::Multiple;
|
||||
self
|
||||
@@ -170,14 +170,14 @@ impl PredicateSchema {
|
||||
}
|
||||
|
||||
/// Mark as built-in.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn builtin(mut self) -> Self {
|
||||
self.builtin = true;
|
||||
self
|
||||
}
|
||||
|
||||
/// Check if an entity kind is in the domain.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn allows_entity(&self, kind: EntityKind) -> bool {
|
||||
self.domain.is_empty() || self.domain.contains(&kind)
|
||||
}
|
||||
@@ -264,7 +264,7 @@ pub struct SchemaRegistry {
|
||||
|
||||
impl SchemaRegistry {
|
||||
/// Create a new registry with built-in schemas.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn new() -> Self {
|
||||
let mut registry = Self {
|
||||
schemas: HashMap::new(),
|
||||
@@ -275,7 +275,7 @@ impl SchemaRegistry {
|
||||
}
|
||||
|
||||
/// Create an empty registry without built-in schemas.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn empty() -> Self {
|
||||
Self {
|
||||
schemas: HashMap::new(),
|
||||
@@ -284,7 +284,7 @@ impl SchemaRegistry {
|
||||
}
|
||||
|
||||
/// Enable strict validation (unknown predicates are rejected).
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn strict(mut self) -> Self {
|
||||
self.strict = true;
|
||||
self
|
||||
@@ -420,13 +420,13 @@ impl SchemaRegistry {
|
||||
}
|
||||
|
||||
/// Get a schema by predicate ID.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn get(&self, predicate: &str) -> Option<&PredicateSchema> {
|
||||
self.schemas.get(predicate)
|
||||
}
|
||||
|
||||
/// Check if a predicate is known.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn contains(&self, predicate: &str) -> bool {
|
||||
self.schemas.contains_key(predicate)
|
||||
}
|
||||
@@ -443,7 +443,9 @@ impl SchemaRegistry {
|
||||
value: &str,
|
||||
entity_kind: Option<EntityKind>,
|
||||
) -> Result<(), SchemaError> {
|
||||
let schema = if let Some(s) = self.schemas.get(predicate) { s } else {
|
||||
let schema = if let Some(s) = self.schemas.get(predicate) {
|
||||
s
|
||||
} else {
|
||||
if self.strict {
|
||||
return Err(SchemaError::UnknownPredicate(predicate.to_string()));
|
||||
}
|
||||
@@ -472,7 +474,7 @@ impl SchemaRegistry {
|
||||
}
|
||||
|
||||
/// Infer a schema from existing memory cards.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn infer_from_values(&self, predicate: &str, values: &[&str]) -> PredicateSchema {
|
||||
let mut schema = PredicateSchema::new(predicate, predicate);
|
||||
|
||||
|
||||
+28
-17
@@ -116,7 +116,6 @@ pub enum SketchVariant {
|
||||
Large = 2,
|
||||
}
|
||||
|
||||
|
||||
impl SketchVariant {
|
||||
/// Get the entry size in bytes for this variant.
|
||||
#[must_use]
|
||||
@@ -591,7 +590,9 @@ pub fn hash_token(token: &str) -> u64 {
|
||||
#[must_use]
|
||||
pub fn hash_token_u32(token: &str) -> u32 {
|
||||
let h = hash_token(token);
|
||||
(h ^ (h >> 32)) as u32
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
let res = (h ^ (h >> 32)) as u32;
|
||||
res
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
@@ -609,9 +610,9 @@ pub fn build_term_filter(token_hashes: &[u64], filter_size_bytes: usize) -> Vec<
|
||||
|
||||
for &hash in token_hashes {
|
||||
// Use 3 hash functions (simulated via rotation)
|
||||
let h1 = hash as usize % filter_bits;
|
||||
let h2 = (hash >> 16) as usize % filter_bits;
|
||||
let h3 = (hash >> 32) as usize % filter_bits;
|
||||
let h1 = usize::try_from(hash % (filter_bits as u64)).unwrap_or(0);
|
||||
let h2 = usize::try_from((hash >> 16) % (filter_bits as u64)).unwrap_or(0);
|
||||
let h3 = usize::try_from((hash >> 32) % (filter_bits as u64)).unwrap_or(0);
|
||||
|
||||
filter[h1 / 8] |= 1 << (h1 % 8);
|
||||
filter[h2 / 8] |= 1 << (h2 % 8);
|
||||
@@ -625,9 +626,9 @@ pub fn build_term_filter(token_hashes: &[u64], filter_size_bytes: usize) -> Vec<
|
||||
#[must_use]
|
||||
pub fn term_filter_maybe_contains(filter: &[u8], token_hash: u64) -> bool {
|
||||
let filter_bits = filter.len() * 8;
|
||||
let h1 = token_hash as usize % filter_bits;
|
||||
let h2 = (token_hash >> 16) as usize % filter_bits;
|
||||
let h3 = (token_hash >> 32) as usize % filter_bits;
|
||||
let h1 = usize::try_from(token_hash % (filter_bits as u64)).unwrap_or(0);
|
||||
let h2 = usize::try_from((token_hash >> 16) % (filter_bits as u64)).unwrap_or(0);
|
||||
let h3 = usize::try_from((token_hash >> 32) % (filter_bits as u64)).unwrap_or(0);
|
||||
|
||||
(filter[h1 / 8] & (1 << (h1 % 8)) != 0)
|
||||
&& (filter[h2 / 8] & (1 << (h2 % 8)) != 0)
|
||||
@@ -684,6 +685,7 @@ pub fn compute_token_weights(
|
||||
.copied()
|
||||
.unwrap_or(1.0)
|
||||
.max(0.1); // Default IDF = 1.0
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
let weight = (capped_tf * idf * 100.0) as i32; // Scale to integer
|
||||
(hash_token(token), weight.max(1))
|
||||
})
|
||||
@@ -701,7 +703,11 @@ pub fn extract_top_terms(weighted_tokens: &[(u64, i32)], k: usize) -> Vec<u32> {
|
||||
weighted_tokens
|
||||
.iter()
|
||||
.take(k)
|
||||
.map(|(h, _)| (*h ^ (*h >> 32)) as u32)
|
||||
.map(|(h, _)| {
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
let res = (*h ^ (*h >> 32)) as u32;
|
||||
res
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
@@ -746,6 +752,7 @@ pub fn generate_sketch(
|
||||
.sum();
|
||||
|
||||
// Length hint: bucket token count (0-255 = 0-2550 tokens in steps of 10)
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
let length_hint = ((token_count / 10).min(255)) as u16;
|
||||
|
||||
let mut flags = SketchFlags::all();
|
||||
@@ -753,12 +760,15 @@ pub fn generate_sketch(
|
||||
flags.set(SketchFlags::SHORT_TEXT);
|
||||
}
|
||||
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
let term_weight_sum = term_weight_sum.min(u32::from(u16::MAX)) as u16;
|
||||
|
||||
SketchEntry {
|
||||
frame_id,
|
||||
simhash,
|
||||
term_filter,
|
||||
top_terms,
|
||||
term_weight_sum: term_weight_sum.min(u32::from(u16::MAX)) as u16,
|
||||
term_weight_sum,
|
||||
flags,
|
||||
length_hint,
|
||||
}
|
||||
@@ -1005,10 +1015,13 @@ impl SketchTrackHeader {
|
||||
/// Create a new header.
|
||||
#[must_use]
|
||||
pub fn new(variant: SketchVariant, entry_count: u64) -> Self {
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
let entry_size = variant.entry_size() as u16;
|
||||
|
||||
Self {
|
||||
magic: SKETCH_TRACK_MAGIC,
|
||||
version: SKETCH_TRACK_VERSION,
|
||||
entry_size: variant.entry_size() as u16,
|
||||
entry_size,
|
||||
entry_count,
|
||||
flags: 0,
|
||||
reserved: 0,
|
||||
@@ -1122,10 +1135,8 @@ pub fn read_sketch_track<R: Read + Seek>(
|
||||
SketchTrackHeader::SIZE as u64 + header.entry_count * u64::from(header.entry_size);
|
||||
if length < expected_length {
|
||||
return Err(MemvidError::InvalidSketchTrack {
|
||||
reason: format!(
|
||||
"Sketch track length {length} less than expected {expected_length}"
|
||||
)
|
||||
.into(),
|
||||
reason: format!("Sketch track length {length} less than expected {expected_length}")
|
||||
.into(),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1200,7 +1211,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_term_filter() {
|
||||
let tokens = vec!["hello", "world", "test"];
|
||||
let tokens = ["hello", "world", "test"];
|
||||
let hashes: Vec<u64> = tokens.iter().map(|t| hash_token(t)).collect();
|
||||
let filter = build_term_filter(&hashes, 16);
|
||||
|
||||
@@ -1306,7 +1317,7 @@ mod tests {
|
||||
// Should find at least some candidates with relaxed threshold
|
||||
// The sketch is optimized for approximate matching, not exact
|
||||
assert!(
|
||||
!candidates.is_empty() || track.len() > 0,
|
||||
!candidates.is_empty() || !track.is_empty(),
|
||||
"Track should have entries"
|
||||
);
|
||||
}
|
||||
|
||||
+29
-29
@@ -160,7 +160,7 @@ pub struct StructuredRow {
|
||||
|
||||
impl StructuredRow {
|
||||
/// Create a new row with cells.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn new(row: usize, cells: Vec<StructuredCell>) -> Self {
|
||||
Self {
|
||||
row,
|
||||
@@ -170,14 +170,14 @@ impl StructuredRow {
|
||||
}
|
||||
|
||||
/// Mark as header row.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn as_header(mut self) -> Self {
|
||||
self.is_header = true;
|
||||
self
|
||||
}
|
||||
|
||||
/// Get cell texts as a vector.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn cell_texts(&self) -> Vec<&str> {
|
||||
self.cells.iter().map(|c| c.text.as_str()).collect()
|
||||
}
|
||||
@@ -219,13 +219,13 @@ impl StructuredTable {
|
||||
}
|
||||
|
||||
/// Get number of data rows.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn data_row_count(&self) -> usize {
|
||||
self.rows.iter().filter(|r| !r.is_header).count()
|
||||
}
|
||||
|
||||
/// Format headers as markdown table header.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn format_header(&self) -> String {
|
||||
if self.headers.is_empty() {
|
||||
return String::new();
|
||||
@@ -243,14 +243,14 @@ impl StructuredTable {
|
||||
}
|
||||
|
||||
/// Format a row as markdown.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn format_row(&self, row: &StructuredRow) -> String {
|
||||
let cells: Vec<&str> = row.cells.iter().map(|c| c.text.as_str()).collect();
|
||||
format!("| {} |", cells.join(" | "))
|
||||
}
|
||||
|
||||
/// Estimate character count.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn char_count(&self) -> usize {
|
||||
self.raw_text.chars().count()
|
||||
}
|
||||
@@ -285,7 +285,7 @@ impl StructuredCodeBlock {
|
||||
}
|
||||
|
||||
/// Format as fenced code block.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn format(&self) -> String {
|
||||
let fence = "```";
|
||||
let lang = self.language.as_deref().unwrap_or("");
|
||||
@@ -293,7 +293,7 @@ impl StructuredCodeBlock {
|
||||
}
|
||||
|
||||
/// Estimate character count.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn char_count(&self) -> usize {
|
||||
self.content.chars().count() + 10 // fences + language
|
||||
}
|
||||
@@ -317,7 +317,7 @@ fn default_start() -> usize {
|
||||
|
||||
impl StructuredList {
|
||||
/// Create a new unordered list.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn unordered(items: Vec<String>) -> Self {
|
||||
Self {
|
||||
ordered: false,
|
||||
@@ -327,7 +327,7 @@ impl StructuredList {
|
||||
}
|
||||
|
||||
/// Create a new ordered list.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn ordered(items: Vec<String>) -> Self {
|
||||
Self {
|
||||
ordered: true,
|
||||
@@ -337,7 +337,7 @@ impl StructuredList {
|
||||
}
|
||||
|
||||
/// Format as markdown list.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn format(&self) -> String {
|
||||
self.items
|
||||
.iter()
|
||||
@@ -354,7 +354,7 @@ impl StructuredList {
|
||||
}
|
||||
|
||||
/// Estimate character count.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn char_count(&self) -> usize {
|
||||
self.items.iter().map(|s| s.chars().count() + 3).sum()
|
||||
}
|
||||
@@ -379,7 +379,7 @@ impl StructuredHeading {
|
||||
}
|
||||
|
||||
/// Format as markdown heading.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn format(&self) -> String {
|
||||
format!("{} {}", "#".repeat(self.level as usize), self.text)
|
||||
}
|
||||
@@ -433,7 +433,7 @@ impl DocumentElement {
|
||||
}
|
||||
|
||||
/// Create a table element.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn table(table: StructuredTable, char_start: usize, char_end: usize) -> Self {
|
||||
Self {
|
||||
element_type: ElementType::Table,
|
||||
@@ -444,7 +444,7 @@ impl DocumentElement {
|
||||
}
|
||||
|
||||
/// Create a code block element.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn code_block(block: StructuredCodeBlock, char_start: usize, char_end: usize) -> Self {
|
||||
Self {
|
||||
element_type: ElementType::CodeBlock,
|
||||
@@ -455,7 +455,7 @@ impl DocumentElement {
|
||||
}
|
||||
|
||||
/// Create a list element.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn list(list: StructuredList, char_start: usize, char_end: usize) -> Self {
|
||||
Self {
|
||||
element_type: ElementType::List,
|
||||
@@ -466,7 +466,7 @@ impl DocumentElement {
|
||||
}
|
||||
|
||||
/// Create a heading element.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn heading(heading: StructuredHeading, char_start: usize, char_end: usize) -> Self {
|
||||
Self {
|
||||
element_type: ElementType::Heading,
|
||||
@@ -477,7 +477,7 @@ impl DocumentElement {
|
||||
}
|
||||
|
||||
/// Get element text content.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn text(&self) -> String {
|
||||
match &self.data {
|
||||
ElementData::Paragraph { text } => text.clone(),
|
||||
@@ -492,19 +492,19 @@ impl DocumentElement {
|
||||
}
|
||||
|
||||
/// Get element character count.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn char_count(&self) -> usize {
|
||||
self.char_end.saturating_sub(self.char_start)
|
||||
}
|
||||
|
||||
/// Check if this is a table element.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn is_table(&self) -> bool {
|
||||
self.element_type == ElementType::Table
|
||||
}
|
||||
|
||||
/// Get table data if this is a table element.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn as_table(&self) -> Option<&StructuredTable> {
|
||||
match &self.data {
|
||||
ElementData::Table(t) => Some(t),
|
||||
@@ -540,7 +540,7 @@ pub struct StructuredDocument {
|
||||
|
||||
impl StructuredDocument {
|
||||
/// Create a new empty structured document.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
@@ -581,7 +581,7 @@ impl StructuredDocument {
|
||||
}
|
||||
|
||||
/// Check if document has any structure (vs plain text).
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn has_structure(&self) -> bool {
|
||||
self.table_count > 0 || self.code_block_count > 0
|
||||
}
|
||||
@@ -683,7 +683,7 @@ impl StructuredChunk {
|
||||
}
|
||||
|
||||
/// Check if this is a table-related chunk.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn is_table(&self) -> bool {
|
||||
matches!(
|
||||
self.chunk_type,
|
||||
@@ -692,13 +692,13 @@ impl StructuredChunk {
|
||||
}
|
||||
|
||||
/// Check if this is a continuation of a split element.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn is_continuation(&self) -> bool {
|
||||
self.part.is_some_and(|p| p > 1)
|
||||
}
|
||||
|
||||
/// Get character count.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn char_count(&self) -> usize {
|
||||
self.text.chars().count()
|
||||
}
|
||||
@@ -775,13 +775,13 @@ pub struct ChunkingResult {
|
||||
|
||||
impl ChunkingResult {
|
||||
/// Create empty result.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn empty() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
/// Total chunks produced.
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn chunk_count(&self) -> usize {
|
||||
self.chunks.len()
|
||||
}
|
||||
|
||||
+6
-5
@@ -9,6 +9,7 @@ fn vec_config() -> impl bincode::config::Config {
|
||||
.with_little_endian()
|
||||
}
|
||||
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
const VEC_DECODE_LIMIT: usize = crate::MAX_INDEX_BYTES as usize;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
@@ -23,7 +24,7 @@ pub struct VecIndexBuilder {
|
||||
}
|
||||
|
||||
impl VecIndexBuilder {
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
@@ -45,7 +46,7 @@ impl VecIndexBuilder {
|
||||
let dimension = self
|
||||
.documents
|
||||
.first()
|
||||
.map_or(0, |doc| doc.embedding.len() as u32);
|
||||
.map_or(0, |doc| u32::try_from(doc.embedding.len()).unwrap_or(0));
|
||||
#[cfg(feature = "parallel_segments")]
|
||||
let bytes_uncompressed = self
|
||||
.documents
|
||||
@@ -149,7 +150,7 @@ impl VecIndex {
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn search(&self, query: &[f32], limit: usize) -> Vec<VecSearchHit> {
|
||||
if query.is_empty() {
|
||||
return Vec::new();
|
||||
@@ -178,7 +179,7 @@ impl VecIndex {
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn entries(&self) -> Box<dyn Iterator<Item = (FrameId, &[f32])> + '_> {
|
||||
match self {
|
||||
VecIndex::Uncompressed { documents } => Box::new(
|
||||
@@ -193,7 +194,7 @@ impl VecIndex {
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn embedding_for(&self, frame_id: FrameId) -> Option<&[f32]> {
|
||||
match self {
|
||||
VecIndex::Uncompressed { documents } => documents
|
||||
|
||||
+16
-12
@@ -21,6 +21,7 @@ fn vec_config() -> impl bincode::config::Config {
|
||||
.with_little_endian()
|
||||
}
|
||||
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
const VEC_DECODE_LIMIT: usize = crate::MAX_INDEX_BYTES as usize;
|
||||
|
||||
/// Product Quantization parameters
|
||||
@@ -62,11 +63,15 @@ impl SubspaceCodebook {
|
||||
let mut best_dist = f32::INFINITY;
|
||||
|
||||
for i in 0..NUM_CENTROIDS {
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
let centroid = self.get_centroid(i as u8);
|
||||
let dist = l2_distance_squared(subspace, centroid);
|
||||
if dist < best_dist {
|
||||
best_dist = dist;
|
||||
best_idx = i as u8;
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
{
|
||||
best_idx = i as u8;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -87,9 +92,7 @@ impl ProductQuantizer {
|
||||
pub fn new(dimension: u32) -> Result<Self> {
|
||||
if dimension as usize != TOTAL_DIM {
|
||||
return Err(MemvidError::InvalidQuery {
|
||||
reason: format!(
|
||||
"PQ only supports {TOTAL_DIM}-dim vectors, got {dimension}"
|
||||
),
|
||||
reason: format!("PQ only supports {TOTAL_DIM}-dim vectors, got {dimension}"),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -136,6 +139,7 @@ impl ProductQuantizer {
|
||||
|
||||
// Store in codebook
|
||||
for (i, centroid) in centroids.iter().enumerate() {
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
self.codebooks[subspace_idx].set_centroid(i as u8, centroid);
|
||||
}
|
||||
}
|
||||
@@ -193,7 +197,7 @@ impl ProductQuantizer {
|
||||
|
||||
/// Compute asymmetric distance between query vector and PQ-encoded vector
|
||||
/// Uses precomputed lookup tables for efficiency
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn asymmetric_distance(&self, query: &[f32], codes: &[u8]) -> f32 {
|
||||
if query.len() != TOTAL_DIM || codes.len() != NUM_SUBSPACES {
|
||||
return f32::INFINITY;
|
||||
@@ -232,7 +236,7 @@ pub struct QuantizedVecIndexBuilder {
|
||||
}
|
||||
|
||||
impl QuantizedVecIndexBuilder {
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
@@ -339,7 +343,7 @@ impl QuantizedVecIndex {
|
||||
// Convert old format to new format
|
||||
let quantizer = ProductQuantizer {
|
||||
codebooks: old_quantizer.codebooks,
|
||||
dimension: (NUM_SUBSPACES * SUBSPACE_DIM) as u32,
|
||||
dimension: u32::try_from(NUM_SUBSPACES * SUBSPACE_DIM).unwrap_or(u32::MAX),
|
||||
};
|
||||
|
||||
Ok(Self {
|
||||
@@ -349,7 +353,7 @@ impl QuantizedVecIndex {
|
||||
}
|
||||
|
||||
/// Search using asymmetric distance computation
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn search(&self, query: &[f32], limit: usize) -> Vec<VecSearchHit> {
|
||||
if query.is_empty() {
|
||||
return Vec::new();
|
||||
@@ -382,7 +386,7 @@ impl QuantizedVecIndex {
|
||||
}
|
||||
|
||||
/// Get compression statistics
|
||||
#[must_use]
|
||||
#[must_use]
|
||||
pub fn compression_stats(&self) -> CompressionStats {
|
||||
let original_bytes = self.documents.len() * TOTAL_DIM * std::mem::size_of::<f32>();
|
||||
let compressed_bytes = self.documents.len() * NUM_SUBSPACES; // 96 bytes per vector
|
||||
@@ -556,7 +560,7 @@ mod tests {
|
||||
}
|
||||
|
||||
// Train quantizer
|
||||
let mut pq = ProductQuantizer::new(TOTAL_DIM as u32).unwrap();
|
||||
let mut pq = ProductQuantizer::new(u32::try_from(TOTAL_DIM).unwrap()).unwrap();
|
||||
pq.train(&training_vecs, 10).unwrap();
|
||||
|
||||
// Encode a vector
|
||||
@@ -588,7 +592,7 @@ mod tests {
|
||||
// Build index
|
||||
let mut builder = QuantizedVecIndexBuilder::new();
|
||||
builder
|
||||
.train_quantizer(&training_vecs, TOTAL_DIM as u32)
|
||||
.train_quantizer(&training_vecs, u32::try_from(TOTAL_DIM).unwrap())
|
||||
.unwrap();
|
||||
|
||||
for (i, vec) in training_vecs.iter().take(10).enumerate() {
|
||||
@@ -599,7 +603,7 @@ mod tests {
|
||||
|
||||
let artifact = builder.finish().unwrap();
|
||||
assert_eq!(artifact.vector_count, 10);
|
||||
assert_eq!(artifact.dimension, TOTAL_DIM as u32);
|
||||
assert_eq!(artifact.dimension, u32::try_from(TOTAL_DIM).unwrap());
|
||||
assert!(artifact.compression_ratio > 10.0);
|
||||
|
||||
// Decode and search
|
||||
|
||||
@@ -60,7 +60,7 @@ fn doctor_rebuilds_tantivy_index() {
|
||||
.unwrap();
|
||||
|
||||
assert!(
|
||||
results.hits.len() > 0,
|
||||
!results.hits.is_empty(),
|
||||
"Search should return results before doctor"
|
||||
);
|
||||
assert!(results.total_hits >= 10, "Should have at least 10 hits");
|
||||
@@ -106,7 +106,7 @@ fn doctor_rebuilds_tantivy_index() {
|
||||
.unwrap();
|
||||
|
||||
assert!(
|
||||
results.hits.len() > 0,
|
||||
!results.hits.is_empty(),
|
||||
"Search should return results after doctor rebuild"
|
||||
);
|
||||
assert_eq!(
|
||||
@@ -250,7 +250,7 @@ fn open_file_with_tantivy_segments_enables_lex() {
|
||||
);
|
||||
|
||||
let results = result.unwrap();
|
||||
assert!(results.hits.len() > 0, "Should find the test document");
|
||||
assert!(!results.hits.is_empty(), "Should find the test document");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -389,8 +389,9 @@ fn doctor_recovers_corrupted_wal() {
|
||||
let end = start + header.wal_size.min(100) as usize;
|
||||
|
||||
// corrupt some bytes
|
||||
#[allow(clippy::needless_range_loop)]
|
||||
for i in start..end {
|
||||
bytes[i] = 0xFF;
|
||||
bytes[i] = 0xFF; // Corrupt bytes
|
||||
}
|
||||
|
||||
write(&mv2_path, &bytes).unwrap();
|
||||
|
||||
+2
-2
@@ -52,9 +52,9 @@ fn create_handles_existing_file() {
|
||||
// Create second time - this tests current behavior
|
||||
// (Either it fails OR it creates a new file - both are valid implementations)
|
||||
let result = Memvid::create(&path);
|
||||
if result.is_ok() {
|
||||
if let Ok(mut mem) = result {
|
||||
// If create succeeds, the old data should be gone (new file)
|
||||
let mut mem = result.unwrap();
|
||||
// let mut mem = result.unwrap();
|
||||
mem.commit().unwrap();
|
||||
// Reopen and verify it's empty (new file was created)
|
||||
let mem = Memvid::open_read_only(&path).unwrap();
|
||||
|
||||
+3
-2
@@ -22,11 +22,12 @@ fn put_bytes_basic() {
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let frame_id = mem.put_bytes_with_options(b"Hello, World!", opts).unwrap();
|
||||
let _frame_id = mem.put_bytes_with_options(b"Hello, World!", opts).unwrap();
|
||||
mem.commit().unwrap();
|
||||
|
||||
// Verify frame was created
|
||||
assert!(frame_id > 0 || frame_id == 0, "Frame ID should be valid");
|
||||
// Verify frame was created (FrameId is u64 so >= 0 is implied)
|
||||
// assert!(frame_id >= 0);
|
||||
|
||||
let mem = Memvid::open_read_only(&path).unwrap();
|
||||
assert_eq!(mem.stats().unwrap().frame_count, 1, "Should have 1 frame");
|
||||
|
||||
+2
-2
@@ -79,7 +79,7 @@ fn search_basic_query() {
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
assert!(results.hits.len() > 0, "Should find quantum document");
|
||||
assert!(!results.hits.is_empty(), "Should find quantum document");
|
||||
assert!(
|
||||
results.hits[0].uri.contains("quantum"),
|
||||
"Top result should be quantum physics"
|
||||
@@ -232,7 +232,7 @@ fn search_returns_snippets() {
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
assert!(results.hits.len() > 0);
|
||||
assert!(!results.hits.is_empty());
|
||||
let hit = &results.hits[0];
|
||||
|
||||
// Snippet should contain matched content
|
||||
|
||||
Reference in New Issue
Block a user