fix: Correctly classify all the binaries files

closes https://github.com/dmtrKovalenko/fff/issues/546 Removes all the heuristics across all the binary size detection, now we scan every single byte up to content searchable cap of fff to detct if the file is not a text + some fff lua size guard
2026-06-01 20:06:10 -07:00
15 changed files with 717 additions and 78 deletions
@@ -1,5 +1,6 @@
+use crate::constants::MAX_OVERFLOW_FILES;
 use crate::error::Error;
-use crate::file_picker::{FFFMode, MAX_OVERFLOW_FILES};
+use crate::file_picker::FFFMode;
 use crate::git::GitStatusCache;
 use crate::shared::{SharedFilePicker, SharedFrecency};
 use crate::sort_buffer::sort_with_buffer;
@@ -25,7 +26,6 @@ pub struct BackgroundWatcher {
 }

 const DEBOUNCE_TIMEOUT: Duration = Duration::from_millis(50);
-const MAX_PATHS_THRESHOLD: usize = 1024;
 /// On macOS, each `watch()` call creates a separate FSEventStream. When the
 /// number of directories exceeds this threshold we fall back to a single
 /// recursive watch to avoid exhausting the per-process stream limit.
@@ -497,10 +497,11 @@ fn handle_debounced_events(
        }

        affected_paths_count += debounced_event.event.paths.len();
-        if affected_paths_count > MAX_PATHS_THRESHOLD {
+        if affected_paths_count > MAX_OVERFLOW_FILES {
            warn!(
-                "Too many affected paths ({}) in a single batch, triggering full rescan",
-                affected_paths_count
+                ?affected_paths_count,
+                max = MAX_OVERFLOW_FILES,
+                "Too many affected paths in a single batch, triggering full rescan",
            );

            need_full_rescan = true;
@@ -1,3 +1,4 @@
+use crate::constants::MAX_INDEXABLE_FILE_SIZE;
 use ahash::AHashMap;
 use rayon::iter::{IndexedParallelIterator, ParallelIterator};
 use rayon::slice::ParallelSlice;
@@ -5,7 +6,7 @@ use std::cell::UnsafeCell;
 use std::sync::OnceLock;
 use std::sync::atomic::{AtomicU16, AtomicUsize, Ordering};

-use crate::FileItem;
+use crate::{FileItem, constants};

 /// Maximum number of distinct bigrams tracked in the inverted index.
 /// 95 printable ASCII chars (32..=126) after lowercasing → ~70 distinct → 4900 possible.
@@ -592,7 +593,6 @@ impl BigramOverlay {
    }
 }

-pub(crate) const MAX_INDEXABLE_FILE_SIZE: usize = 2 * 1024 * 1024;
 const BIGRAM_CHUNK_FILES: usize = 4 * 64;

 /// Sparse-column cutoff for the skip-1 sub-index. Rare skip columns add
@@ -680,7 +680,7 @@ pub(crate) fn build_bigram_index(
                            // an invalid text sequence if this is not a binary file.
                            //
                            // Need to find a better way to do this.
-                            file.set_binary(crate::file_picker::detect_binary_content(content));
+                            file.set_binary(crate::types::detect_binary_content(content));

                            builder.add_file_content(&skip_builder, file_idx, content);
                        }
@@ -706,6 +706,28 @@ pub(crate) fn build_bigram_index(
    index
 }

+#[tracing::instrument(skip_all, name = "Sniffing Large Files Binary", level = tracing::Level::DEBUG)]
+pub(crate) fn sniff_binary_for_non_indexable(
+    files: &[FileItem],
+    base_path: &std::path::Path,
+    arena: crate::simd_path::ArenaPtr,
+) {
+    // Non-indexable files are few in a typical repo, so a serial pass with a
+    // single reused chunk buffer beats spinning up the thread pool.
+    let mut path_buf = [0u8; crate::simd_path::PATH_BUF_SIZE];
+    let mut chunk = vec![0u8; crate::types::BINARY_CLASSIFICATION_CHUNK_SIZE];
+
+    for file in files {
+        // check only the files that we are able to grep
+        if file.size == 0 || file.size > constants::MAX_FFFILE_SIZE {
+            continue;
+        }
+
+        let abs = file.write_absolute_path(arena, base_path, &mut path_buf);
+        file.detect_binary_per_byte(abs, &mut chunk);
+    }
+}
+
 /// Open the base directory for the `openat` fast path. Returns `-1` on
 /// failure — callers interpret a negative fd as "fall back to absolute
 /// paths".
@@ -0,0 +1,39 @@
+/// Largest file whose full content fff will touch: the default grep read cap
+/// (`GrepSearchOptions::max_file_size`) and the content-cache mmap cap
+/// (`ContentCacheBudget::max_file_size`). Binary detection also streams up to
+/// this far so nothing grep would read is left unclassified.
+pub const MAX_FFFILE_SIZE: u64 = 10 * 1024 * 1024;
+
+/// Upper bound on a file the bigram builder will build, if the file is very large there is a
+/// big probability it will only bloat the available bigrams and will anyway pop ut from the prefilter
+pub const MAX_INDEXABLE_FILE_SIZE: usize = 2 * 1024 * 1024;
+
+/// Total bytes the persistent content mmap cache may hold for a small repo.
+pub const MAX_CACHED_CONTENT_BYTES: u64 = 512 * 1024 * 1024;
+
+/// Files below one page waste the remainder when mmapped, so the cache skips
+/// them and falls back to chunked reads. Unused on Windows (no content cache).
+#[cfg(all(not(target_os = "windows"), target_arch = "aarch64"))]
+pub const MMAP_THRESHOLD: u64 = 16 * 1024;
+#[cfg(all(not(target_os = "windows"), not(target_arch = "aarch64")))]
+pub const MMAP_THRESHOLD: u64 = 4 * 1024;
+
+/// Capacity reserved for files the watcher discovers after the initial scan;
+/// exceeding it forces a full rescan.
+pub const MAX_OVERFLOW_FILES: usize = 1024;
+
+/// Fresh-mmap threshold: files at or above this size get mmapped directly on
+/// cache miss instead of chunked reads into Vec. Empirically tuned per-platform.
+/// Only referenced on Unix; Windows uses the `std::fs::read` fallback so this
+/// constant is gated to non-Windows targets to keep `-D unused-imports` happy.
+#[cfg(target_os = "macos")]
+pub const FRESH_MMAP_THRESHOLD: u64 = 1024 * 1024;
+#[cfg(all(not(target_os = "macos"), not(target_os = "windows")))]
+pub const FRESH_MMAP_THRESHOLD: u64 = 256 * 1024;
+
+// we do not support 32kb path limit on windows
+#[cfg(target_os = "windows")]
+pub const PATH_BUF_SIZE: usize = 4096;
+
+#[cfg(not(target_os = "windows"))]
+pub const PATH_BUF_SIZE: usize = libc::PATH_MAX as usize;
@@ -33,6 +33,7 @@
 use crate::FFFStringStorage;
 use crate::background_watcher::{BackgroundWatcher, is_git_file};
 use crate::bigram_filter::{BigramFilter, BigramOverlay};
+use crate::constants::{MAX_OVERFLOW_FILES, PATH_BUF_SIZE};
 use crate::error::Error;
 use crate::frecency::FrecencyTracker;
 use crate::git::GitStatusCache;
@@ -42,7 +43,7 @@ use crate::query_tracker::QueryTracker;
 use crate::scan::{ScanConfig, ScanJob, ScanSignals};
 use crate::score::fuzzy_match_and_score_files;
 use crate::shared::{SharedFilePicker, SharedFrecency};
-use crate::simd_path::{ArenaPtr, PATH_BUF_SIZE};
+use crate::simd_path::ArenaPtr;
 use crate::stable_vec::StableVec;
 use crate::types::{
    ContentCacheBudget, DirItem, DirSearchResult, FileItem, MixedItemRef, MixedSearchResult,
@@ -62,11 +63,6 @@ use std::thread::JoinHandle;
 use std::time::SystemTime;
 use tracing::{Level, debug, error, info, warn};

-/// Max overflow files before the watcher triggers a full rescan.
-/// `walk_filesystem` reserves this much extra capacity so the Vec never
-/// reallocates while raw pointers are held during post-scan.
-pub(crate) const MAX_OVERFLOW_FILES: usize = 1024;
-
 /// Dedicated thread pool for background work (scan, warmup, bigram build).
 /// Uses fewer threads than the global rayon pool so Neovim's event loop
 /// and search queries can still get CPU time.
@@ -812,8 +808,6 @@ impl FilePicker {

        self.sync_data = sync;

-        // Recalculate cache budget based on actual file count (unless
-        // the caller provided an explicit budget via FilePickerOptions).
        if !self.has_explicit_cache_budget {
            let file_count = self.sync_data.files().len();
            self.cache_budget = Arc::new(ContentCacheBudget::new_for_repo(file_count));
@@ -821,14 +815,18 @@ impl FilePicker {
            self.cache_budget.reset();
        }

-        // Apply git status synchronously.
        if let Some(handle) = git_handle
            && let Ok(Some(git_cache)) = handle.join()
        {
+            let mut path_buf = [0u8; crate::simd_path::PATH_BUF_SIZE];
+
            let arena = self.arena_base_ptr();
            for file in self.sync_data.files.iter_mut() {
-                file.git_status =
-                    git_cache.lookup_status(&file.absolute_path(arena, &self.base_path));
+                file.git_status = git_cache.lookup_status(file.write_absolute_path(
+                    arena,
+                    &self.base_path,
+                    &mut path_buf,
+                ));
            }
        }

@@ -864,6 +862,7 @@ impl FilePicker {
    /// The query should be parsed using [`FFFQuery`]::parse() before calling
    /// this function. If a [`QueryTracker`] is provided, the search will
    /// automatically look up the last selected file for this query and boost it
+    #[tracing::instrument(skip_all, name = "Fuzzy file search", fields(query = query.raw_query))]
    pub fn fuzzy_search<'q>(
        &self,
        query: &'q FFFQuery<'q>,
@@ -1273,9 +1272,9 @@ impl FilePicker {
            base_count: self.sync_data.base_count,
            indexable_count: self.sync_data.indexable_count,
            base_path: self.base_path.clone(),
-            budget: Arc::clone(&self.cache_budget),
            cancelled: Arc::clone(&self.signals.cancelled),
            post_scan_flag: Arc::clone(&self.signals.post_scan_indexing_active),
+            _budget: Arc::clone(&self.cache_budget),
        })
    }

@@ -1421,7 +1420,14 @@ impl FilePicker {

        file.update_metadata(&self.cache_budget, modified_time, Some(size));

-        // only base-region entries participate in the bigram overlay
+        // Re-classify binary status from current content (chunked, fixed
+        // buffer). Already-binary files are left alone.
+        if !file.is_binary() {
+            let mut chunk = [0u8; crate::types::BINARY_CLASSIFICATION_CHUNK_SIZE];
+            file.detect_binary_per_byte(path, &mut chunk);
+        }
+
+        // Indexable base-region files feed fresh content to the bigram overlay.
        if matches!(slot, FileSlot::Base(_))
            && let Some(ref overlay) = overlay
        {
@@ -1451,12 +1457,10 @@ impl FilePicker {
        } else if let Ok(c) = crate::path_utils::canonicalize(path) {
            Some(c)
        } else {
-            let parent = path.parent()?;
-            let file_name = path.file_name()?;
-            let mut p = crate::path_utils::canonicalize(parent).ok()?;
-            p.push(file_name);
-            Some(p)
+            tracing::error!(path = ?path.display(), "Failed to canonicalize file path to add");
+            return None;
        };
+
        #[cfg(windows)]
        let path_for_index: &Path = canonical_buf.as_deref().unwrap_or(path);
        #[cfg(not(windows))]
@@ -1465,13 +1469,20 @@ impl FilePicker {
        let (mut file_item, rel_path) =
            FileItem::new(path_for_index.to_path_buf(), &self.base_path, None);

+        // we have to perform manual classification for every new file this will be
+        // batched during the scan, this is the path when the file is ad-hoc added to the sync
+        file_item.detect_binary_per_byte(
+            path_for_index,
+            // inline chunk buf
+            &mut [0u8; crate::types::BINARY_CLASSIFICATION_CHUNK_SIZE],
+        );
+
        let builder = self.sync_data.overflow_builder.get_or_insert_with(|| {
            // we know that overflow would never create more files during the file
            crate::simd_path::ChunkedPathStoreBuilder::new(MAX_OVERFLOW_FILES)
        });

-        let chunked_path = builder.add_file_immediate(&rel_path, file_item.path.filename_offset);
-        file_item.set_path(chunked_path);
+        file_item.set_path(builder.add_file_immediate(&rel_path, file_item.path.filename_offset));
        file_item.set_overflow(true);

        if !self.sync_data.files.push(file_item) {
@@ -1658,7 +1669,8 @@ pub(crate) struct PostScanUnsafeSnapshot {
    pub files: StableVec<FileItem>,
    pub dirs: StableVec<crate::types::DirItem>,
    pub arena: Option<Arc<crate::simd_path::ChunkedPathStore>>,
-    pub budget: Arc<crate::types::ContentCacheBudget>,
+    // TODO figure this out
+    pub _budget: Arc<crate::types::ContentCacheBudget>,
    pub base_count: usize,
    pub indexable_count: usize,
    pub base_path: PathBuf,
@@ -1847,7 +1859,7 @@ impl FileSync {
        let is_indexable = |f: &FileItem| {
            !f.is_binary()
                && f.size > 0
-                && f.size <= crate::bigram_filter::MAX_INDEXABLE_FILE_SIZE as u64
+                && f.size <= crate::constants::MAX_INDEXABLE_FILE_SIZE as u64
        };

        BACKGROUND_THREAD_POOL.install(|| {
@@ -2034,11 +2046,6 @@ pub fn is_known_binary_extension(path: &Path) -> bool {
    )
 }

-#[inline]
-pub(crate) fn detect_binary_content(content: &[u8]) -> bool {
-    memchr::memchr(0, content).is_some()
-}
-
 /// Length of the longest shared directory prefix of two relative dir
 /// paths (without a trailing separator), measured as the number of bytes
 /// up to and including the last shared separator — plus the full shorter
@@ -333,6 +333,8 @@ pub struct GrepResult<'a> {
    pub regex_fallback_error: Option<String>,
 }

+pub use crate::constants::MAX_FFFILE_SIZE;
+
 /// Options for grep search.
 #[derive(Debug, Clone)]
 pub struct GrepSearchOptions {
@@ -371,7 +373,7 @@ pub struct GrepSearchOptions {
 impl Default for GrepSearchOptions {
    fn default() -> Self {
        Self {
-            max_file_size: 10 * 1024 * 1024,
+            max_file_size: MAX_FFFILE_SIZE,
            max_matches_per_file: 200,
            smart_case: true,
            file_offset: 0,
@@ -1243,16 +1245,16 @@ where
    for chunk in files_to_search.chunks(chunk_size) {
        let chunk_offset = files_consumed;

-        // Parallel phase: search all files in this chunk concurrently.
-        // Within a chunk every file is visited (no gaps), so pagination
-        // offsets remain correct across chunk boundaries.
        let chunk_results: Vec<(usize, &'a FileItem, Vec<GrepMatch>)> = chunk
            .par_iter()
            .enumerate()
            .map_init(
                // Per-thread scratch: a reusable read buffer for small files
                // and an mmap slot for cache-miss large files (≥ FRESH_MMAP_THRESHOLD).
-                || (Vec::with_capacity(64 * 1024), MmapSlot::default()),
+                || {
+                    tracing::info!("LMAOTHREAD");
+                    (Vec::with_capacity(64 * 1024), MmapSlot::default())
+                },
                |(buf, mmap_slot), (local_idx, file)| {
                    if ctx.abort_signal.load(Ordering::Relaxed) {
                        budget_exceeded.store(true, Ordering::Relaxed);
@@ -2433,7 +2435,7 @@ mod tests {
        let arena = picker.arena_base_ptr();

        let options = super::GrepSearchOptions {
-            max_file_size: 10 * 1024 * 1024,
+            max_file_size: MAX_FFFILE_SIZE,
            max_matches_per_file: 0,
            smart_case: true,
            file_offset: 0,
@@ -2617,7 +2619,7 @@ mod tests {
        // (a, b, c in base + f, g, h in overflow).
        let query = super::parse_grep_query("unicorn");
        let options = super::GrepSearchOptions {
-            max_file_size: 10 * 1024 * 1024,
+            max_file_size: MAX_FFFILE_SIZE,
            max_matches_per_file: 0,
            smart_case: true,
            file_offset: 0,
@@ -98,6 +98,7 @@ mod scan;
 #[doc(hidden)]
 pub mod bigram_filter;
 pub mod bigram_query;
+pub mod constants;
 mod constraints;
 mod error;
 mod score;
@@ -7,7 +7,7 @@ use tracing::{error, info};

 use crate::FileSync;
 use crate::background_watcher::BackgroundWatcher;
-use crate::bigram_filter::build_bigram_index;
+use crate::bigram_filter::{build_bigram_index, sniff_binary_for_non_indexable};
 use crate::error::Error;
 use crate::file_picker::{BACKGROUND_THREAD_POOL, FFFMode};
 use crate::git::GitStatusCache;
@@ -211,8 +211,9 @@ impl ScanJob {

        // 3. Post-scan warmup + bigram build — runs in parallel with the
        // git-status thread to overlap the two expensive phases.
-        if (config.warmup || config.content_indexing)
-            && !signals.cancelled.load(Ordering::Acquire)
+        // Always runs (even with both flags off) so binary-content files
+        // with unknown extensions get reclassified before user search hits.
+        if !signals.cancelled.load(Ordering::Acquire)
            && let Some(snap) = snapshot.as_ref()
        {
            Self::run_post_scan(&shared_picker, &signals, &config, snap);
@@ -289,20 +290,23 @@ impl ScanJob {
        config: &ScanConfig,
        unsafe_snapshot: &crate::file_picker::PostScanUnsafeSnapshot,
    ) {
-        let arena = unsafe_snapshot
-            .arena
+        let Some(arena) = unsafe_snapshot
+            .arena // we are never touching overlays so this arena is always correct
            .as_ref()
            .map(|s| s.as_arena_ptr())
-            .unwrap_or(ArenaPtr::null());
-        let _budget: &ContentCacheBudget = &unsafe_snapshot.budget;
-        let files: &[crate::types::FileItem] = &unsafe_snapshot.files[..unsafe_snapshot.base_count];
+        else {
+            tracing::error!("Failed to run post scan: arena is invalid");
+            return;
+        };

+        let files: &[crate::types::FileItem] = &unsafe_snapshot.files[..unsafe_snapshot.base_count];
        if signals.cancelled.load(Ordering::Acquire) {
            return;
        }

        if config.content_indexing {
-            let indexable_files = &files[..unsafe_snapshot.indexable_count.min(files.len())];
+            let indexable_count = unsafe_snapshot.indexable_count.min(files.len());
+            let (indexable_files, non_indexable_files) = files.split_at(indexable_count);
            let index = build_bigram_index(indexable_files, &unsafe_snapshot.base_path, arena);

            if let Ok(mut guard) = shared_picker.write()
@@ -310,9 +314,23 @@ impl ScanJob {
            {
                picker.set_bigram_index(index);
            }
+
+            // Bigram only sniffs files <= MAX_INDEXABLE_FILE_SIZE; large
+            // unknown-extension binaries slip past it and would otherwise be
+            // grep-able as text. Cheap header sniff catches those.
+            if !signals.cancelled.load(Ordering::Acquire) {
+                sniff_binary_for_non_indexable(
+                    non_indexable_files,
+                    &unsafe_snapshot.base_path,
+                    arena,
+                );
+            }
+        } else {
+            // this potentially a long running as we are not parallelizing it but it's okay
+            sniff_binary_for_non_indexable(files, &unsafe_snapshot.base_path, arena);
        }

-        // Skipped as potentially unsafe - figure this out later
+        // TODO Skipped as potentially unsafe - figure this out later
        // if config.warmup && !signals.cancelled.load(Ordering::Acquire) {
        //     warmup_mmaps(files, budget, &unsafe_snapshot.base_path, arena);
        // }
@@ -60,7 +60,7 @@ impl std::fmt::Debug for SimdChunk {
    }
 }

-pub const PATH_BUF_SIZE: usize = 4096;
+pub use crate::constants::PATH_BUF_SIZE;

 /// Indices into a shared `SimdChunk` arena representing a file path.
 ///
@@ -4,9 +4,12 @@ use std::path::{Path, PathBuf};
 use std::sync::OnceLock;
 use std::sync::atomic::{AtomicI32, AtomicU8, AtomicU64, AtomicUsize, Ordering};

+#[cfg(not(target_os = "windows"))]
+use crate::constants::{FRESH_MMAP_THRESHOLD, MMAP_THRESHOLD};
+use crate::constants::{MAX_CACHED_CONTENT_BYTES, MAX_FFFILE_SIZE, PATH_BUF_SIZE};
 use crate::constraints::Constrainable;
 use crate::query_tracker::QueryMatchEntry;
-use crate::simd_path::{ArenaPtr, PATH_BUF_SIZE};
+use crate::simd_path::ArenaPtr;
 use fff_query_parser::{FFFQuery, FuzzyQuery, Location};

 /// Different sources of the string storage used by FFF
@@ -237,6 +240,18 @@ impl Clone for FileItem {
    }
 }

+/// Single-block read used by the binary classifier. Most binaries reveal a
+/// NUL byte within the first filesystem block, so 16 KB lets one read settle
+/// the classification for typical files while keeping the scratch buffer
+/// small enough to live on the stack.
+pub const BINARY_CLASSIFICATION_CHUNK_SIZE: usize = 16 * 1024;
+
+/// A file is treated as binary if any NUL byte appears in the scanned prefix.
+#[inline]
+pub(crate) fn detect_binary_content(content: &[u8]) -> bool {
+    memchr::memchr(0, content).is_some()
+}
+
 impl FileItem {
    pub fn new_raw(
        filename_start: u16,
@@ -499,6 +514,38 @@ impl FileItem {
        }
    }

+    /// Chunked classifier of the binary content of the file chunk by chunk
+    /// accepts path which to reuse the allocated buffer for absolute path read
+    pub(crate) fn detect_binary_per_byte(&self, path: &Path, chunk: &mut [u8]) {
+        if self.size == 0 {
+            return;
+        }
+
+        let Ok(mut file) = std::fs::OpenOptions::new()
+            .write(false)
+            .read(true)
+            .open(path)
+        else {
+            tracing::error!(path = ?path.display(), "Failed to open indexed file");
+            return;
+        };
+
+        loop {
+            match file.read(chunk) {
+                Ok(0) => break,
+                Err(e) => {
+                    tracing::error!(?e, "Failed to read file chunk");
+                    break;
+                }
+                Ok(n) => {
+                    if detect_binary_content(&chunk[..n]) {
+                        self.set_binary(true);
+                    }
+                }
+            }
+        }
+    }
+
    #[inline]
    pub fn is_deleted(&self) -> bool {
        self.flags.load(Ordering::Relaxed) & FileItemFlags::DELETED != 0
@@ -686,22 +733,6 @@ impl FileItem {
    }
 }

-#[cfg(all(not(target_os = "windows"), target_arch = "aarch64"))]
-const MMAP_THRESHOLD: u64 = 16 * 1024;
-#[cfg(all(not(target_os = "windows"), not(target_arch = "aarch64")))]
-const MMAP_THRESHOLD: u64 = 4 * 1024;
-
-// these are empirically set values for the benchmarks. Theory is simple:
-// the larger the file is - the more syscalls needed to read the file, so at some
-// point it becomes better strategy to mmap file and process instead of doing chunking
-#[cfg(target_os = "macos")]
-pub(crate) const FRESH_MMAP_THRESHOLD: u64 = 1024 * 1024;
-#[cfg(target_os = "windows")]
-#[allow(dead_code)]
-pub(crate) const FRESH_MMAP_THRESHOLD: u64 = 0;
-#[cfg(not(any(target_os = "macos", target_os = "windows")))]
-pub(crate) const FRESH_MMAP_THRESHOLD: u64 = 256 * 1024;
-
 /// Per-thread scratch slot owning a transient mmap returned from
 /// [`FileItem::get_content_for_search`]. `Option<Mmap>` on Unix,
 /// unit on Windows where mmap is unused.
@@ -825,10 +856,6 @@ impl Default for MixedItemRef<'_> {
    }
 }

-const MAX_MMAP_FILE_SIZE: u64 = 10 * 1024 * 1024;
-
-const MAX_CACHED_CONTENT_BYTES: u64 = 512 * 1024 * 1024;
-
 #[derive(Debug)]
 pub struct ContentCacheBudget {
    pub max_files: usize,
@@ -843,7 +870,7 @@ impl ContentCacheBudget {
        Self {
            max_files: usize::MAX,
            max_bytes: u64::MAX,
-            max_file_size: MAX_MMAP_FILE_SIZE,
+            max_file_size: MAX_FFFILE_SIZE,
            cached_count: AtomicUsize::new(0),
            cached_bytes: AtomicU64::new(0),
        }
@@ -885,7 +912,7 @@ impl ContentCacheBudget {
        Self {
            max_files,
            max_bytes,
-            max_file_size: MAX_MMAP_FILE_SIZE,
+            max_file_size: MAX_FFFILE_SIZE,
            cached_count: AtomicUsize::new(0),
            cached_bytes: AtomicU64::new(0),
        }
@@ -390,6 +390,357 @@ fn binary_payload_after_long_ascii_header_is_detected() {
    );
 }

+#[test]
+fn unknown_extension_binary_added_after_scan_is_reclassified() {
+    // The initial-scan path runs detect_binary_content as part of bigram build,
+    // but the watcher path used to fall back to extension-only triage and
+    // missed binary files with unknown extensions like `.codex`.
+    use fff_search::file_picker::FFFMode;
+    use fff_search::{SharedFilePicker, SharedFrecency};
+    use std::time::Duration;
+
+    let tmp = TempDir::new().unwrap();
+    let base = tmp.path();
+
+    // Seed one tracked text file so the initial scan has something to work with.
+    fs::write(base.join("seed.txt"), b"seed\n").unwrap();
+
+    let shared_picker = SharedFilePicker::default();
+    let shared_frecency = SharedFrecency::default();
+
+    FilePicker::new_with_shared_state(
+        shared_picker.clone(),
+        shared_frecency.clone(),
+        FilePickerOptions {
+            base_path: base.to_string_lossy().to_string(),
+            enable_mmap_cache: false,
+            enable_content_indexing: true,
+            mode: FFFMode::Neovim,
+            watch: false,
+            ..Default::default()
+        },
+    )
+    .expect("Failed to create FilePicker");
+
+    let deadline = std::time::Instant::now() + Duration::from_secs(10);
+    loop {
+        std::thread::sleep(Duration::from_millis(25));
+        let ready = shared_picker
+            .read()
+            .ok()
+            .and_then(|g| {
+                g.as_ref()
+                    .map(|p| !p.is_scan_active() && p.bigram_index().is_some())
+            })
+            .unwrap_or(false);
+        if ready {
+            break;
+        }
+        assert!(
+            std::time::Instant::now() < deadline,
+            "Timed out waiting for bigram build"
+        );
+    }
+
+    // Drop the file on disk after indexing finished, then announce it through
+    // the watcher entry point. `.codex` is intentionally not in the extension
+    // allow-list — only a content sniff can flag it.
+    let mut payload = vec![0x03u8, 0x00, 0x04, 0x05];
+    payload.extend(std::iter::repeat_n(0u8, 256));
+    let new_path = base.join("snapshot.codex");
+    fs::write(&new_path, &payload).unwrap();
+
+    {
+        let mut guard = shared_picker.write().unwrap();
+        let picker = guard.as_mut().unwrap();
+        assert!(
+            picker.handle_create_or_modify(&new_path).is_some(),
+            "handle_create_or_modify must accept the new file"
+        );
+    }
+
+    let guard = shared_picker.read().unwrap();
+    let picker = guard.as_ref().unwrap();
+    let was_flagged = picker
+        .get_files()
+        .iter()
+        .any(|f| f.relative_path(picker).contains("snapshot.codex") && f.is_binary());
+    assert!(
+        was_flagged,
+        "snapshot.codex must be flagged binary when added via the watcher path"
+    );
+}
+
+#[test]
+fn text_file_modified_to_binary_is_reclassified() {
+    // A file that started life as text and later got rewritten with NUL bytes
+    // (e.g. a generator overwrote a .log) must lose its text classification.
+    use fff_search::file_picker::FFFMode;
+    use fff_search::{SharedFilePicker, SharedFrecency};
+    use std::time::Duration;
+
+    let tmp = TempDir::new().unwrap();
+    let base = tmp.path();
+
+    // Start as plain text with a known extension.
+    fs::write(base.join("notes.txt"), b"hello world\n").unwrap();
+
+    let shared_picker = SharedFilePicker::default();
+    let shared_frecency = SharedFrecency::default();
+
+    FilePicker::new_with_shared_state(
+        shared_picker.clone(),
+        shared_frecency.clone(),
+        FilePickerOptions {
+            base_path: base.to_string_lossy().to_string(),
+            enable_mmap_cache: false,
+            enable_content_indexing: true,
+            mode: FFFMode::Neovim,
+            watch: false,
+            ..Default::default()
+        },
+    )
+    .expect("Failed to create FilePicker");
+
+    let deadline = std::time::Instant::now() + Duration::from_secs(10);
+    loop {
+        std::thread::sleep(Duration::from_millis(25));
+        let ready = shared_picker
+            .read()
+            .ok()
+            .and_then(|g| {
+                g.as_ref()
+                    .map(|p| !p.is_scan_active() && p.bigram_index().is_some())
+            })
+            .unwrap_or(false);
+        if ready {
+            break;
+        }
+        assert!(
+            std::time::Instant::now() < deadline,
+            "Timed out waiting for bigram build"
+        );
+    }
+
+    // Sanity: it's text right now.
+    {
+        let guard = shared_picker.read().unwrap();
+        let picker = guard.as_ref().unwrap();
+        let is_text = picker
+            .get_files()
+            .iter()
+            .any(|f| f.relative_path(picker).contains("notes.txt") && !f.is_binary());
+        assert!(is_text, "notes.txt should start as text");
+    }
+
+    // Overwrite with binary content and replay through the watcher entry point.
+    // Bump mtime so update_metadata records it as a real change.
+    std::thread::sleep(Duration::from_secs(1));
+    let mut payload = b"header text\n".to_vec();
+    payload.extend(std::iter::repeat_n(0u8, 256));
+    fs::write(base.join("notes.txt"), &payload).unwrap();
+
+    {
+        let mut guard = shared_picker.write().unwrap();
+        let picker = guard.as_mut().unwrap();
+        assert!(
+            picker
+                .handle_create_or_modify(base.join("notes.txt"))
+                .is_some(),
+            "handle_create_or_modify must succeed for the modify case"
+        );
+    }
+
+    let guard = shared_picker.read().unwrap();
+    let picker = guard.as_ref().unwrap();
+    let now_binary = picker
+        .get_files()
+        .iter()
+        .any(|f| f.relative_path(picker).contains("notes.txt") && f.is_binary());
+    assert!(
+        now_binary,
+        "notes.txt must flip to binary after being overwritten with NULs"
+    );
+}
+
+#[test]
+fn large_unknown_extension_binary_is_classified_at_scan_time() {
+    // Files larger than MAX_INDEXABLE_FILE_SIZE never enter build_bigram_index,
+    // so without a separate header sniff they default to is_binary=false and
+    // pollute grep results with NUL-laden lines (e.g. a committed ELF blob
+    // named `codex_view` with no extension).
+    use fff_search::file_picker::FFFMode;
+    use fff_search::grep::{GrepSearchOptions, parse_grep_query};
+    use fff_search::{SharedFilePicker, SharedFrecency};
+    use std::time::Duration;
+
+    let tmp = TempDir::new().unwrap();
+    let base = tmp.path();
+
+    // 3 MiB: above the 2 MiB bigram cap and below the 10 MiB grep cap.
+    // ELF-like header with NULs at the very start, then ASCII filler so a
+    // grep for "match this text" would otherwise return polluted lines.
+    let mut blob = Vec::new();
+    blob.extend_from_slice(b"\x7fELF\x02\x01\x01\x00");
+    blob.extend(std::iter::repeat_n(0u8, 256));
+    blob.extend_from_slice(b"\nmatch this text\n");
+    blob.extend(std::iter::repeat_n(b'A', 3 * 1024 * 1024));
+    blob.extend_from_slice(b"\nmatch this text\n");
+    fs::write(base.join("codex_view"), &blob).unwrap();
+    fs::write(base.join("plain.txt"), b"match this text\n").unwrap();
+
+    let shared_picker = SharedFilePicker::default();
+    let shared_frecency = SharedFrecency::default();
+
+    FilePicker::new_with_shared_state(
+        shared_picker.clone(),
+        shared_frecency.clone(),
+        FilePickerOptions {
+            base_path: base.to_string_lossy().to_string(),
+            enable_mmap_cache: false,
+            enable_content_indexing: true,
+            mode: FFFMode::Neovim,
+            watch: false,
+            ..Default::default()
+        },
+    )
+    .expect("Failed to create FilePicker");
+
+    let deadline = std::time::Instant::now() + Duration::from_secs(15);
+    loop {
+        std::thread::sleep(Duration::from_millis(25));
+        let ready = shared_picker
+            .read()
+            .ok()
+            .and_then(|g| {
+                g.as_ref()
+                    .map(|p| !p.is_scan_active() && p.bigram_index().is_some())
+            })
+            .unwrap_or(false);
+        if ready {
+            break;
+        }
+        assert!(
+            std::time::Instant::now() < deadline,
+            "Timed out waiting for bigram build"
+        );
+    }
+
+    let guard = shared_picker.read().unwrap();
+    let picker = guard.as_ref().unwrap();
+    let was_flagged = picker
+        .get_files()
+        .iter()
+        .any(|f| f.relative_path(picker).contains("codex_view") && f.is_binary());
+    assert!(
+        was_flagged,
+        "large no-extension binary must be flagged via the header sniff"
+    );
+
+    let parsed = parse_grep_query("match this text");
+    let opts = GrepSearchOptions {
+        max_file_size: 10 * 1024 * 1024,
+        ..plain_opts()
+    };
+    let result = picker.grep(&parsed, &opts);
+    assert_eq!(
+        result.files.len(),
+        1,
+        "only plain.txt should be searched; codex_view must be skipped as binary"
+    );
+    assert!(
+        result.files[0].relative_path(picker).contains("plain.txt"),
+        "the only match should come from plain.txt"
+    );
+}
+
+#[test]
+fn large_binary_with_nuls_past_header_is_classified() {
+    // Guards the streaming sniff: a >2 MB file that is pure ASCII well past any
+    // fixed header window (the old code only checked the first 8 KB) but has
+    // NULs deeper in. Grep reads the whole file up to max_file_size, so the
+    // detector must scan the same range or the binary tail leaks as "text".
+    use fff_search::file_picker::FFFMode;
+    use fff_search::grep::{GrepSearchOptions, parse_grep_query};
+    use fff_search::{SharedFilePicker, SharedFrecency};
+    use std::time::Duration;
+
+    let tmp = TempDir::new().unwrap();
+    let base = tmp.path();
+
+    // 1 MiB of clean ASCII (with a grep marker) — dwarfs any header sniff —
+    // then NUL bytes, keeping the total above the 2 MiB non-indexable cap.
+    let mut blob = Vec::new();
+    blob.extend_from_slice(b"match this text\n");
+    blob.extend(std::iter::repeat_n(b'A', 1024 * 1024));
+    blob.extend_from_slice(b"match this text\n");
+    blob.extend(std::iter::repeat_n(0u8, 1024 * 1024 + 4096)); // NULs start ~1 MiB in
+    blob.extend_from_slice(b"match this text\n");
+    assert!(blob.len() > 2 * 1024 * 1024);
+    fs::write(base.join("late_nul.dat"), &blob).unwrap();
+    fs::write(base.join("plain.txt"), b"match this text\n").unwrap();
+
+    let shared_picker = SharedFilePicker::default();
+    let shared_frecency = SharedFrecency::default();
+    FilePicker::new_with_shared_state(
+        shared_picker.clone(),
+        shared_frecency.clone(),
+        FilePickerOptions {
+            base_path: base.to_string_lossy().to_string(),
+            enable_mmap_cache: false,
+            enable_content_indexing: true,
+            mode: FFFMode::Neovim,
+            watch: false,
+            ..Default::default()
+        },
+    )
+    .expect("Failed to create FilePicker");
+
+    let deadline = std::time::Instant::now() + Duration::from_secs(15);
+    loop {
+        std::thread::sleep(Duration::from_millis(25));
+        let ready = shared_picker
+            .read()
+            .ok()
+            .and_then(|g| {
+                g.as_ref()
+                    .map(|p| !p.is_scan_active() && p.bigram_index().is_some())
+            })
+            .unwrap_or(false);
+        if ready {
+            break;
+        }
+        assert!(
+            std::time::Instant::now() < deadline,
+            "Timed out waiting for bigram build"
+        );
+    }
+
+    let guard = shared_picker.read().unwrap();
+    let picker = guard.as_ref().unwrap();
+    let flagged = picker
+        .get_files()
+        .iter()
+        .any(|f| f.relative_path(picker).contains("late_nul.dat") && f.is_binary());
+    assert!(
+        flagged,
+        "NULs past the 8 KB header window must still be detected by the streaming scan"
+    );
+
+    let parsed = parse_grep_query("match this text");
+    let opts = GrepSearchOptions {
+        max_file_size: 10 * 1024 * 1024,
+        ..plain_opts()
+    };
+    let result = picker.grep(&parsed, &opts);
+    assert_eq!(
+        result.files.len(),
+        1,
+        "only plain.txt should match; late_nul.dat must be skipped as binary"
+    );
+    assert!(result.files[0].relative_path(picker).contains("plain.txt"));
+}
+
 #[test]
 fn plain_text_max_matches_per_file() {
    let tmp = TempDir::new().unwrap();
@@ -0,0 +1,153 @@
+//! Real-world binary fixture regression.
+//!
+//! Reproduces the exact bug chain we hit with `codex_view` (4.5 MB ELF, no
+//! extension) and `codex_view.codex` (127 KB, unknown extension): both are
+//! binary by content but slip past extension-only triage, so a plain grep
+//! used to surface their NUL-laden bytes as "text" matches.
+//!
+//! The fixtures live in `tests/fixtures/binaries/`. `MARKER` is a string that
+//! is present (as raw bytes) in BOTH binaries — the test first asserts that,
+//! then drops the two binaries plus a single plain-text file containing the
+//! same marker into a closed temp dir and greps for it. Only the text file may
+//! come back; if binary detection ever regresses, a binary file re-enters the
+//! results and this test fails.
+
+use std::fs;
+use std::path::PathBuf;
+use std::time::Duration;
+
+use fff_search::file_picker::{FFFMode, FilePicker};
+use fff_search::grep::{GrepMode, GrepSearchOptions, parse_grep_query};
+use fff_search::{FilePickerOptions, SharedFilePicker, SharedFrecency};
+
+const MARKER: &str = "__jai_runtime_init";
+
+fn fixtures_dir() -> PathBuf {
+    PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/binaries")
+}
+
+fn plain_opts() -> GrepSearchOptions {
+    GrepSearchOptions {
+        max_file_size: 10 * 1024 * 1024,
+        max_matches_per_file: 200,
+        smart_case: true,
+        file_offset: 0,
+        page_limit: 200,
+        mode: GrepMode::PlainText,
+        time_budget_ms: 0,
+        before_context: 0,
+        after_context: 0,
+        classify_definitions: false,
+        trim_whitespace: false,
+        abort_signal: None,
+    }
+}
+
+#[test]
+fn real_binary_fixtures_are_detected_and_excluded_from_grep() {
+    let fixtures = fixtures_dir();
+    let large = fixtures.join("codex_view"); // 4.5 MB ELF, no extension (> 2 MB)
+    let small = fixtures.join("codex_view.codex"); // 127 KB, unknown extension (< 2 MB)
+
+    assert!(
+        large.exists() && small.exists(),
+        "missing binary fixtures in {}",
+        fixtures.display()
+    );
+
+    // Both fixtures must really contain the marker bytes, otherwise the grep
+    // exclusion assertion below would be vacuous.
+    let large_bytes = fs::read(&large).unwrap();
+    let small_bytes = fs::read(&small).unwrap();
+    assert!(
+        contains_subslice(&large_bytes, MARKER.as_bytes()),
+        "fixture codex_view no longer contains the marker {MARKER:?}"
+    );
+    assert!(
+        contains_subslice(&small_bytes, MARKER.as_bytes()),
+        "fixture codex_view.codex no longer contains the marker {MARKER:?}"
+    );
+    // Sanity on the size split that drives the two distinct code paths.
+    assert!(
+        large_bytes.len() > 2 * 1024 * 1024,
+        "codex_view must exceed the 2 MB non-indexable threshold"
+    );
+    assert!(
+        small_bytes.len() < 2 * 1024 * 1024,
+        "codex_view.codex must stay under the 2 MB bigram cap"
+    );
+
+    // Closed environment: the two real binaries + one plain-text file that
+    // legitimately contains the marker.
+    let tmp = tempfile::TempDir::new().unwrap();
+    let base = tmp.path();
+    fs::copy(&large, base.join("codex_view")).unwrap();
+    fs::copy(&small, base.join("codex_view.codex")).unwrap();
+    fs::write(
+        base.join("marker.txt"),
+        format!("the only legitimate hit lives here: {MARKER}\n"),
+    )
+    .unwrap();
+
+    let shared_picker = SharedFilePicker::default();
+    let shared_frecency = SharedFrecency::default();
+    FilePicker::new_with_shared_state(
+        shared_picker.clone(),
+        shared_frecency.clone(),
+        FilePickerOptions {
+            base_path: base.to_string_lossy().to_string(),
+            enable_mmap_cache: false,
+            enable_content_indexing: true,
+            mode: FFFMode::Neovim,
+            watch: false,
+            ..Default::default()
+        },
+    )
+    .expect("failed to create FilePicker");
+
+    shared_picker.wait_for_indexing_complete(Duration::from_secs(5));
+
+    let guard = shared_picker.read().unwrap();
+    let picker = guard.as_ref().unwrap();
+
+    // Both binaries must be classified binary.
+    for name in ["codex_view", "codex_view.codex"] {
+        let flagged = picker
+            .get_files()
+            .iter()
+            .any(|f| f.relative_path(picker).ends_with(name) && f.is_binary());
+        assert!(flagged, "{name} must be flagged is_binary");
+    }
+
+    // we need to make sure that marker.txt ONLY can match as we have to match
+    // grep as binaries are excluded from the matching process
+    let parsed = parse_grep_query(MARKER);
+    let result = picker.grep(&parsed, &plain_opts());
+
+    let matched: Vec<String> = result
+        .files
+        .iter()
+        .map(|f| f.relative_path(picker))
+        .collect();
+
+    assert_eq!(
+        result.files.len(),
+        1,
+        "exactly one file should match {MARKER:?}, got: {matched:?}"
+    );
+    assert!(
+        matched[0].ends_with("marker.txt"),
+        "the only match must be marker.txt, got {:?}",
+        matched[0]
+    );
+}
+
+/// Tiny substring search over raw bytes (the marker may be surrounded by NULs).
+fn contains_subslice(haystack: &[u8], needle: &[u8]) -> bool {
+    if needle.is_empty() || haystack.len() < needle.len() {
+        return false;
+    }
+    haystack
+        .windows(needle.len())
+        .any(|window| window == needle)
+}
@@ -239,6 +239,11 @@ impl IntoLua for GrepResultLua<'_> {
            item.set("line_number", m.line_number)?;
            item.set("col", m.col)?;
            item.set("byte_offset", m.byte_offset)?;
+
+            // There is a little race window when fff can return matches inside of a non-binary
+            // classified entities, the window is minimal but it errors out neovim so guard it
+            let is_binary_content = m.line_content.as_bytes().contains(&0u8);
+            item.set("is_binary_content", is_binary_content)?;
            item.set("line_content", m.line_content.as_str())?;

            // Match byte ranges within line_content
@@ -48,6 +48,8 @@ local function format_location(item, ctx)
  return str
 end

+local BINARY_PLACEHOLDER = '<binary content>'
+
 local function render_match_line(item, ctx)
  local location = format_location(item, ctx)
  local separator = '  '
@@ -55,6 +57,7 @@ local function render_match_line(item, ctx)
  local raw_content = item.line_content
  if type(raw_content) ~= 'string' then raw_content = raw_content and tostring(raw_content) or '' end
  local content = raw_content
+  if item.is_binary_content then content = BINARY_PLACEHOLDER end

  -- Indent + location + separator + content
  local indent = ' '
@@ -138,7 +141,17 @@ local function apply_match_highlights(item, ctx, item_idx, buf, ns_id, row, line
  -- Priority 120: above CursorLine (100) so syntax is visible on cursor line,
  -- below IncSearch match ranges (200) so search matches take precedence.
  local content_start = sep_end
-  if item._trimmed_content and item.name then
+
+  if item.is_binary_content then
+    local content_end = content_start + #BINARY_PLACEHOLDER
+    if content_end <= #line_content then
+      pcall(vim.api.nvim_buf_set_extmark, buf, ns_id, row, content_start, {
+        end_col = content_end,
+        hl_group = 'Comment',
+        priority = 150,
+      })
+    end
+  elseif item._trimmed_content and item.name then
    -- Resolve language once per file group (cache on the render context)
    ctx._ts_lang_cache = ctx._ts_lang_cache or {}
    local lang = ctx._ts_lang_cache[item.name]
@@ -166,7 +179,7 @@ local function apply_match_highlights(item, ctx, item_idx, buf, ns_id, row, line
  -- 5. Match ranges highlighted with IncSearch
  -- Use extmarks with priority > cursor line (100) so IncSearch renders
  -- properly on the selected line instead of being overridden by CursorLine.
-  if item.match_ranges then
+  if item.match_ranges and not item.is_binary_content then
    for _, range in ipairs(item.match_ranges) do
      local raw_start = range[1] or 0
      local raw_end = range[2] or 0