Compare commits

..

2 Commits

Author SHA1 Message Date
Dmitriy Kovalenko 2e8a0d8fb7 fix(lua): Resolve CI lint errors
- main.lua: drop duplicate `open_file_under_cursor` impl,
  superseded by the path-resolving variant
- picker_ui.lua: replace undefined `canonicalize_fff_path`
  with `utils.canonicalize_picker_path`
- programmatic_search_spec.lua: cast `hit` to non-nil after
  the `assert.is_not_nil` so lua-ls stops flagging the
  follow-up field accesses
2026-05-22 08:58:34 -07:00
Dmitriy Kovalenko 1ba34a9541 feat(lua): Add programmatic api for lua 2026-05-22 06:19:24 -07:00
42 changed files with 240 additions and 1112 deletions
+18 -16
View File
@@ -2,7 +2,7 @@ name: Prebuild
on:
push:
branches: [main, fix/use-trusted-publishing]
branches: [main, fix/download-version]
tags:
- "v*"
pull_request:
@@ -16,7 +16,6 @@ jobs:
runs-on: ${{ matrix.os }}
permissions:
contents: read
id-token: write
strategy:
matrix:
include:
@@ -363,7 +362,7 @@ jobs:
name: Release
needs: [build-nvim, build-c, build-mcp]
runs-on: ubuntu-latest
if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/fix/use-trusted-publishing' || startsWith(github.ref, 'refs/tags/v'))
if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/fix/download-version' || startsWith(github.ref, 'refs/tags/v'))
permissions:
contents: write
steps:
@@ -470,14 +469,10 @@ jobs:
name: Publish Rust crates
needs: [build-nvim, build-c, build-mcp]
runs-on: ubuntu-latest
if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/fix/use-trusted-publishing' || startsWith(github.ref, 'refs/tags/v'))
permissions:
contents: read
id-token: write
if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/fix/download-version' || startsWith(github.ref, 'refs/tags/v'))
steps:
- uses: actions/checkout@v5
- uses: rust-lang/crates-io-auth-action@v1
id: auth
- name: Install Lua
uses: leafo/gh-actions-lua@v12
@@ -494,17 +489,16 @@ jobs:
- name: Publish crates
env:
CARGO_REGISTRY_TOKEN: ${{ steps.auth.outputs.token }}
CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
run: make publish-crates V="${{ steps.version.outputs.version }}"
npm-publish:
name: Publish npm packages
needs: [build-c]
runs-on: ubuntu-latest
if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/fix/use-trusted-publishing' || startsWith(github.ref, 'refs/tags/v'))
if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/fix/download-version' || startsWith(github.ref, 'refs/tags/v'))
permissions:
contents: read
id-token: write
steps:
- uses: actions/checkout@v5
@@ -528,6 +522,8 @@ jobs:
path: ./npm-packages
- name: Publish platform packages
env:
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
run: |
VERSION="${{ steps.version.outputs.version }}"
TAG="${{ steps.version.outputs.npm_tag }}"
@@ -540,12 +536,14 @@ jobs:
make set-npm-version PKG="$pkg_dir" VERSION="$VERSION"
cd "$pkg_dir"
npm publish --tag "$TAG" --access public --provenance
npm publish --tag "$TAG" --access public || echo "Failed to publish ${pkg_name} (may already exist)"
cd -
fi
done
- name: Publish bun package
env:
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
run: |
VERSION="${{ steps.version.outputs.version }}"
TAG="${{ steps.version.outputs.npm_tag }}"
@@ -554,9 +552,11 @@ jobs:
make set-npm-version PKG=packages/fff-bun VERSION="$VERSION"
cd packages/fff-bun
npm publish --tag "$TAG" --access public --provenance
npm publish --tag "$TAG" --access public || echo "Failed to publish @ff-labs/fff-bun (may already exist)"
- name: Publish Node.js package
env:
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
run: |
VERSION="${{ steps.version.outputs.version }}"
TAG="${{ steps.version.outputs.npm_tag }}"
@@ -567,9 +567,11 @@ jobs:
cd packages/fff-node
npm install
npm run build
npm publish --tag "$TAG" --access public --provenance
npm publish --tag "$TAG" --access public || echo "Failed to publish @ff-labs/fff-node (may already exist)"
- name: Publish pi-fff package
env:
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
run: |
VERSION="${{ steps.version.outputs.version }}"
TAG="${{ steps.version.outputs.npm_tag }}"
@@ -578,4 +580,4 @@ jobs:
make set-npm-version PKG=packages/pi-fff VERSION="$VERSION"
cd packages/pi-fff
npm publish --tag "$TAG" --access public --provenance
npm publish --tag "$TAG" --access public || echo "Failed to publish @ff-labs/pi-fff (may already exist)"
Generated
+10 -10
View File
@@ -633,7 +633,7 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
[[package]]
name = "fff-c"
version = "0.8.4"
version = "0.8.1"
dependencies = [
"fff-query-parser",
"fff-search",
@@ -643,7 +643,7 @@ dependencies = [
[[package]]
name = "fff-grep"
version = "0.8.4"
version = "0.8.1"
dependencies = [
"bstr",
"memchr",
@@ -651,7 +651,7 @@ dependencies = [
[[package]]
name = "fff-mcp"
version = "0.8.4"
version = "0.8.1"
dependencies = [
"clap",
"fff-query-parser",
@@ -682,7 +682,7 @@ dependencies = [
[[package]]
name = "fff-nvim"
version = "0.8.4"
version = "0.8.1"
dependencies = [
"ahash",
"chrono",
@@ -701,7 +701,7 @@ dependencies = [
[[package]]
name = "fff-query-parser"
version = "0.8.4"
version = "0.8.1"
dependencies = [
"criterion",
"zlob",
@@ -709,7 +709,7 @@ dependencies = [
[[package]]
name = "fff-search"
version = "0.8.4"
version = "0.8.1"
dependencies = [
"ahash",
"aho-corasick",
@@ -2032,9 +2032,9 @@ checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a"
[[package]]
name = "rmcp"
version = "1.7.0"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0810a9f717d9828f475fe1f629f4c305c8464b7f496c3a854b58d29e65f4058e"
checksum = "ba6b9d2f0efe2258b23767f1f9e0054cfbcac9c2d6f81a031214143096d7864f"
dependencies = [
"async-trait",
"base64",
@@ -2054,9 +2054,9 @@ dependencies = [
[[package]]
name = "rmcp-macros"
version = "1.7.0"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6aefac48c364756e97f04c0401ba3231e8607882c7c1d92da0437dc16307904d"
checksum = "ab9d95d7ed26ad8306352b0d5f05b593222b272790564589790d210aa15caa9e"
dependencies = [
"darling",
"proc-macro2",
+2 -2
View File
@@ -10,8 +10,8 @@ members = [
resolver = "2"
[workspace.dependencies]
fff-grep = { version = "0.8.4", path = "crates/fff-grep" }
fff-query-parser = { version = "0.8.4", path = "crates/fff-query-parser", default-features = false }
fff-grep = { version = "0.8.1", path = "crates/fff-grep" }
fff-query-parser = { version = "0.8.1", path = "crates/fff-query-parser", default-features = false }
# Shared dependencies
ahash = "0.8"
+1 -18
View File
@@ -321,7 +321,6 @@ require('fff').setup({
time_budget_ms = 150,
modes = { 'plain', 'regex', 'fuzzy' },
trim_whitespace = false,
location_format = ':%d:%d', -- printf format for line:col prefix in grep results, e.g. ':%d' for line-only
},
debug = {
enabled = false, -- show the file info panel next to the preview
@@ -387,22 +386,6 @@ Sign-column indicators are on by default. To color filename text by git status,
The picker maps its float content to `NormalFloat` (via `hl.normal`) and the border to `FloatBorder`. Default `FloatBorder` links to `NormalFloat`, so border and content share a background out of the box and the picker reads as a single popup. Override `hl.normal = 'Normal'` to make the picker blend with the editor instead.
For finer control, set `hl.winhl` to override the per-window `winhighlight`. It accepts either a single string applied to every picker window, or a table with optional `prompt`, `list`, `preview`, and `file_info` keys. Missing keys fall back to the default built from `hl.normal`, `hl.border`, and `hl.title`.
```lua
-- Apply the same winhighlight to all picker windows
hl = { winhl = 'Normal:NormalFloat,FloatBorder:FloatBorder,FloatTitle:Title' }
-- Or override specific windows only
hl = {
winhl = {
prompt = 'Normal:Pmenu,FloatBorder:FloatBorder',
list = 'Normal:NormalFloat,FloatBorder:FloatBorder',
preview = 'Normal:NormalFloat,FloatBorder:FloatBorder',
},
}
```
### File info panel
Enable with `debug.enabled = true`. The panel sits above the preview and shows
@@ -613,7 +596,7 @@ Algorithm for fuzzy matching is much more comprehensive than fzf's algorithm it
### What the core actually does
- **Frecency-ranked fuzzy matching.** Every indexed file carries an access score and a modification score. Searches rank files you have opened recently and frequently above cold results. This is the same idea as VS Code's recently-opened list, but applied to every search result, not just a sidebar.
- **Typo-resistant matching for both paths and content.** Smith-Waterman fuzzy scoring is available on the grep path; path search uses SIMD-accelerated fuzzy matching (via the [`frizbee`](https://github.com/saghen/frizbee)-derived core) that survives dropped characters and reorderings.
- **Typo-resistant matching for both paths and content.** Smith-Waterman fuzzy scoring is available on the grep path; path search uses SIMD-accelerated fuzzy matching (via the [`frizbee`](https://github.com/saghm/frizbee)-derived core) that survives dropped characters and reorderings.
- **Content grep with three modes.** Plain literal (SIMD memmem), regex (the Rust `regex` crate), and fuzzy (Smith-Waterman per line). Auto-detects which mode to use from the pattern, falls back to fuzzy when a plain search returns zero hits.
- **Multi-pattern OR search.** SIMD Aho-Corasick for "find any of these 20 identifiers at once", which is faster than regex alternation and a lot faster than 20 separate ripgrep runs.
- **Background file watcher.** The index updates as files change. You never pay for a rescan on the hot path.
+3 -3
View File
@@ -1,6 +1,6 @@
[package]
name = "fff-c"
version = "0.8.4"
version = "0.8.1"
edition = "2024"
description = "Raw C api of FFF file finder"
license = "MIT"
@@ -15,6 +15,6 @@ zlob = ["fff/zlob"]
[dependencies]
git2.workspace = true
fff = { package = "fff-search", path = "../fff-core" , version = "0.8.4" }
fff-query-parser = { path = "../fff-query-parser" , version = "0.8.4" }
fff = { package = "fff-search", path = "../fff-core" , version = "0.8.1" }
fff-query-parser = { path = "../fff-query-parser" , version = "0.8.1" }
serde_json = "1.0"
+3 -3
View File
@@ -1,6 +1,6 @@
[package]
name = "fff-search"
version = "0.8.4"
version = "0.8.1"
edition = "2024"
license = "MIT"
authors = ["Dmitriy Kovalenko <dmtr.kovalenko@outlook.com>"]
@@ -39,14 +39,14 @@ rayon = { workspace = true }
smallvec = { workspace = true }
thiserror = { workspace = true }
tracing = { workspace = true }
fff-query-parser = { workspace = true , version = "0.8.2" }
fff-query-parser = { workspace = true }
blake3 = { workspace = true }
dirs = { workspace = true }
libc = "0.2"
git2 = { workspace = true }
glidesort = { workspace = true }
globset = { workspace = true }
fff-grep = { workspace = true , version = "0.8.2" }
fff-grep = { workspace = true }
aho-corasick = "1"
memchr = "2"
heed = { workspace = true }
+5 -6
View File
@@ -1,6 +1,5 @@
use crate::constants::MAX_OVERFLOW_FILES;
use crate::error::Error;
use crate::file_picker::FFFMode;
use crate::file_picker::{FFFMode, MAX_OVERFLOW_FILES};
use crate::git::GitStatusCache;
use crate::shared::{SharedFilePicker, SharedFrecency};
use crate::sort_buffer::sort_with_buffer;
@@ -26,6 +25,7 @@ pub struct BackgroundWatcher {
}
const DEBOUNCE_TIMEOUT: Duration = Duration::from_millis(50);
const MAX_PATHS_THRESHOLD: usize = 1024;
/// On macOS, each `watch()` call creates a separate FSEventStream. When the
/// number of directories exceeds this threshold we fall back to a single
/// recursive watch to avoid exhausting the per-process stream limit.
@@ -497,11 +497,10 @@ fn handle_debounced_events(
}
affected_paths_count += debounced_event.event.paths.len();
if affected_paths_count > MAX_OVERFLOW_FILES {
if affected_paths_count > MAX_PATHS_THRESHOLD {
warn!(
?affected_paths_count,
max = MAX_OVERFLOW_FILES,
"Too many affected paths in a single batch, triggering full rescan",
"Too many affected paths ({}) in a single batch, triggering full rescan",
affected_paths_count
);
need_full_rescan = true;
+14 -37
View File
@@ -1,4 +1,3 @@
use crate::constants::MAX_INDEXABLE_FILE_SIZE;
use ahash::AHashMap;
use rayon::iter::{IndexedParallelIterator, ParallelIterator};
use rayon::slice::ParallelSlice;
@@ -6,8 +5,6 @@ use std::cell::UnsafeCell;
use std::sync::OnceLock;
use std::sync::atomic::{AtomicU16, AtomicUsize, Ordering};
use crate::{FileItem, constants};
/// Maximum number of distinct bigrams tracked in the inverted index.
/// 95 printable ASCII chars (32..=126) after lowercasing → ~70 distinct → 4900 possible.
/// We cap at 5000 to cover all printable bigrams with margin.
@@ -110,8 +107,14 @@ impl BigramIndexBuilder {
&slab[start..start + self.words]
}
// `pub` (via `#[doc(hidden)]`) only for benchmarking
// External consumers should use `build_bigram_index` instead.
// `pub` (via `#[doc(hidden)]`) only so the criterion bench can drive
// `add_file_content` directly. External consumers should use
// `build_bigram_index` instead.
///
/// SAFETY: concurrent callers must partition `file_idx` by
/// word-aligned ranges so that `file_idx / 64` never collides across
/// threads. The `file_picker::build_bigram_index` driver enforces
/// this via `par_chunks` with a word-aligned chunk size.
#[doc(hidden)]
pub fn add_file_content(&self, skip_builder: &Self, file_idx: usize, content: &[u8]) {
if content.len() < 2 {
@@ -593,6 +596,7 @@ impl BigramOverlay {
}
}
pub(crate) const MAX_INDEXABLE_FILE_SIZE: usize = 2 * 1024 * 1024;
const BIGRAM_CHUNK_FILES: usize = 4 * 64;
/// Sparse-column cutoff for the skip-1 sub-index. Rare skip columns add
@@ -612,7 +616,7 @@ thread_local! {
/// mmap should only be used by the locked version of grep which absolutely minimizes any riscs
#[inline]
fn read_bigram_chunk<'a>(
file: &FileItem,
file: &crate::types::FileItem,
base_fd: libc::c_int,
base_path: &std::path::Path,
arena: crate::simd_path::ArenaPtr,
@@ -626,7 +630,10 @@ fn read_bigram_chunk<'a>(
}
let data = &buf[..filled];
if crate::file_picker::detect_binary_content(data) {
file.set_binary(true);
return None;
}
Some(data)
}
@@ -674,14 +681,6 @@ pub(crate) fn build_bigram_index(
&mut buf[..],
&mut path_buf,
) {
// we have to manually ensure that every byte is a valid text byte to
// perform this we have to scan every file, first 512 bytes is not enough
// so basically we rely on the fact that first 2MB will always contain
// an invalid text sequence if this is not a binary file.
//
// Need to find a better way to do this.
file.set_binary(crate::types::detect_binary_content(content));
builder.add_file_content(&skip_builder, file_idx, content);
}
});
@@ -706,28 +705,6 @@ pub(crate) fn build_bigram_index(
index
}
#[tracing::instrument(skip_all, name = "Sniffing Large Files Binary", level = tracing::Level::DEBUG)]
pub(crate) fn sniff_binary_for_non_indexable(
files: &[FileItem],
base_path: &std::path::Path,
arena: crate::simd_path::ArenaPtr,
) {
// Non-indexable files are few in a typical repo, so a serial pass with a
// single reused chunk buffer beats spinning up the thread pool.
let mut path_buf = [0u8; crate::simd_path::PATH_BUF_SIZE];
let mut chunk = vec![0u8; crate::types::BINARY_CLASSIFICATION_CHUNK_SIZE];
for file in files {
// check only the files that we are able to grep
if file.size == 0 || file.size > constants::MAX_FFFILE_SIZE {
continue;
}
let abs = file.write_absolute_path(arena, base_path, &mut path_buf);
file.detect_binary_per_byte(abs, &mut chunk);
}
}
/// Open the base directory for the `openat` fast path. Returns `-1` on
/// failure — callers interpret a negative fd as "fall back to absolute
/// paths".
-39
View File
@@ -1,39 +0,0 @@
/// Largest file whose full content fff will touch: the default grep read cap
/// (`GrepSearchOptions::max_file_size`) and the content-cache mmap cap
/// (`ContentCacheBudget::max_file_size`). Binary detection also streams up to
/// this far so nothing grep would read is left unclassified.
pub const MAX_FFFILE_SIZE: u64 = 10 * 1024 * 1024;
/// Upper bound on a file the bigram builder will build, if the file is very large there is a
/// big probability it will only bloat the available bigrams and will anyway pop ut from the prefilter
pub const MAX_INDEXABLE_FILE_SIZE: usize = 2 * 1024 * 1024;
/// Total bytes the persistent content mmap cache may hold for a small repo.
pub const MAX_CACHED_CONTENT_BYTES: u64 = 512 * 1024 * 1024;
/// Files below one page waste the remainder when mmapped, so the cache skips
/// them and falls back to chunked reads. Unused on Windows (no content cache).
#[cfg(all(not(target_os = "windows"), target_arch = "aarch64"))]
pub const MMAP_THRESHOLD: u64 = 16 * 1024;
#[cfg(all(not(target_os = "windows"), not(target_arch = "aarch64")))]
pub const MMAP_THRESHOLD: u64 = 4 * 1024;
/// Capacity reserved for files the watcher discovers after the initial scan;
/// exceeding it forces a full rescan.
pub const MAX_OVERFLOW_FILES: usize = 1024;
/// Fresh-mmap threshold: files at or above this size get mmapped directly on
/// cache miss instead of chunked reads into Vec. Empirically tuned per-platform.
/// Only referenced on Unix; Windows uses the `std::fs::read` fallback so this
/// constant is gated to non-Windows targets to keep `-D unused-imports` happy.
#[cfg(target_os = "macos")]
pub const FRESH_MMAP_THRESHOLD: u64 = 1024 * 1024;
#[cfg(all(not(target_os = "macos"), not(target_os = "windows")))]
pub const FRESH_MMAP_THRESHOLD: u64 = 256 * 1024;
// we do not support 32kb path limit on windows
#[cfg(target_os = "windows")]
pub const PATH_BUF_SIZE: usize = 4096;
#[cfg(not(target_os = "windows"))]
pub const PATH_BUF_SIZE: usize = libc::PATH_MAX as usize;
+44 -54
View File
@@ -33,7 +33,6 @@
use crate::FFFStringStorage;
use crate::background_watcher::{BackgroundWatcher, is_git_file};
use crate::bigram_filter::{BigramFilter, BigramOverlay};
use crate::constants::{MAX_OVERFLOW_FILES, PATH_BUF_SIZE};
use crate::error::Error;
use crate::frecency::FrecencyTracker;
use crate::git::GitStatusCache;
@@ -43,7 +42,7 @@ use crate::query_tracker::QueryTracker;
use crate::scan::{ScanConfig, ScanJob, ScanSignals};
use crate::score::fuzzy_match_and_score_files;
use crate::shared::{SharedFilePicker, SharedFrecency};
use crate::simd_path::ArenaPtr;
use crate::simd_path::{ArenaPtr, PATH_BUF_SIZE};
use crate::stable_vec::StableVec;
use crate::types::{
ContentCacheBudget, DirItem, DirSearchResult, FileItem, MixedItemRef, MixedSearchResult,
@@ -63,6 +62,11 @@ use std::thread::JoinHandle;
use std::time::SystemTime;
use tracing::{Level, debug, error, info, warn};
/// Max overflow files before the watcher triggers a full rescan.
/// `walk_filesystem` reserves this much extra capacity so the Vec never
/// reallocates while raw pointers are held during post-scan.
pub(crate) const MAX_OVERFLOW_FILES: usize = 1024;
/// Dedicated thread pool for background work (scan, warmup, bigram build).
/// Uses fewer threads than the global rayon pool so Neovim's event loop
/// and search queries can still get CPU time.
@@ -808,6 +812,8 @@ impl FilePicker {
self.sync_data = sync;
// Recalculate cache budget based on actual file count (unless
// the caller provided an explicit budget via FilePickerOptions).
if !self.has_explicit_cache_budget {
let file_count = self.sync_data.files().len();
self.cache_budget = Arc::new(ContentCacheBudget::new_for_repo(file_count));
@@ -815,18 +821,14 @@ impl FilePicker {
self.cache_budget.reset();
}
// Apply git status synchronously.
if let Some(handle) = git_handle
&& let Ok(Some(git_cache)) = handle.join()
{
let mut path_buf = [0u8; crate::simd_path::PATH_BUF_SIZE];
let arena = self.arena_base_ptr();
for file in self.sync_data.files.iter_mut() {
file.git_status = git_cache.lookup_status(file.write_absolute_path(
arena,
&self.base_path,
&mut path_buf,
));
file.git_status =
git_cache.lookup_status(&file.absolute_path(arena, &self.base_path));
}
}
@@ -862,7 +864,6 @@ impl FilePicker {
/// The query should be parsed using [`FFFQuery`]::parse() before calling
/// this function. If a [`QueryTracker`] is provided, the search will
/// automatically look up the last selected file for this query and boost it
#[tracing::instrument(skip_all, name = "Fuzzy file search", fields(query = query.raw_query))]
pub fn fuzzy_search<'q>(
&self,
query: &'q FFFQuery<'q>,
@@ -1209,13 +1210,11 @@ impl FilePicker {
pub fn get_scan_progress(&self) -> ScanProgress {
let scanned_count = self.scanned_files_count.load(Ordering::Relaxed);
let is_scanning = self.signals.scanning.load(Ordering::Relaxed);
ScanProgress {
scanned_files_count: scanned_count,
is_scanning,
is_watcher_ready: self.signals.watcher_ready.load(Ordering::Relaxed),
is_warmup_complete: !self.enable_content_indexing
|| self.sync_data.bigram_index.is_some(),
is_warmup_complete: self.sync_data.bigram_index.is_some(),
}
}
@@ -1272,9 +1271,9 @@ impl FilePicker {
base_count: self.sync_data.base_count,
indexable_count: self.sync_data.indexable_count,
base_path: self.base_path.clone(),
budget: Arc::clone(&self.cache_budget),
cancelled: Arc::clone(&self.signals.cancelled),
post_scan_flag: Arc::clone(&self.signals.post_scan_indexing_active),
_budget: Arc::clone(&self.cache_budget),
})
}
@@ -1420,14 +1419,7 @@ impl FilePicker {
file.update_metadata(&self.cache_budget, modified_time, Some(size));
// Re-classify binary status from current content (chunked, fixed
// buffer). Already-binary files are left alone.
if !file.is_binary() {
let mut chunk = [0u8; crate::types::BINARY_CLASSIFICATION_CHUNK_SIZE];
file.detect_binary_per_byte(path, &mut chunk);
}
// Indexable base-region files feed fresh content to the bigram overlay.
// only base-region entries participate in the bigram overlay
if matches!(slot, FileSlot::Base(_))
&& let Some(ref overlay) = overlay
{
@@ -1457,10 +1449,12 @@ impl FilePicker {
} else if let Ok(c) = crate::path_utils::canonicalize(path) {
Some(c)
} else {
tracing::error!(path = ?path.display(), "Failed to canonicalize file path to add");
return None;
let parent = path.parent()?;
let file_name = path.file_name()?;
let mut p = crate::path_utils::canonicalize(parent).ok()?;
p.push(file_name);
Some(p)
};
#[cfg(windows)]
let path_for_index: &Path = canonical_buf.as_deref().unwrap_or(path);
#[cfg(not(windows))]
@@ -1469,20 +1463,14 @@ impl FilePicker {
let (mut file_item, rel_path) =
FileItem::new(path_for_index.to_path_buf(), &self.base_path, None);
// we have to perform manual classification for every new file this will be
// batched during the scan, this is the path when the file is ad-hoc added to the sync
file_item.detect_binary_per_byte(
path_for_index,
// inline chunk buf
&mut [0u8; crate::types::BINARY_CLASSIFICATION_CHUNK_SIZE],
);
// Lazily create the shared overflow builder if not exists yet
let builder = self
.sync_data
.overflow_builder
.get_or_insert_with(|| crate::simd_path::ChunkedPathStoreBuilder::new(64));
let builder = self.sync_data.overflow_builder.get_or_insert_with(|| {
// we know that overflow would never create more files during the file
crate::simd_path::ChunkedPathStoreBuilder::new(MAX_OVERFLOW_FILES)
});
file_item.set_path(builder.add_file_immediate(&rel_path, file_item.path.filename_offset));
let chunked_path = builder.add_file_immediate(&rel_path, file_item.path.filename_offset);
file_item.set_path(chunked_path);
file_item.set_overflow(true);
if !self.sync_data.files.push(file_item) {
@@ -1669,8 +1657,7 @@ pub(crate) struct PostScanUnsafeSnapshot {
pub files: StableVec<FileItem>,
pub dirs: StableVec<crate::types::DirItem>,
pub arena: Option<Arc<crate::simd_path::ChunkedPathStore>>,
// TODO figure this out
pub _budget: Arc<crate::types::ContentCacheBudget>,
pub budget: Arc<crate::types::ContentCacheBudget>,
pub base_count: usize,
pub indexable_count: usize,
pub base_path: PathBuf,
@@ -1859,7 +1846,7 @@ impl FileSync {
let is_indexable = |f: &FileItem| {
!f.is_binary()
&& f.size > 0
&& f.size <= crate::constants::MAX_INDEXABLE_FILE_SIZE as u64
&& f.size <= crate::bigram_filter::MAX_INDEXABLE_FILE_SIZE as u64
};
BACKGROUND_THREAD_POOL.install(|| {
@@ -1999,12 +1986,7 @@ pub fn is_known_binary_extension(path: &Path) -> bool {
ext,
// Images
"png" | "jpg" | "jpeg" | "gif" | "bmp" | "ico" | "webp" | "tiff" | "tif" | "avif" |
"heic" | "heif" | "jxl" | "jp2" | "j2k" | "psd" | "icns" | "cur" | "cr2" |
"nef" | "dng" | "tga" |
// GPU / VFX texture formats
"rgbe" | "hdr" | "exr" | "dds" | "ktx" | "ktx2" | "pvr" | "astc" |
// Adobe Illustrator (PDF wrapper) / Apple webarchive / MIME HTML archive
"ai" | "webarchive" | "mhtml" |
"heic" | "psd" | "icns" | "cur" | "raw" | "cr2" | "nef" | "dng" | "tga" |
// Video/Audio
"mp4" | "avi" | "mov" | "wmv" | "mkv" | "mp3" | "wav" | "flac" | "ogg" | "m4a" |
"aac" | "webm" | "flv" | "mpg" | "mpeg" | "wma" | "opus" | "pcm" | "reapeaks" |
@@ -2028,24 +2010,32 @@ pub fn is_known_binary_extension(path: &Path) -> bool {
// Compiled/Runtime
"class" | "pyc" | "pyo" | "wasm" | "dex" | "jar" | "war" |
// OCaml / Swift / Objective-C build artefacts
"cmi" | "cmt" | "cmti" | "cmx" | "nib" |
"cmi" | "cmt" | "cmti" | "cmx" | "cof" | "cop" | "nib" |
"swiftdeps" | "swiftdeps~" | "swiftdoc" | "swiftmodule" | "swiftsourceinfo" |
// ML/Data Science
"npy" | "npz" | "h5" | "hdf5" | "pt" | "onnx" |
"safetensors" | "tfrecord" | "tflite" | "gguf" | "ggml" | "joblib" |
"npy" | "npz" | "pkl" | "pickle" | "h5" | "hdf5" | "pt" | "pth" | "onnx" |
"safetensors" | "tfrecord" |
// 3D/Game assets
"glb" | "blend" | "blp" |
// Gzipped-XML / binary maps
"dia" | "bcmap" |
"glb" | "fbx" | "blend" | "blp" |
// Compressed-text formats (gzip/binary on disk)
"dia" | "tfx" | "flm" | "bcmap" | "journal" |
// Protobuf wire format
"pb" |
// Data/serialized
"parquet" | "arrow" |
// IDE/OS metadata
"suo"
"DS_Store" | "suo"
)
}
/// Detect binary content by checking for NUL bytes in the first 512 bytes.
/// Called lazily when file content is first loaded, not during initial scan.
#[inline]
pub(crate) fn detect_binary_content(content: &[u8]) -> bool {
let check_len = content.len().min(512);
content[..check_len].contains(&0)
}
/// Length of the longest shared directory prefix of two relative dir
/// paths (without a trailing separator), measured as the number of bytes
/// up to and including the last shared separator — plus the full shorter
+15 -27
View File
@@ -11,7 +11,7 @@ use crate::{
constraints::apply_constraints,
extract_bigrams,
sort_buffer::sort_with_buffer,
types::{ContentCacheBudget, FileItem, FileSliceExt, MmapSlot},
types::{ContentCacheBudget, FileItem, FileSliceExt},
};
use aho_corasick::AhoCorasick;
pub use fff_grep::{
@@ -333,8 +333,6 @@ pub struct GrepResult<'a> {
pub regex_fallback_error: Option<String>,
}
pub use crate::constants::MAX_FFFILE_SIZE;
/// Options for grep search.
#[derive(Debug, Clone)]
pub struct GrepSearchOptions {
@@ -373,7 +371,7 @@ pub struct GrepSearchOptions {
impl Default for GrepSearchOptions {
fn default() -> Self {
Self {
max_file_size: MAX_FFFILE_SIZE,
max_file_size: 10 * 1024 * 1024,
max_matches_per_file: 200,
smart_case: true,
file_offset: 0,
@@ -1245,17 +1243,16 @@ where
for chunk in files_to_search.chunks(chunk_size) {
let chunk_offset = files_consumed;
// Parallel phase: search all files in this chunk concurrently.
// Within a chunk every file is visited (no gaps), so pagination
// offsets remain correct across chunk boundaries.
let chunk_results: Vec<(usize, &'a FileItem, Vec<GrepMatch>)> = chunk
.par_iter()
.enumerate()
.map_init(
// Per-thread scratch: a reusable read buffer for small files
// and an mmap slot for cache-miss large files (≥ FRESH_MMAP_THRESHOLD).
|| {
tracing::info!("LMAOTHREAD");
(Vec::with_capacity(64 * 1024), MmapSlot::default())
},
|(buf, mmap_slot), (local_idx, file)| {
// allocatge a single reusable buffer per thread
|| Vec::with_capacity(64 * 1024),
|buf, (local_idx, file)| {
if ctx.abort_signal.load(Ordering::Relaxed) {
budget_exceeded.store(true, Ordering::Relaxed);
return None;
@@ -1271,7 +1268,6 @@ where
let content = file.get_content_for_search(
buf,
mmap_slot,
ctx.arena_for_file(file),
ctx.base_path,
ctx.budget,
@@ -1639,21 +1635,14 @@ fn fuzzy_grep_search<'a>(
let budget_exceeded = AtomicBool::new(false);
let max_matches_per_file = options.max_matches_per_file;
// Parallel phase with `map_init`: each rayon worker thread clones the
// matcher once and gets a reusable read buffer + mmap slot. Buffer holds
// small files, slot holds fresh mmap for cache-miss files
// ≥ FRESH_MMAP_THRESHOLD.
// matcher once and gets a reusable read buffer. The buffer avoids
// mmap/munmap syscalls for non-cached files.
let per_file_results: Vec<(usize, &'a FileItem, Vec<GrepMatch>)> = files_to_search
.par_iter()
.enumerate()
.map_init(
|| {
(
matcher.clone(),
Vec::with_capacity(64 * 1024),
MmapSlot::default(),
)
},
|(matcher, buf, mmap_slot), (idx, file)| {
|| (matcher.clone(), Vec::with_capacity(64 * 1024)),
|(matcher, buf), (idx, file)| {
if abort_signal.load(Ordering::Relaxed) {
budget_exceeded.store(true, Ordering::Relaxed);
return None;
@@ -1671,8 +1660,7 @@ fn fuzzy_grep_search<'a>(
} else {
arena
};
let file_bytes =
file.get_content_for_search(buf, mmap_slot, file_arena, base_path, budget)?;
let file_bytes = file.get_content_for_search(buf, file_arena, base_path, budget)?;
// File-level prefilter: check if enough distinct needle chars
// exist anywhere in the file bytes. Uses memchr for speed.
@@ -2435,7 +2423,7 @@ mod tests {
let arena = picker.arena_base_ptr();
let options = super::GrepSearchOptions {
max_file_size: MAX_FFFILE_SIZE,
max_file_size: 10 * 1024 * 1024,
max_matches_per_file: 0,
smart_case: true,
file_offset: 0,
@@ -2619,7 +2607,7 @@ mod tests {
// (a, b, c in base + f, g, h in overflow).
let query = super::parse_grep_query("unicorn");
let options = super::GrepSearchOptions {
max_file_size: MAX_FFFILE_SIZE,
max_file_size: 10 * 1024 * 1024,
max_matches_per_file: 0,
smart_case: true,
file_offset: 0,
-1
View File
@@ -98,7 +98,6 @@ mod scan;
#[doc(hidden)]
pub mod bigram_filter;
pub mod bigram_query;
pub mod constants;
mod constraints;
mod error;
mod score;
+10 -28
View File
@@ -7,7 +7,7 @@ use tracing::{error, info};
use crate::FileSync;
use crate::background_watcher::BackgroundWatcher;
use crate::bigram_filter::{build_bigram_index, sniff_binary_for_non_indexable};
use crate::bigram_filter::build_bigram_index;
use crate::error::Error;
use crate::file_picker::{BACKGROUND_THREAD_POOL, FFFMode};
use crate::git::GitStatusCache;
@@ -211,9 +211,8 @@ impl ScanJob {
// 3. Post-scan warmup + bigram build — runs in parallel with the
// git-status thread to overlap the two expensive phases.
// Always runs (even with both flags off) so binary-content files
// with unknown extensions get reclassified before user search hits.
if !signals.cancelled.load(Ordering::Acquire)
if (config.warmup || config.content_indexing)
&& !signals.cancelled.load(Ordering::Acquire)
&& let Some(snap) = snapshot.as_ref()
{
Self::run_post_scan(&shared_picker, &signals, &config, snap);
@@ -290,23 +289,20 @@ impl ScanJob {
config: &ScanConfig,
unsafe_snapshot: &crate::file_picker::PostScanUnsafeSnapshot,
) {
let Some(arena) = unsafe_snapshot
.arena // we are never touching overlays so this arena is always correct
let arena = unsafe_snapshot
.arena
.as_ref()
.map(|s| s.as_arena_ptr())
else {
tracing::error!("Failed to run post scan: arena is invalid");
return;
};
.unwrap_or(ArenaPtr::null());
let _budget: &ContentCacheBudget = &unsafe_snapshot.budget;
let files: &[crate::types::FileItem] = &unsafe_snapshot.files[..unsafe_snapshot.base_count];
if signals.cancelled.load(Ordering::Acquire) {
return;
}
if config.content_indexing {
let indexable_count = unsafe_snapshot.indexable_count.min(files.len());
let (indexable_files, non_indexable_files) = files.split_at(indexable_count);
let indexable_files = &files[..unsafe_snapshot.indexable_count.min(files.len())];
let index = build_bigram_index(indexable_files, &unsafe_snapshot.base_path, arena);
if let Ok(mut guard) = shared_picker.write()
@@ -314,23 +310,9 @@ impl ScanJob {
{
picker.set_bigram_index(index);
}
// Bigram only sniffs files <= MAX_INDEXABLE_FILE_SIZE; large
// unknown-extension binaries slip past it and would otherwise be
// grep-able as text. Cheap header sniff catches those.
if !signals.cancelled.load(Ordering::Acquire) {
sniff_binary_for_non_indexable(
non_indexable_files,
&unsafe_snapshot.base_path,
arena,
);
}
} else {
// this potentially a long running as we are not parallelizing it but it's okay
sniff_binary_for_non_indexable(files, &unsafe_snapshot.base_path, arena);
}
// TODO Skipped as potentially unsafe - figure this out later
// Skipped as potentially unsafe - figure this out later
// if config.warmup && !signals.cancelled.load(Ordering::Acquire) {
// warmup_mmaps(files, budget, &unsafe_snapshot.base_path, arena);
// }
+4 -5
View File
@@ -60,7 +60,7 @@ impl std::fmt::Debug for SimdChunk {
}
}
pub use crate::constants::PATH_BUF_SIZE;
pub const PATH_BUF_SIZE: usize = 4096;
/// Indices into a shared `SimdChunk` arena representing a file path.
///
@@ -295,11 +295,10 @@ pub(crate) struct ChunkedPathStoreBuilder {
impl ChunkedPathStoreBuilder {
pub fn new(estimated_files: usize) -> Self {
let est_chunks = estimated_files * INLINE_CHUNKS; // we know that most of repos will fit
// most paths into 64 = 16 * INLINE_CHUNKS
let est_chunks = estimated_files * 3;
Self {
arena: Vec::with_capacity(est_chunks),
chunk_dedup: AHashMap::with_capacity(est_chunks),
arena: Vec::with_capacity(est_chunks / 2),
chunk_dedup: AHashMap::with_capacity(est_chunks / 2),
}
}
+35 -80
View File
@@ -4,12 +4,9 @@ use std::path::{Path, PathBuf};
use std::sync::OnceLock;
use std::sync::atomic::{AtomicI32, AtomicU8, AtomicU64, AtomicUsize, Ordering};
#[cfg(not(target_os = "windows"))]
use crate::constants::{FRESH_MMAP_THRESHOLD, MMAP_THRESHOLD};
use crate::constants::{MAX_CACHED_CONTENT_BYTES, MAX_FFFILE_SIZE, PATH_BUF_SIZE};
use crate::constraints::Constrainable;
use crate::query_tracker::QueryMatchEntry;
use crate::simd_path::ArenaPtr;
use crate::simd_path::{ArenaPtr, PATH_BUF_SIZE};
use fff_query_parser::{FFFQuery, FuzzyQuery, Location};
/// Different sources of the string storage used by FFF
@@ -240,18 +237,6 @@ impl Clone for FileItem {
}
}
/// Single-block read used by the binary classifier. Most binaries reveal a
/// NUL byte within the first filesystem block, so 16 KB lets one read settle
/// the classification for typical files while keeping the scratch buffer
/// small enough to live on the stack.
pub const BINARY_CLASSIFICATION_CHUNK_SIZE: usize = 16 * 1024;
/// A file is treated as binary if any NUL byte appears in the scanned prefix.
#[inline]
pub(crate) fn detect_binary_content(content: &[u8]) -> bool {
memchr::memchr(0, content).is_some()
}
impl FileItem {
pub fn new_raw(
filename_start: u16,
@@ -514,38 +499,6 @@ impl FileItem {
}
}
/// Chunked classifier of the binary content of the file chunk by chunk
/// accepts path which to reuse the allocated buffer for absolute path read
pub(crate) fn detect_binary_per_byte(&self, path: &Path, chunk: &mut [u8]) {
if self.size == 0 {
return;
}
let Ok(mut file) = std::fs::OpenOptions::new()
.write(false)
.read(true)
.open(path)
else {
tracing::error!(path = ?path.display(), "Failed to open indexed file");
return;
};
loop {
match file.read(chunk) {
Ok(0) => break,
Err(e) => {
tracing::error!(?e, "Failed to read file chunk");
break;
}
Ok(n) => {
if detect_binary_content(&chunk[..n]) {
self.set_binary(true);
}
}
}
}
}
#[inline]
pub fn is_deleted(&self) -> bool {
self.flags.load(Ordering::Relaxed) & FileItemFlags::DELETED != 0
@@ -644,7 +597,13 @@ impl FileItem {
/// Returns a reference to a cached mmap of the file's contents.
///
/// SAFETY-CRITICAL: callers must hold the picker read lock for as long as the returned slice is in use.
/// SAFETY-CRITICAL: callers must hold the picker read lock for as long as
/// the returned slice is in use. The watcher mutates `FileItem` (including
/// `invalidate_mmap`) under the picker write lock, so the read lock is
/// what prevents UAF (`OnceLock` reset → `munmap`) and SIGBUS (in-place
/// truncate → access past new EOF). Detached background tasks (e.g. the
/// bigram builder running on `BACKGROUND_THREAD_POOL`) MUST NOT call this
/// — use `read_trimmed_into_buf` instead.
#[cfg(not(target_os = "windows"))]
pub(crate) fn get_cached_content(
&self,
@@ -656,6 +615,10 @@ impl FileItem {
return Some(content);
}
// Skip caching when mmap can't pay for itself. Files under one page
// worth of bytes waste kernel VM structures and a per-file syscall
// pair — the chunked `read_into_buf` fallback is cheaper for them
// and hits the OS page cache on repeat reads anyway.
if self.size < MMAP_THRESHOLD || self.size > budget.max_file_size {
return None;
}
@@ -691,20 +654,16 @@ impl FileItem {
#[inline]
pub(crate) fn get_content_for_search<'a>(
&'a self,
buf: &'a mut Vec<u8>,
#[cfg_attr(target_os = "windows", allow(unused_variables))] mmap_slot: &'a mut MmapSlot,
buf: &'a mut Vec<u8>, // we allow it to grow
arena: ArenaPtr,
base_path: &Path,
budget: &ContentCacheBudget,
) -> Option<&'a [u8]> {
#[cfg(not(target_os = "windows"))]
{
// Fast path: persistent cache hit (zero-copy). Safe here because
// grep callers hold the picker read lock for the lifetime of the
// returned slice — see [`Self::get_cached_content`] safety note.
if let Some(cached) = self.get_cached_content(arena, base_path, budget) {
return Some(cached);
}
// Fast path: persistent cache hit (zero-copy). Safe here because grep
// callers hold the picker read lock for the lifetime of the returned
// slice — see [`Self::get_cached_content`] safety note.
if let Some(cached) = self.get_cached_content(arena, base_path, budget) {
return Some(cached);
}
let max_file_size = budget.max_file_size;
@@ -712,34 +671,26 @@ impl FileItem {
return None;
}
// Slow path: read into the reusable buffer — open() + read_exact() + close().
// No mmap()/munmap() syscalls, no page table setup/teardown.
// We know the exact size so we use read_exact (1 read syscall) instead of
// read_to_end (2 read syscalls — one for data, one for EOF confirmation).
let abs = self.absolute_path(arena, base_path);
#[cfg(not(target_os = "windows"))]
if self.size >= FRESH_MMAP_THRESHOLD {
let file = std::fs::File::open(&abs).ok()?;
let mmap = unsafe { memmap2::Mmap::map(&file) }.ok()?;
let stored = mmap_slot.insert(mmap);
return Some(&stored[..]);
} else {
let _ = (mmap_slot, arena);
}
let len = self.size as usize;
buf.resize(len, 0);
let mut file = std::fs::File::open(&abs).ok()?;
file.read_exact(buf).ok()?;
Some(buf.as_slice())
}
}
/// Per-thread scratch slot owning a transient mmap returned from
/// [`FileItem::get_content_for_search`]. `Option<Mmap>` on Unix,
/// unit on Windows where mmap is unused.
#[cfg(not(target_os = "windows"))]
pub type MmapSlot = Option<memmap2::Mmap>;
#[cfg(target_os = "windows")]
pub type MmapSlot = ();
/// Files smaller than one page waste the remainder when mmapped.
/// Files smaller than one page waste the remainder when mmapped. Unused
/// on Windows where the persistent content cache is disabled.
#[cfg(all(not(target_os = "windows"), target_arch = "aarch64"))]
const MMAP_THRESHOLD: u64 = 16 * 1024;
#[cfg(all(not(target_os = "windows"), not(target_arch = "aarch64")))]
const MMAP_THRESHOLD: u64 = 4 * 1024;
impl Constrainable for FileItem {
#[inline]
@@ -856,6 +807,10 @@ impl Default for MixedItemRef<'_> {
}
}
const MAX_MMAP_FILE_SIZE: u64 = 10 * 1024 * 1024;
const MAX_CACHED_CONTENT_BYTES: u64 = 512 * 1024 * 1024;
#[derive(Debug)]
pub struct ContentCacheBudget {
pub max_files: usize,
@@ -870,7 +825,7 @@ impl ContentCacheBudget {
Self {
max_files: usize::MAX,
max_bytes: u64::MAX,
max_file_size: MAX_FFFILE_SIZE,
max_file_size: MAX_MMAP_FILE_SIZE,
cached_count: AtomicUsize::new(0),
cached_bytes: AtomicU64::new(0),
}
@@ -912,7 +867,7 @@ impl ContentCacheBudget {
Self {
max_files,
max_bytes,
max_file_size: MAX_FFFILE_SIZE,
max_file_size: MAX_MMAP_FILE_SIZE,
cached_count: AtomicUsize::new(0),
cached_bytes: AtomicU64::new(0),
}
Binary file not shown.
Binary file not shown.
-439
View File
@@ -302,445 +302,6 @@ fn plain_text_binary_files_are_skipped() {
assert!(result.files[0].relative_path(&picker).contains("text.txt"));
}
#[test]
fn binary_payload_after_long_ascii_header_is_detected() {
// Mimics formats like Radiance .hdr / Apple bplist / Adobe .ai where the
// first ~1KB is plain ASCII and the binary payload (NULs) starts later.
// The legacy 512-byte sniff missed these; the bigram-build memchr scan
// over the whole indexed buffer must catch them.
use fff_search::file_picker::FFFMode;
use fff_search::{SharedFilePicker, SharedFrecency};
use std::time::Duration;
let tmp = TempDir::new().unwrap();
let base = tmp.path();
let mut content = Vec::new();
// 1 KiB of plain ASCII header — escapes any small fixed-window NUL sniff.
content.extend(std::iter::repeat_n(b'A', 1024));
content.extend_from_slice(b"\nmatch this text\n");
// Binary payload: NUL bytes that prove the file is not text.
content.extend(std::iter::repeat_n(0u8, 256));
content.extend_from_slice(b"\nmatch this text\n");
// Use a *text* extension so the scan-time heuristic does NOT pre-flag it.
// Only the bigram-time content scan can mark it binary.
fs::write(base.join("header.txt"), &content).unwrap();
fs::write(base.join("plain.txt"), b"match this text\n").unwrap();
let shared_picker = SharedFilePicker::default();
let shared_frecency = SharedFrecency::default();
FilePicker::new_with_shared_state(
shared_picker.clone(),
shared_frecency.clone(),
FilePickerOptions {
base_path: base.to_string_lossy().to_string(),
enable_mmap_cache: false,
enable_content_indexing: true,
mode: FFFMode::Neovim,
watch: false,
..Default::default()
},
)
.expect("Failed to create FilePicker");
let deadline = std::time::Instant::now() + Duration::from_secs(10);
loop {
std::thread::sleep(Duration::from_millis(25));
let ready = shared_picker
.read()
.ok()
.and_then(|g| {
g.as_ref()
.map(|p| !p.is_scan_active() && p.bigram_index().is_some())
})
.unwrap_or(false);
if ready {
break;
}
assert!(
std::time::Instant::now() < deadline,
"Timed out waiting for bigram build"
);
}
let guard = shared_picker.read().unwrap();
let picker = guard.as_ref().unwrap();
let was_flagged = picker
.get_files()
.iter()
.any(|f| f.relative_path(picker).contains("header.txt") && f.is_binary());
assert!(
was_flagged,
"header.txt with NULs past 512 bytes must be flagged binary by the whole-buffer memchr scan"
);
let parsed = parse_grep_query("match this text");
let result = picker.grep(&parsed, &plain_opts());
assert_eq!(
result.files.len(),
1,
"only plain.txt should be searched; header.txt must be skipped as binary"
);
assert!(
result.files[0].relative_path(picker).contains("plain.txt"),
"the only match should come from plain.txt"
);
}
#[test]
fn unknown_extension_binary_added_after_scan_is_reclassified() {
// The initial-scan path runs detect_binary_content as part of bigram build,
// but the watcher path used to fall back to extension-only triage and
// missed binary files with unknown extensions like `.codex`.
use fff_search::file_picker::FFFMode;
use fff_search::{SharedFilePicker, SharedFrecency};
use std::time::Duration;
let tmp = TempDir::new().unwrap();
let base = tmp.path();
// Seed one tracked text file so the initial scan has something to work with.
fs::write(base.join("seed.txt"), b"seed\n").unwrap();
let shared_picker = SharedFilePicker::default();
let shared_frecency = SharedFrecency::default();
FilePicker::new_with_shared_state(
shared_picker.clone(),
shared_frecency.clone(),
FilePickerOptions {
base_path: base.to_string_lossy().to_string(),
enable_mmap_cache: false,
enable_content_indexing: true,
mode: FFFMode::Neovim,
watch: false,
..Default::default()
},
)
.expect("Failed to create FilePicker");
let deadline = std::time::Instant::now() + Duration::from_secs(10);
loop {
std::thread::sleep(Duration::from_millis(25));
let ready = shared_picker
.read()
.ok()
.and_then(|g| {
g.as_ref()
.map(|p| !p.is_scan_active() && p.bigram_index().is_some())
})
.unwrap_or(false);
if ready {
break;
}
assert!(
std::time::Instant::now() < deadline,
"Timed out waiting for bigram build"
);
}
// Drop the file on disk after indexing finished, then announce it through
// the watcher entry point. `.codex` is intentionally not in the extension
// allow-list — only a content sniff can flag it.
let mut payload = vec![0x03u8, 0x00, 0x04, 0x05];
payload.extend(std::iter::repeat_n(0u8, 256));
let new_path = base.join("snapshot.codex");
fs::write(&new_path, &payload).unwrap();
{
let mut guard = shared_picker.write().unwrap();
let picker = guard.as_mut().unwrap();
assert!(
picker.handle_create_or_modify(&new_path).is_some(),
"handle_create_or_modify must accept the new file"
);
}
let guard = shared_picker.read().unwrap();
let picker = guard.as_ref().unwrap();
let was_flagged = picker
.get_files()
.iter()
.any(|f| f.relative_path(picker).contains("snapshot.codex") && f.is_binary());
assert!(
was_flagged,
"snapshot.codex must be flagged binary when added via the watcher path"
);
}
#[test]
fn text_file_modified_to_binary_is_reclassified() {
// A file that started life as text and later got rewritten with NUL bytes
// (e.g. a generator overwrote a .log) must lose its text classification.
use fff_search::file_picker::FFFMode;
use fff_search::{SharedFilePicker, SharedFrecency};
use std::time::Duration;
let tmp = TempDir::new().unwrap();
let base = tmp.path();
// Start as plain text with a known extension.
fs::write(base.join("notes.txt"), b"hello world\n").unwrap();
let shared_picker = SharedFilePicker::default();
let shared_frecency = SharedFrecency::default();
FilePicker::new_with_shared_state(
shared_picker.clone(),
shared_frecency.clone(),
FilePickerOptions {
base_path: base.to_string_lossy().to_string(),
enable_mmap_cache: false,
enable_content_indexing: true,
mode: FFFMode::Neovim,
watch: false,
..Default::default()
},
)
.expect("Failed to create FilePicker");
let deadline = std::time::Instant::now() + Duration::from_secs(10);
loop {
std::thread::sleep(Duration::from_millis(25));
let ready = shared_picker
.read()
.ok()
.and_then(|g| {
g.as_ref()
.map(|p| !p.is_scan_active() && p.bigram_index().is_some())
})
.unwrap_or(false);
if ready {
break;
}
assert!(
std::time::Instant::now() < deadline,
"Timed out waiting for bigram build"
);
}
// Sanity: it's text right now.
{
let guard = shared_picker.read().unwrap();
let picker = guard.as_ref().unwrap();
let is_text = picker
.get_files()
.iter()
.any(|f| f.relative_path(picker).contains("notes.txt") && !f.is_binary());
assert!(is_text, "notes.txt should start as text");
}
// Overwrite with binary content and replay through the watcher entry point.
// Bump mtime so update_metadata records it as a real change.
std::thread::sleep(Duration::from_secs(1));
let mut payload = b"header text\n".to_vec();
payload.extend(std::iter::repeat_n(0u8, 256));
fs::write(base.join("notes.txt"), &payload).unwrap();
{
let mut guard = shared_picker.write().unwrap();
let picker = guard.as_mut().unwrap();
assert!(
picker
.handle_create_or_modify(base.join("notes.txt"))
.is_some(),
"handle_create_or_modify must succeed for the modify case"
);
}
let guard = shared_picker.read().unwrap();
let picker = guard.as_ref().unwrap();
let now_binary = picker
.get_files()
.iter()
.any(|f| f.relative_path(picker).contains("notes.txt") && f.is_binary());
assert!(
now_binary,
"notes.txt must flip to binary after being overwritten with NULs"
);
}
#[test]
fn large_unknown_extension_binary_is_classified_at_scan_time() {
// Files larger than MAX_INDEXABLE_FILE_SIZE never enter build_bigram_index,
// so without a separate header sniff they default to is_binary=false and
// pollute grep results with NUL-laden lines (e.g. a committed ELF blob
// named `codex_view` with no extension).
use fff_search::file_picker::FFFMode;
use fff_search::grep::{GrepSearchOptions, parse_grep_query};
use fff_search::{SharedFilePicker, SharedFrecency};
use std::time::Duration;
let tmp = TempDir::new().unwrap();
let base = tmp.path();
// 3 MiB: above the 2 MiB bigram cap and below the 10 MiB grep cap.
// ELF-like header with NULs at the very start, then ASCII filler so a
// grep for "match this text" would otherwise return polluted lines.
let mut blob = Vec::new();
blob.extend_from_slice(b"\x7fELF\x02\x01\x01\x00");
blob.extend(std::iter::repeat_n(0u8, 256));
blob.extend_from_slice(b"\nmatch this text\n");
blob.extend(std::iter::repeat_n(b'A', 3 * 1024 * 1024));
blob.extend_from_slice(b"\nmatch this text\n");
fs::write(base.join("codex_view"), &blob).unwrap();
fs::write(base.join("plain.txt"), b"match this text\n").unwrap();
let shared_picker = SharedFilePicker::default();
let shared_frecency = SharedFrecency::default();
FilePicker::new_with_shared_state(
shared_picker.clone(),
shared_frecency.clone(),
FilePickerOptions {
base_path: base.to_string_lossy().to_string(),
enable_mmap_cache: false,
enable_content_indexing: true,
mode: FFFMode::Neovim,
watch: false,
..Default::default()
},
)
.expect("Failed to create FilePicker");
let deadline = std::time::Instant::now() + Duration::from_secs(15);
loop {
std::thread::sleep(Duration::from_millis(25));
let ready = shared_picker
.read()
.ok()
.and_then(|g| {
g.as_ref()
.map(|p| !p.is_scan_active() && p.bigram_index().is_some())
})
.unwrap_or(false);
if ready {
break;
}
assert!(
std::time::Instant::now() < deadline,
"Timed out waiting for bigram build"
);
}
let guard = shared_picker.read().unwrap();
let picker = guard.as_ref().unwrap();
let was_flagged = picker
.get_files()
.iter()
.any(|f| f.relative_path(picker).contains("codex_view") && f.is_binary());
assert!(
was_flagged,
"large no-extension binary must be flagged via the header sniff"
);
let parsed = parse_grep_query("match this text");
let opts = GrepSearchOptions {
max_file_size: 10 * 1024 * 1024,
..plain_opts()
};
let result = picker.grep(&parsed, &opts);
assert_eq!(
result.files.len(),
1,
"only plain.txt should be searched; codex_view must be skipped as binary"
);
assert!(
result.files[0].relative_path(picker).contains("plain.txt"),
"the only match should come from plain.txt"
);
}
#[test]
fn large_binary_with_nuls_past_header_is_classified() {
// Guards the streaming sniff: a >2 MB file that is pure ASCII well past any
// fixed header window (the old code only checked the first 8 KB) but has
// NULs deeper in. Grep reads the whole file up to max_file_size, so the
// detector must scan the same range or the binary tail leaks as "text".
use fff_search::file_picker::FFFMode;
use fff_search::grep::{GrepSearchOptions, parse_grep_query};
use fff_search::{SharedFilePicker, SharedFrecency};
use std::time::Duration;
let tmp = TempDir::new().unwrap();
let base = tmp.path();
// 1 MiB of clean ASCII (with a grep marker) — dwarfs any header sniff —
// then NUL bytes, keeping the total above the 2 MiB non-indexable cap.
let mut blob = Vec::new();
blob.extend_from_slice(b"match this text\n");
blob.extend(std::iter::repeat_n(b'A', 1024 * 1024));
blob.extend_from_slice(b"match this text\n");
blob.extend(std::iter::repeat_n(0u8, 1024 * 1024 + 4096)); // NULs start ~1 MiB in
blob.extend_from_slice(b"match this text\n");
assert!(blob.len() > 2 * 1024 * 1024);
fs::write(base.join("late_nul.dat"), &blob).unwrap();
fs::write(base.join("plain.txt"), b"match this text\n").unwrap();
let shared_picker = SharedFilePicker::default();
let shared_frecency = SharedFrecency::default();
FilePicker::new_with_shared_state(
shared_picker.clone(),
shared_frecency.clone(),
FilePickerOptions {
base_path: base.to_string_lossy().to_string(),
enable_mmap_cache: false,
enable_content_indexing: true,
mode: FFFMode::Neovim,
watch: false,
..Default::default()
},
)
.expect("Failed to create FilePicker");
let deadline = std::time::Instant::now() + Duration::from_secs(15);
loop {
std::thread::sleep(Duration::from_millis(25));
let ready = shared_picker
.read()
.ok()
.and_then(|g| {
g.as_ref()
.map(|p| !p.is_scan_active() && p.bigram_index().is_some())
})
.unwrap_or(false);
if ready {
break;
}
assert!(
std::time::Instant::now() < deadline,
"Timed out waiting for bigram build"
);
}
let guard = shared_picker.read().unwrap();
let picker = guard.as_ref().unwrap();
let flagged = picker
.get_files()
.iter()
.any(|f| f.relative_path(picker).contains("late_nul.dat") && f.is_binary());
assert!(
flagged,
"NULs past the 8 KB header window must still be detected by the streaming scan"
);
let parsed = parse_grep_query("match this text");
let opts = GrepSearchOptions {
max_file_size: 10 * 1024 * 1024,
..plain_opts()
};
let result = picker.grep(&parsed, &opts);
assert_eq!(
result.files.len(),
1,
"only plain.txt should match; late_nul.dat must be skipped as binary"
);
assert!(result.files[0].relative_path(picker).contains("plain.txt"));
}
#[test]
fn plain_text_max_matches_per_file() {
let tmp = TempDir::new().unwrap();
@@ -1,153 +0,0 @@
//! Real-world binary fixture regression.
//!
//! Reproduces the exact bug chain we hit with `codex_view` (4.5 MB ELF, no
//! extension) and `codex_view.codex` (127 KB, unknown extension): both are
//! binary by content but slip past extension-only triage, so a plain grep
//! used to surface their NUL-laden bytes as "text" matches.
//!
//! The fixtures live in `tests/fixtures/binaries/`. `MARKER` is a string that
//! is present (as raw bytes) in BOTH binaries — the test first asserts that,
//! then drops the two binaries plus a single plain-text file containing the
//! same marker into a closed temp dir and greps for it. Only the text file may
//! come back; if binary detection ever regresses, a binary file re-enters the
//! results and this test fails.
use std::fs;
use std::path::PathBuf;
use std::time::Duration;
use fff_search::file_picker::{FFFMode, FilePicker};
use fff_search::grep::{GrepMode, GrepSearchOptions, parse_grep_query};
use fff_search::{FilePickerOptions, SharedFilePicker, SharedFrecency};
const MARKER: &str = "__jai_runtime_init";
fn fixtures_dir() -> PathBuf {
PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/binaries")
}
fn plain_opts() -> GrepSearchOptions {
GrepSearchOptions {
max_file_size: 10 * 1024 * 1024,
max_matches_per_file: 200,
smart_case: true,
file_offset: 0,
page_limit: 200,
mode: GrepMode::PlainText,
time_budget_ms: 0,
before_context: 0,
after_context: 0,
classify_definitions: false,
trim_whitespace: false,
abort_signal: None,
}
}
#[test]
fn real_binary_fixtures_are_detected_and_excluded_from_grep() {
let fixtures = fixtures_dir();
let large = fixtures.join("codex_view"); // 4.5 MB ELF, no extension (> 2 MB)
let small = fixtures.join("codex_view.codex"); // 127 KB, unknown extension (< 2 MB)
assert!(
large.exists() && small.exists(),
"missing binary fixtures in {}",
fixtures.display()
);
// Both fixtures must really contain the marker bytes, otherwise the grep
// exclusion assertion below would be vacuous.
let large_bytes = fs::read(&large).unwrap();
let small_bytes = fs::read(&small).unwrap();
assert!(
contains_subslice(&large_bytes, MARKER.as_bytes()),
"fixture codex_view no longer contains the marker {MARKER:?}"
);
assert!(
contains_subslice(&small_bytes, MARKER.as_bytes()),
"fixture codex_view.codex no longer contains the marker {MARKER:?}"
);
// Sanity on the size split that drives the two distinct code paths.
assert!(
large_bytes.len() > 2 * 1024 * 1024,
"codex_view must exceed the 2 MB non-indexable threshold"
);
assert!(
small_bytes.len() < 2 * 1024 * 1024,
"codex_view.codex must stay under the 2 MB bigram cap"
);
// Closed environment: the two real binaries + one plain-text file that
// legitimately contains the marker.
let tmp = tempfile::TempDir::new().unwrap();
let base = tmp.path();
fs::copy(&large, base.join("codex_view")).unwrap();
fs::copy(&small, base.join("codex_view.codex")).unwrap();
fs::write(
base.join("marker.txt"),
format!("the only legitimate hit lives here: {MARKER}\n"),
)
.unwrap();
let shared_picker = SharedFilePicker::default();
let shared_frecency = SharedFrecency::default();
FilePicker::new_with_shared_state(
shared_picker.clone(),
shared_frecency.clone(),
FilePickerOptions {
base_path: base.to_string_lossy().to_string(),
enable_mmap_cache: false,
enable_content_indexing: true,
mode: FFFMode::Neovim,
watch: false,
..Default::default()
},
)
.expect("failed to create FilePicker");
shared_picker.wait_for_indexing_complete(Duration::from_secs(5));
let guard = shared_picker.read().unwrap();
let picker = guard.as_ref().unwrap();
// Both binaries must be classified binary.
for name in ["codex_view", "codex_view.codex"] {
let flagged = picker
.get_files()
.iter()
.any(|f| f.relative_path(picker).ends_with(name) && f.is_binary());
assert!(flagged, "{name} must be flagged is_binary");
}
// we need to make sure that marker.txt ONLY can match as we have to match
// grep as binaries are excluded from the matching process
let parsed = parse_grep_query(MARKER);
let result = picker.grep(&parsed, &plain_opts());
let matched: Vec<String> = result
.files
.iter()
.map(|f| f.relative_path(picker))
.collect();
assert_eq!(
result.files.len(),
1,
"exactly one file should match {MARKER:?}, got: {matched:?}"
);
assert!(
matched[0].ends_with("marker.txt"),
"the only match must be marker.txt, got {:?}",
matched[0]
);
}
/// Tiny substring search over raw bytes (the marker may be surrounded by NULs).
fn contains_subslice(haystack: &[u8], needle: &[u8]) -> bool {
if needle.is_empty() || haystack.len() < needle.len() {
return false;
}
haystack
.windows(needle.len())
.any(|window| window == needle)
}
+1 -1
View File
@@ -3,7 +3,7 @@ name = "fff-grep"
description = "File grepping logic for fff"
license = "MIT"
authors = ["Dmitriy Kovalenko <dmtr.kovalenko@outlok.com>"]
version = "0.8.4"
version = "0.8.1"
edition = "2024"
[dependencies]
+4 -4
View File
@@ -1,6 +1,6 @@
[package]
name = "fff-mcp"
version = "0.8.4"
version = "0.8.1"
edition = "2024"
description = "MCP server for FFF file finder - drop-in replacement for AI code assistant search tools"
license = "MIT"
@@ -14,10 +14,10 @@ default = ["zlob"]
zlob = ["fff/zlob"]
[dependencies]
fff = { package = "fff-search", path = "../fff-core", default-features = false , version = "0.8.4" }
fff-query-parser = { path = "../fff-query-parser", default-features = false , version = "0.8.4" }
fff = { package = "fff-search", path = "../fff-core", default-features = false , version = "0.8.1" }
fff-query-parser = { path = "../fff-query-parser", default-features = false , version = "0.8.1" }
mimalloc = { workspace = true }
rmcp = { version = "1.7.0", features = ["server", "transport-io"] }
rmcp = { version = "1.1.0", features = ["server", "transport-io"] }
schemars = "1.0"
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
+3
View File
@@ -14,6 +14,7 @@ use fff::grep::{GrepMode, GrepSearchOptions, has_regex_metacharacters};
use fff::types::{FileItem, PaginationArgs};
use fff::{FuzzySearchOptions, QueryParser, SharedFilePicker, SharedFrecency};
use fff_query_parser::AiGrepConfig;
use rmcp::handler::server::router::tool::ToolRouter;
use rmcp::handler::server::wrapper::Parameters;
use rmcp::model::*;
use rmcp::{ServerHandler, schemars, tool, tool_handler, tool_router};
@@ -185,6 +186,7 @@ pub struct FffServer {
frecency: SharedFrecency,
cursor_store: Arc<Mutex<CursorStore>>,
update_notice_sent: Arc<AtomicBool>,
tool_router: ToolRouter<Self>,
}
impl FffServer {
@@ -194,6 +196,7 @@ impl FffServer {
frecency,
cursor_store: Arc::new(Mutex::new(CursorStore::new())),
update_notice_sent: Arc::new(AtomicBool::new(false)),
tool_router: Self::tool_router(),
}
}
+3 -3
View File
@@ -1,6 +1,6 @@
[package]
name = "fff-nvim"
version = "0.8.4"
version = "0.8.1"
edition = "2024"
[lib]
@@ -61,10 +61,10 @@ ahash = { workspace = true }
tracing = { workspace = true }
# Local crates
fff = { package = "fff-search", path = "../fff-core", version = "0.8.4", features = [
fff = { package = "fff-search", path = "../fff-core", version = "0.8.1", features = [
"mimalloc-collect",
] }
fff-query-parser = { path = "../fff-query-parser", version = "0.8.4" }
fff-query-parser = { path = "../fff-query-parser", version = "0.8.1" }
chrono = { version = "0.4", features = ["serde"] }
ctrlc = "3.4.2"
git2 = { workspace = true }
-5
View File
@@ -239,11 +239,6 @@ impl IntoLua for GrepResultLua<'_> {
item.set("line_number", m.line_number)?;
item.set("col", m.col)?;
item.set("byte_offset", m.byte_offset)?;
// There is a little race window when fff can return matches inside of a non-binary
// classified entities, the window is minimal but it errors out neovim so guard it
let is_binary_content = m.line_content.as_bytes().contains(&0u8);
item.set("is_binary_content", is_binary_content)?;
item.set("line_content", m.line_content.as_str())?;
// Match byte ranges within line_content
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "fff-query-parser"
version = "0.8.4"
version = "0.8.1"
edition = "2024"
description = "Query parser for fff file finder - includes specific syntax for various constraints like globs, extensions, regex etc"
license = "MIT"
+4 -1
View File
@@ -21,8 +21,11 @@ param(
[string]$Version = $env:FFF_MCP_VERSION,
[string]$InstallDir = $env:FFF_MCP_INSTALL_DIR,
[ValidateSet('User', 'Profile', 'None')]
[string]$PathScope = $(if ($env:FFF_MCP_PATH_SCOPE) { $env:FFF_MCP_PATH_SCOPE } else { 'User' })
[string]$PathScope
)
if (-not $PathScope) {
$PathScope = if ($env:FFF_MCP_PATH_SCOPE) { $env:FFF_MCP_PATH_SCOPE } else { 'User' }
}
$ErrorActionPreference = 'Stop'
-10
View File
@@ -57,7 +57,6 @@ local M = {}
--- @field time_budget_ms number
--- @field modes string[]
--- @field trim_whitespace boolean
--- @field location_format string
--- @class FffConfig
--- @field base_path string
@@ -325,11 +324,6 @@ local function init()
file_info_match_type = 'FFFFileInfoMatchType', -- match_type label (bold)
file_info_score_pos = 'FFFFileInfoScorePos', -- Positive score components
file_info_score_neg = 'FFFFileInfoScoreNeg', -- Negative score components / penalties
-- Per-window 'winhighlight' overrides. When nil, falls back to a combination of `normal`, `border`, and `title` above.
-- Accepts either a string applied to every picker window, or a table with optional `prompt`, `list`, `preview`, `file_info` keys.
-- Example: `winhl = 'Normal:NormalFloat,FloatBorder:FloatBorder,FloatTitle:Title'`
-- Example: `winhl = { prompt = 'Normal:Pmenu,...', list = 'Normal:NormalFloat,...' }`
winhl = nil,
},
-- Store file open frecency
frecency = {
@@ -377,10 +371,6 @@ local function init()
time_budget_ms = 150, -- Max search time in ms per call (prevents UI freeze, 0 = no limit)
modes = { 'plain', 'regex', 'fuzzy' }, -- Available grep modes and their cycling order
trim_whitespace = false, -- Strip leading whitespace from matched lines (useful for cleaner display)
-- Format string for the line/column location prefix in grep results.
-- Uses vim's printf-style format: %d placeholders for line and column (1-based).
-- Default ':%d:%d' renders as ':356:1'. Use ':%d' for line-only ':356'.
location_format = ':%d:%d',
},
}
+4 -24
View File
@@ -41,23 +41,13 @@ end
---@param item table Grep match item
---@param ctx table Render context
---@return string The match line string
local function format_location(item, ctx)
local fmt = (ctx.config and ctx.config.grep and ctx.config.grep.location_format) or ':%d:%d'
local ok, str = pcall(string.format, fmt, item.line_number or 0, (item.col or 0) + 1)
if not ok then str = string.format(':%d:%d', item.line_number or 0, (item.col or 0) + 1) end
return str
end
local BINARY_PLACEHOLDER = '<binary content>'
local function render_match_line(item, ctx)
local location = format_location(item, ctx)
local location = string.format(':%d:%d', item.line_number or 0, (item.col or 0) + 1)
local separator = ' '
-- vim.json.decode may return Blobs for strings with NUL bytes; coerce to string.
local raw_content = item.line_content
if type(raw_content) ~= 'string' then raw_content = raw_content and tostring(raw_content) or '' end
local content = raw_content
if item.is_binary_content then content = BINARY_PLACEHOLDER end
-- Indent + location + separator + content
local indent = ' '
@@ -115,7 +105,7 @@ local function apply_match_highlights(item, ctx, item_idx, buf, ns_id, row, line
end
-- 2. Location (:line:col) dimmed — use extmark with priority so it layers with cursor
local location_str = format_location(item, ctx)
local location_str = string.format(':%d:%d', item.line_number or 0, (item.col or 0) + 1)
local loc_start = indent
local loc_end = loc_start + #location_str
if loc_end <= #line_content then
@@ -141,17 +131,7 @@ local function apply_match_highlights(item, ctx, item_idx, buf, ns_id, row, line
-- Priority 120: above CursorLine (100) so syntax is visible on cursor line,
-- below IncSearch match ranges (200) so search matches take precedence.
local content_start = sep_end
if item.is_binary_content then
local content_end = content_start + #BINARY_PLACEHOLDER
if content_end <= #line_content then
pcall(vim.api.nvim_buf_set_extmark, buf, ns_id, row, content_start, {
end_col = content_end,
hl_group = 'Comment',
priority = 150,
})
end
elseif item._trimmed_content and item.name then
if item._trimmed_content and item.name then
-- Resolve language once per file group (cache on the render context)
ctx._ts_lang_cache = ctx._ts_lang_cache or {}
local lang = ctx._ts_lang_cache[item.name]
@@ -179,7 +159,7 @@ local function apply_match_highlights(item, ctx, item_idx, buf, ns_id, row, line
-- 5. Match ranges highlighted with IncSearch
-- Use extmarks with priority > cursor line (100) so IncSearch renders
-- properly on the selected line instead of being overridden by CursorLine.
if item.match_ranges and not item.is_binary_content then
if item.match_ranges then
for _, range in ipairs(item.match_ranges) do
local raw_start = range[1] or 0
local raw_end = range[2] or 0
+1 -2
View File
@@ -223,7 +223,6 @@ end
--- @param list_buf number List buffer handle
--- @param list_win number List window handle
--- @param ns_id number Highlight namespace
--- @return table<number, ItemLineMapping> item_to_lines
--- @return number|nil separator_line 1-based buffer line of the separator (post-padding), nil if none
function M.render(ctx, list_buf, list_win, ns_id)
local lines, item_to_lines, separator_line = generate_item_lines(ctx)
@@ -252,7 +251,7 @@ function M.render(ctx, list_buf, list_win, ns_id)
end
end
return item_to_lines, separator_line
return separator_line
end
return M
+38 -119
View File
@@ -47,7 +47,7 @@ M.state = {
cursor = 1,
top = 1,
query = '',
line_to_item = {},
item_line_map = {},
location = nil, -- Current location from search results
-- History cycling state
@@ -113,17 +113,6 @@ M.state = {
suggestion_source = nil,
}
function M.resolve_winhl(kind)
local hl = M.state.config.hl
local winhl = hl.winhl
local default_winhl = string.format('Normal:%s,FloatBorder:%s,FloatTitle:%s', hl.normal, hl.border, hl.title)
if winhl == nil then return default_winhl end
if type(winhl) == 'string' then return winhl end
if type(winhl) == 'table' then return winhl[kind] or default_winhl end
return default_winhl
end
local function open_preview(win_cfg)
if not win_cfg then return end
if M.state.preview_win and vim.api.nvim_win_is_valid(M.state.preview_win) then return end
@@ -139,7 +128,8 @@ local function open_preview(win_cfg)
M.state.preview_win = vim.api.nvim_open_win(M.state.preview_buf, false, win_cfg)
local win_hl = M.resolve_winhl('preview')
local hl = M.state.config.hl
local win_hl = string.format('Normal:%s,FloatBorder:%s,FloatTitle:%s', hl.normal, hl.border, hl.title)
local cursorlineopt = utils.resolve_config_value(
preview_config.cursorlineopt,
vim.o.columns,
@@ -279,9 +269,8 @@ function M.setup_buffers()
end
function M.setup_windows()
local prompt_win_hl = M.resolve_winhl('prompt')
local list_win_hl = M.resolve_winhl('list')
local file_info_win_hl = M.resolve_winhl('file_info')
local hl = M.state.config.hl
local win_hl = string.format('Normal:%s,FloatBorder:%s,FloatTitle:%s', hl.normal, hl.border, hl.title)
vim.api.nvim_set_option_value('wrap', false, { win = M.state.input_win })
vim.api.nvim_set_option_value('cursorline', false, { win = M.state.input_win })
@@ -289,7 +278,7 @@ function M.setup_windows()
vim.api.nvim_set_option_value('relativenumber', false, { win = M.state.input_win })
vim.api.nvim_set_option_value('signcolumn', 'no', { win = M.state.input_win })
vim.api.nvim_set_option_value('foldcolumn', '0', { win = M.state.input_win })
vim.api.nvim_set_option_value('winhighlight', prompt_win_hl, { win = M.state.input_win })
vim.api.nvim_set_option_value('winhighlight', win_hl, { win = M.state.input_win })
vim.api.nvim_set_option_value('wrap', false, { win = M.state.list_win })
vim.api.nvim_set_option_value('cursorline', false, { win = M.state.list_win })
@@ -297,7 +286,7 @@ function M.setup_windows()
vim.api.nvim_set_option_value('relativenumber', false, { win = M.state.list_win })
vim.api.nvim_set_option_value('signcolumn', 'yes:1', { win = M.state.list_win }) -- Enable signcolumn for git status borders
vim.api.nvim_set_option_value('foldcolumn', '0', { win = M.state.list_win })
vim.api.nvim_set_option_value('winhighlight', list_win_hl, { win = M.state.list_win })
vim.api.nvim_set_option_value('winhighlight', win_hl, { win = M.state.list_win })
if M.state.file_info_win and vim.api.nvim_win_is_valid(M.state.file_info_win) then
vim.api.nvim_set_option_value('wrap', false, { win = M.state.file_info_win })
@@ -306,7 +295,7 @@ function M.setup_windows()
vim.api.nvim_set_option_value('relativenumber', false, { win = M.state.file_info_win })
vim.api.nvim_set_option_value('signcolumn', 'no', { win = M.state.file_info_win })
vim.api.nvim_set_option_value('foldcolumn', '0', { win = M.state.file_info_win })
vim.api.nvim_set_option_value('winhighlight', file_info_win_hl, { win = M.state.file_info_win })
vim.api.nvim_set_option_value('winhighlight', win_hl, { win = M.state.file_info_win })
end
local picker_group = vim.api.nvim_create_augroup('fff_picker_focus', { clear = true })
@@ -396,36 +385,6 @@ function M.focus_preview_win()
vim.api.nvim_set_current_win(M.state.preview_win)
end
local function handle_mouse_click_or_fallback(action, fallback)
local pos = vim.fn.getmousepos()
if M.state.active and pos.winid == M.state.list_win then
local item_idx = M.state.line_to_item[pos.line]
if not item_idx then return '' end
vim.schedule(function()
if not M.state.active then return end
if not M.state.filtered_items[item_idx] then return end
if M.state.cursor ~= item_idx then
M.state.cursor = item_idx
M.render_list()
if M.state.mode == 'grep' or M.state.suggestion_source == 'grep' then
M.update_preview_smart()
else
M.update_preview()
end
M.update_status()
end
if action then M.select(action) end
end)
return ''
end
return fallback
end
local function move_list_cursor(direction)
if not M.state.active then return end
@@ -483,7 +442,6 @@ function M.setup_keymaps()
set_keymap('i', keymaps.cycle_forward_query, M.cycle_forward_query, input_opts)
set_keymap('n', 'j', M.move_down, input_opts)
set_keymap('n', 'k', M.move_up, input_opts)
set_keymap('n', 'q', M.close, input_opts)
set_keymap('n', keymaps.focus_list, M.focus_list_win, input_opts)
set_keymap('n', keymaps.focus_preview, M.focus_preview_win, input_opts)
@@ -505,20 +463,6 @@ function M.setup_keymaps()
set_keymap({ 'i', 'n' }, keymaps.send_to_quickfix, M.send_to_quickfix, input_opts)
set_keymap({ 'i', 'n' }, keymaps.cycle_grep_modes, M.cycle_grep_modes, input_opts)
local input_mouse_opts = vim.tbl_extend('force', input_opts, { expr = true, replace_keycodes = true })
set_keymap(
{ 'i', 'n' },
'<LeftMouse>',
function() return handle_mouse_click_or_fallback(nil, '<LeftMouse>') end,
input_mouse_opts
)
set_keymap(
{ 'i', 'n' },
'<2-LeftMouse>',
function() return handle_mouse_click_or_fallback('edit', '<2-LeftMouse>') end,
input_mouse_opts
)
-- List buffer
set_keymap('n', keymaps.close, M.close, list_opts)
set_keymap('n', 'q', M.close, list_opts)
@@ -536,20 +480,6 @@ function M.setup_keymaps()
set_keymap('n', keymaps.toggle_select, M.toggle_select, list_opts)
set_keymap('n', keymaps.send_to_quickfix, M.send_to_quickfix, list_opts)
local list_mouse_opts = vim.tbl_extend('force', list_opts, { expr = true, replace_keycodes = true })
set_keymap(
'n',
'<LeftMouse>',
function() return handle_mouse_click_or_fallback(nil, '<LeftMouse>') end,
list_mouse_opts
)
set_keymap(
'n',
'<2-LeftMouse>',
function() return handle_mouse_click_or_fallback('edit', '<2-LeftMouse>') end,
list_mouse_opts
)
-- Preview buffer
if M.state.preview_buf then
local preview_opts = { buffer = M.state.preview_buf, noremap = true, silent = true }
@@ -1313,21 +1243,11 @@ function M.render_list()
local ctx = build_render_context()
if M.state.mode == 'grep' and #ctx.items == 0 then
M.state.line_to_item = {}
render_grep_empty_state(ctx)
return
end
local item_to_lines, separator_line = list_renderer.render(ctx, M.state.list_buf, M.state.list_win, M.state.ns_id)
local line_to_item = {}
for item_idx, mapping in pairs(item_to_lines) do
for line = mapping.first, mapping.last do
line_to_item[line] = item_idx
end
end
M.state.line_to_item = line_to_item
local separator_line = list_renderer.render(ctx, M.state.list_buf, M.state.list_win, M.state.ns_id)
-- For bottom prompt, always ensure content is anchored at the bottom after rendering
-- This prevents results from appearing in the middle when there are few items
if ctx.prompt_position == 'bottom' then scroll_to_bottom() end
@@ -2149,40 +2069,39 @@ function M.select(action)
vim.cmd('stopinsert')
M.close()
-- Defer file open past picker float teardown. Without this, foldexpr is not
-- recomputed on the new window (folds appear missing) on some platforms.
vim.schedule(function()
if action == 'edit' then
local current_win = vim.api.nvim_get_current_win()
local current_buf = vim.api.nvim_get_current_buf()
local current_buftype = vim.api.nvim_get_option_value('buftype', { buf = current_buf })
local current_buf_modifiable = vim.api.nvim_get_option_value('modifiable', { buf = current_buf })
local current_winfixbuf = window_has_winfixbuf(current_win)
if action == 'edit' then
local current_win = vim.api.nvim_get_current_win()
local current_buf = vim.api.nvim_get_current_buf()
local current_buftype = vim.api.nvim_get_option_value('buftype', { buf = current_buf })
local current_buf_modifiable = vim.api.nvim_get_option_value('modifiable', { buf = current_buf })
local current_winfixbuf = window_has_winfixbuf(current_win)
-- If the current window can't host a new buffer (special buftype, non-modifiable,
-- or 'winfixbuf' locking it), retarget a suitable window or fall back to a split.
-- Without this, :edit raises E1513 ("Cannot switch buffer. 'winfixbuf' is enabled")
-- whenever the picker is invoked from a window pinned via :h winfixbuf.
local opened_via_split = false
if current_buftype ~= '' or not current_buf_modifiable or current_winfixbuf then
local suitable_win = find_suitable_window()
if suitable_win then
vim.api.nvim_set_current_win(suitable_win)
elseif current_winfixbuf then
vim.cmd('split ' .. vim.fn.fnameescape(relative_path))
opened_via_split = true
end
-- If the current window can't host a new buffer (special buftype, non-modifiable,
-- or 'winfixbuf' locking it), retarget a suitable window or fall back to a split.
-- Without this, :edit raises E1513 ("Cannot switch buffer. 'winfixbuf' is enabled")
-- whenever the picker is invoked from a window pinned via :h winfixbuf.
local opened_via_split = false
if current_buftype ~= '' or not current_buf_modifiable or current_winfixbuf then
local suitable_win = find_suitable_window()
if suitable_win then
vim.api.nvim_set_current_win(suitable_win)
elseif current_winfixbuf then
vim.cmd('split ' .. vim.fn.fnameescape(relative_path))
opened_via_split = true
end
if not opened_via_split then vim.cmd('edit ' .. vim.fn.fnameescape(relative_path)) end
elseif action == 'split' then
vim.cmd('split ' .. vim.fn.fnameescape(relative_path))
elseif action == 'vsplit' then
vim.cmd('vsplit ' .. vim.fn.fnameescape(relative_path))
elseif action == 'tab' then
vim.cmd('tabedit ' .. vim.fn.fnameescape(relative_path))
end
if not opened_via_split then vim.cmd('edit ' .. vim.fn.fnameescape(relative_path)) end
elseif action == 'split' then
vim.cmd('split ' .. vim.fn.fnameescape(relative_path))
elseif action == 'vsplit' then
vim.cmd('vsplit ' .. vim.fn.fnameescape(relative_path))
elseif action == 'tab' then
vim.cmd('tabedit ' .. vim.fn.fnameescape(relative_path))
end
-- Derive side effects on vim schedule to ensure they run after the file is opened
vim.schedule(function()
if location then location_utils.jump_to_location(location) end
if query and query ~= '' then
+1 -1
View File
@@ -12,7 +12,7 @@
"license": "MIT",
"repository": {
"type": "git",
"url": "git+https://github.com/dmtrKovalenko/fff.git",
"url": "git+https://github.com/dmtrKovalenko/fff.nvim.git",
"directory": "packages/fff-bin-darwin-arm64"
}
}
+1 -1
View File
@@ -12,7 +12,7 @@
"license": "MIT",
"repository": {
"type": "git",
"url": "git+https://github.com/dmtrKovalenko/fff.git",
"url": "git+https://github.com/dmtrKovalenko/fff.nvim.git",
"directory": "packages/fff-bin-darwin-x64"
}
}
@@ -12,7 +12,7 @@
"license": "MIT",
"repository": {
"type": "git",
"url": "git+https://github.com/dmtrKovalenko/fff.git",
"url": "git+https://github.com/dmtrKovalenko/fff.nvim.git",
"directory": "packages/fff-bin-linux-arm64-gnu"
},
"libc": ["glibc"]
@@ -12,7 +12,7 @@
"license": "MIT",
"repository": {
"type": "git",
"url": "git+https://github.com/dmtrKovalenko/fff.git",
"url": "git+https://github.com/dmtrKovalenko/fff.nvim.git",
"directory": "packages/fff-bin-linux-arm64-musl"
},
"libc": ["musl"]
+1 -1
View File
@@ -12,7 +12,7 @@
"license": "MIT",
"repository": {
"type": "git",
"url": "git+https://github.com/dmtrKovalenko/fff.git",
"url": "git+https://github.com/dmtrKovalenko/fff.nvim.git",
"directory": "packages/fff-bin-linux-x64-gnu"
},
"libc": ["glibc"]
+1 -1
View File
@@ -12,7 +12,7 @@
"license": "MIT",
"repository": {
"type": "git",
"url": "git+https://github.com/dmtrKovalenko/fff.git",
"url": "git+https://github.com/dmtrKovalenko/fff.nvim.git",
"directory": "packages/fff-bin-linux-x64-musl"
},
"libc": ["musl"]
+1 -1
View File
@@ -12,7 +12,7 @@
"license": "MIT",
"repository": {
"type": "git",
"url": "git+https://github.com/dmtrKovalenko/fff.git",
"url": "git+https://github.com/dmtrKovalenko/fff.nvim.git",
"directory": "packages/fff-bin-win32-arm64"
}
}
+1 -1
View File
@@ -12,7 +12,7 @@
"license": "MIT",
"repository": {
"type": "git",
"url": "git+https://github.com/dmtrKovalenko/fff.git",
"url": "git+https://github.com/dmtrKovalenko/fff.nvim.git",
"directory": "packages/fff-bin-win32-x64"
}
}
+3 -3
View File
@@ -39,7 +39,7 @@
],
"repository": {
"type": "git",
"url": "git+https://github.com/dmtrKovalenko/fff.git",
"url": "git+https://github.com/dmtrKovalenko/fff.nvim.git",
"directory": "packages/fff"
},
"keywords": [
@@ -58,9 +58,9 @@
"access": "public"
},
"bugs": {
"url": "https://github.com/dmtrKovalenko/fff/issues"
"url": "https://github.com/dmtrKovalenko/fff.nvim/issues"
},
"homepage": "https://github.com/dmtrKovalenko/fff#readme",
"homepage": "https://github.com/dmtrKovalenko/fff.nvim#readme",
"optionalDependencies": {
"@ff-labs/fff-bin-darwin-arm64": "0.0.0",
"@ff-labs/fff-bin-darwin-x64": "0.0.0",
+3 -3
View File
@@ -34,7 +34,7 @@
],
"repository": {
"type": "git",
"url": "git+https://github.com/dmtrKovalenko/fff.git",
"url": "git+https://github.com/dmtrKovalenko/fff.nvim.git",
"directory": "packages/fff-node"
},
"keywords": [
@@ -53,9 +53,9 @@
"access": "public"
},
"bugs": {
"url": "https://github.com/dmtrKovalenko/fff/issues"
"url": "https://github.com/dmtrKovalenko/fff.nvim/issues"
},
"homepage": "https://github.com/dmtrKovalenko/fff#readme",
"homepage": "https://github.com/dmtrKovalenko/fff.nvim#readme",
"dependencies": {
"ffi-rs": "^1.0.0"
},
+3 -3
View File
@@ -7,12 +7,12 @@
"license": "MIT",
"repository": {
"type": "git",
"url": "git+https://github.com/dmtrKovalenko/fff.git",
"url": "git+https://github.com/dmtrKovalenko/fff.nvim.git",
"directory": "packages/pi-fff"
},
"homepage": "https://github.com/dmtrKovalenko/fff/tree/main/packages/pi-fff",
"homepage": "https://github.com/dmtrKovalenko/fff.nvim/tree/main/packages/pi-fff",
"bugs": {
"url": "https://github.com/dmtrKovalenko/fff/issues"
"url": "https://github.com/dmtrKovalenko/fff.nvim/issues"
},
"keywords": [
"pi",
-4
View File
@@ -131,10 +131,6 @@ describe('picker find_files_in_dir path resolution (issue #389)', function()
picker_ui.select('edit')
-- select('edit') defers the actual :edit via vim.schedule (see picker_ui.lua)
-- to let picker float teardown finish before opening the file. Flush here.
vim.wait(2000, function() return vim.api.nvim_buf_get_name(0) ~= '' end)
local bufname = vim.api.nvim_buf_get_name(0)
assert.is_true(bufname ~= '', 'expected :edit to open a buffer with a non-empty name')