github-featured/dmtrkovalenko--fff
2
0
Fork 0
Code Issues Pull Requests Actions 8 Deployments Models Agent Notes Packages Projects Releases Wiki Activity

perf: Improve macos indexing wall time (#457)

Browse Source 
* perf: Improve macos indexing wall time

* wip: try to get rid of arcs

* chore: Update docs for - wip: try to get rid of arcs

* chore: expanad fuzzy test suite

* fix ci

* fix: Parallelize git & indexing
This commit is contained in:
Dmitriy Kovalenko
2026-05-15 14:37:15 -07:00
committed by GitHub
parent d56006d26f
commit 4693adfe02
27 changed files with 1702 additions and 1022 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+4 -4
View File
@@ -668,9 +668,9 @@ dependencies = [
[[package]]
name = "fff-notify-debouncer-full"
version = "0.9.3"
version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c6f0c16164d10c082af931377766f5495440bee66a9a841bbcea1af5de8878c"
checksum = "29a4ebea7b8a2840cd59358bbf396f6f04313ce8eae84ac79703ce80298b8731"
dependencies = [
"file-id",
"log",
@@ -1535,9 +1535,9 @@ dependencies = [
[[package]]
name = "notify"
version = "9.0.0-rc.3"
version = "9.0.0-rc.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "783683ce6e1059e3747190a6c05688db21e07a846bf90babb351365f9133fe3e"
checksum = "b44b771d4dd781ef14c84078693e67495da6b47f609f72e8a4da8420a861240e"
dependencies = [
"bitflags 2.11.0",
"inotify",
Cargo.toml
+1 -1
View File
@@ -35,7 +35,7 @@ zlob = "1.3.3"
mlua = { version = "0.11.1", features = ["module", "luajit"] }
neo_frizbee = { version = "0.10.2", features = ["match_end_col"] }
notify = { version = "9.0.0-rc.3" }
notify-debouncer-full = { package = "fff-notify-debouncer-full", version = "0.9.3" }
notify-debouncer-full = { package = "fff-notify-debouncer-full", version = "0.9.4" }
once_cell = "1.20.2"
parking_lot = "0.12"
pathdiff = "0.2.1"
Makefile
+12 -4
View File
@@ -8,7 +8,7 @@ INCLUDEDIR ?= $(PREFIX)/include
STRESS_RUSTFLAGS := --cfg stress
FFF_STRESS_DEFAULT_SEED ?= 0xDEADBEEFCAFEBABE
.PHONY: build build-c-lib install uninstall test test-rust test-lua test-version test-bun test-node prepare-bun prepare-node set-npm-version header test-stress test-stress-seeded test-stress-random
.PHONY: build build-c-lib install uninstall test test-rust test-lua test-version test-bun test-node prepare-bun prepare-node set-npm-version header test-stress test-stress-seeded test-stress-random test-stress-repos
all: format test lint
@@ -101,7 +101,7 @@ test: test-rust test-lua test-version test-bun test-node
test-stress-seeded:
FFF_STRESS_SEED="$${FFF_STRESS_SEED:-$(FFF_STRESS_DEFAULT_SEED)}" \
RUSTFLAGS="$(STRESS_RUSTFLAGS)" \
cargo test \
cargo test --release \
-p fff-search \
--test fuzz_git_watcher_stress \
--features zlob \
@@ -109,13 +109,21 @@ test-stress-seeded:
test-stress-random:
RUSTFLAGS="$(STRESS_RUSTFLAGS)" \
cargo test \
cargo test --release \
-p fff-search \
--test fuzz_git_watcher_stress \
--features zlob \
-- --nocapture stress_random
test-stress: test-stress-seeded test-stress-random
test-stress-repos:
RUSTFLAGS="$(STRESS_RUSTFLAGS)" \
cargo test --release \
-p fff-search \
--test fuzz_real_repos \
--features zlob \
-- --nocapture
test-stress: test-stress-seeded test-stress-random test-stress-repos
# Update version in a package.json, including optionalDependencies.
# Usage: make set-npm-version PKG=packages/fff-bun VERSION=1.0.0-nightly.abc1234
crates/fff-c/src/lib.rs
+10 -13
View File
@@ -298,9 +298,9 @@ pub unsafe extern "C" fn fff_destroy(fff_handle: *mut c_void) {
let instance = unsafe { Box::from_raw(fff_handle as *mut FffInstance) };
if let Ok(mut guard) = instance.picker.write()
&& let Some(mut picker) = guard.take()
&& let Some(picker) = guard.take()
{
picker.stop_background_monitor();
drop(picker);
}
if let Ok(mut guard) = instance.frecency.write() {
@@ -896,22 +896,19 @@ pub unsafe extern "C" fn fff_restart_index(
Err(e) => return FffResult::err(&format!("Failed to canonicalize path: {}", e)),
};
let mut guard = match inst.picker.write() {
let guard = match inst.picker.write() {
Ok(g) => g,
Err(e) => return FffResult::err(&format!("Failed to acquire file picker lock: {}", e)),
};
let (warmup_caches, content_indexing, watch, mode) = if let Some(mut picker) = guard.take() {
let warmup = picker.has_mmap_cache();
let enable_content_indexing = picker.has_content_indexing();
let watch = picker.has_watcher();
let mode = picker.mode();
picker.stop_background_monitor();
(warmup, enable_content_indexing, watch, mode)
let (warmup_caches, content_indexing, watch, mode) = if let Some(ref picker) = *guard {
(
picker.has_mmap_cache(),
picker.has_content_indexing(),
picker.has_watcher(),
picker.mode(),
)
} else {
// this is error state anyway
(false, true, true, FFFMode::default())
};
crates/fff-core/src/background_watcher.rs
+42 -35
View File
@@ -593,7 +593,6 @@ fn handle_debounced_events(
files_updated = files_to_update_git_status.len(),
overflow_count, "File index changes applied",
);
if need_full_rescan || overflow_count > MAX_OVERFLOW_FILES {
info!("Watcher faced limit of index overflow. Triggering rescan");
if let Err(e) = shared_picker.trigger_full_rescan_async(shared_frecency) {
@@ -645,46 +644,54 @@ fn handle_debounced_events(
}
}
// Git status updates require a repository.
let Some(repo) = repo.as_ref() else {
debug!("No git repo available, skipping git status updates");
return new_dirs_to_watch;
};
// do not try to update the paths if we anyway going to rescan everything from scratch
if !need_full_rescan && (need_full_git_rescan || !files_to_update_git_status.is_empty()) {
let git_workdir = repo
.as_ref()
.map(|r| r.workdir().unwrap_or_else(|| r.path()).to_path_buf());
if need_full_git_rescan && !need_full_rescan {
info!("Triggering full git rescan");
let shared_picker = shared_picker.clone();
let shared_frecency = shared_frecency.clone();
if let Err(e) = shared_picker.refresh_git_status(shared_frecency) {
error!("Failed to refresh git status: {:?}", e);
}
}
// git status query even with a pathspec could be really slow, if we do this syncrhronously
// within the event handler, we actually risk of forming a snow ball of conflicting events
crate::file_picker::BACKGROUND_THREAD_POOL.spawn(move || {
let Some(git_path) = git_workdir else { return };
let Ok(repo) = Repository::open(&git_path) else {
error!("Failed to open git repo for async status update");
return;
};
// do not update the git status if the
if !files_to_update_git_status.is_empty() && !need_full_git_rescan {
info!(
"Fetching git status for {} files",
files_to_update_git_status.len()
);
let status = match GitStatusCache::git_status_for_paths(repo, &files_to_update_git_status) {
Ok(status) => status,
Err(e) => {
tracing::error!(?e, "Failed to query git status");
return new_dirs_to_watch;
if need_full_git_rescan && !need_full_rescan {
info!("Async: triggering full git rescan");
if let Err(e) = shared_picker.refresh_git_status(&shared_frecency) {
error!("Failed to refresh git status: {:?}", e);
}
}
};
if let Ok(mut guard) = shared_picker.write()
&& let Some(ref mut picker) = *guard
{
if let Err(e) = picker.update_git_statuses(status, shared_frecency) {
error!("Failed to update git statuses: {:?}", e);
} else {
info!("Successfully updated git statuses in picker");
if !files_to_update_git_status.is_empty() {
let status = match GitStatusCache::git_status_for_paths(
&repo,
&files_to_update_git_status,
) {
Ok(s) => s,
Err(e) => {
error!("Failed to query git status: {:?}", e);
return;
}
};
if let Ok(mut guard) = shared_picker.write()
&& let Some(ref mut picker) = *guard
{
if let Err(e) = picker.update_git_statuses(status, &shared_frecency) {
error!("Failed to update git statuses: {:?}", e);
} else {
info!("Async: git statuses updated");
}
}
}
} else {
error!("Failed to acquire picker lock for git status update");
}
});
}
new_dirs_to_watch
crates/fff-core/src/bigram_filter.rs
+74 -215
View File
@@ -596,7 +596,7 @@ impl BigramOverlay {
}
}
pub const BIGRAM_CONTENT_CAP: usize = 64 * 1024;
pub(crate) const MAX_INDEXABLE_FILE_SIZE: usize = 2 * 1024 * 1024;
const BIGRAM_CHUNK_FILES: usize = 4 * 64;
/// Sparse-column cutoff for the skip-1 sub-index. Rare skip columns add
@@ -605,24 +605,48 @@ const BIGRAM_CHUNK_FILES: usize = 4 * 64;
const SKIP_INDEX_MIN_DENSITY_PCT: u32 = 12;
thread_local! {
/// Per-rayon-worker reusable read buffer. 64 KB is too large to
/// keep on the default pthread stack (macOS ships 512 KB), so the
/// buffer lives on the heap behind a `Box<[u8; N]>`. TLS keeps the
/// allocation alive for the thread's lifetime so we pay the cost
/// once, not per file.
static READ_BUF: std::cell::RefCell<Box<[u8; BIGRAM_CONTENT_CAP]>> =
std::cell::RefCell::new(Box::new([0u8; BIGRAM_CONTENT_CAP]));
/// Reusable read buffer that is allocated per thread and used for reading files
static READ_BUF: std::cell::RefCell<Box<[u8]>> =
std::cell::RefCell::new(vec![0u8; MAX_INDEXABLE_FILE_SIZE].into_boxed_slice());
}
/// Outcome of processing one file's content.
enum FileOutcome {
/// Content contained a NUL byte — mark the file as binary so future
/// greps skip it without re-reading.
Binary,
/// Read succeeded and the content was fed to the bigram builder.
Indexed,
/// File was empty or failed to open; nothing to do.
Skipped,
/// reads a chunk for bigram either from new warmed up cache or from the file directly
#[inline]
#[allow(clippy::too_many_arguments)]
fn read_bigram_chunk<'a>(
file: &'a crate::types::FileItem,
base_fd: libc::c_int,
base_path: &std::path::Path,
arena: crate::simd_path::ArenaPtr,
budget: &crate::types::ContentCacheBudget,
warmup: bool,
buf: &'a mut [u8],
path_buf: &mut [u8; crate::simd_path::PATH_BUF_SIZE],
) -> Option<&'a [u8]> {
if warmup
&& !file.is_likely_hot()
&& let Some(cached) = file.get_cached_content(arena, base_path, budget)
{
if crate::file_picker::detect_binary_content(cached) {
file.set_binary(true);
return None;
}
return Some(&cached[..cached.len().min(MAX_INDEXABLE_FILE_SIZE)]);
}
let want = (file.size as usize).min(MAX_INDEXABLE_FILE_SIZE);
let filled = file.read_trimmed_into_buf(base_fd, base_path, arena, path_buf, &mut buf[..want]);
if filled == 0 {
return None;
}
let data = &buf[..filled];
if crate::file_picker::detect_binary_content(data) {
file.set_binary(true);
return None;
}
Some(data)
}
#[tracing::instrument(skip_all, name = "Building Bigram Index", level = tracing::Level::DEBUG)]
@@ -631,24 +655,22 @@ pub(crate) fn build_bigram_index(
budget: &crate::types::ContentCacheBudget,
base_path: &std::path::Path,
arena: crate::simd_path::ArenaPtr,
) -> (BigramFilter, Vec<usize>) {
let start = std::time::Instant::now();
tracing::info!("Building bigram index for {} files...", files.len());
warmup: bool,
) -> BigramFilter {
let builder = BigramIndexBuilder::new(files.len());
let skip_builder = BigramIndexBuilder::new(files.len());
// this does remove a memcpy for every single file + actually reducing open time on macos
#[cfg(unix)]
let base_fd: libc::c_int = open_base_dir_fd(base_path);
#[cfg(not(unix))]
let base_fd: i32 = -1;
// `content_binary` is only touched from the Binary branch below, so
// the mutex is cold in practice. A lock-free collector wasn't worth
// the complexity.
let content_binary: std::sync::Mutex<Vec<usize>> = std::sync::Mutex::new(Vec::new());
// Single unified pass: every file is bigram-indexed, and (when `warmup`)
// the content cache is opportunistically filled. We SKIP caching files
// that are likely already hot in the OS page cache (recent frecency hits
// or dirty-per-git) so our limited cache budget goes to the cold tail
// that actually benefits from a pinned mmap. Natural traversal order,
// no pre-sort, no separate warmup pass.
crate::file_picker::BACKGROUND_THREAD_POOL.install(|| {
files
.par_chunks(BIGRAM_CHUNK_FILES)
@@ -657,183 +679,47 @@ pub(crate) fn build_bigram_index(
let base_idx = chunk_idx * BIGRAM_CHUNK_FILES;
for (offset, file) in chunk.iter().enumerate() {
let file_idx = base_idx + offset;
let outcome = process_file(
file,
file_idx,
&builder,
&skip_builder,
base_fd,
base_path,
arena,
budget,
);
if matches!(outcome, FileOutcome::Binary) {
content_binary.lock().unwrap().push(file_idx);
if file.is_binary() || file.size == 0 {
return;
}
READ_BUF.with(|read_cell| {
let mut buf = read_cell.borrow_mut();
let mut path_buf = [0u8; crate::simd_path::PATH_BUF_SIZE];
if let Some(content) = read_bigram_chunk(
file,
base_fd,
base_path,
arena,
budget,
warmup,
&mut buf[..],
&mut path_buf,
) {
builder.add_file_content(&skip_builder, file_idx, content);
}
});
}
});
});
#[cfg(unix)]
if base_fd >= 0 {
// SAFETY: we opened `base_fd` at the top of this function and
// no worker still references it once the rayon pool joined.
unsafe { libc::close(base_fd) };
}
let content_binary_vec = content_binary.into_inner().unwrap();
let cols = builder.columns_used();
let mut index = builder.compress(None);
let skip_index = skip_builder.compress(Some(SKIP_INDEX_MIN_DENSITY_PCT));
index.set_skip_index(skip_index);
// Builder buffers were freed by `compress()` above (one deallocation
// each); nudge mimalloc to return them (and any transient allocs)
// to the OS.
// in progress bigram walk + rust's ignore crate allocates shit ton of garbage memory
// all custom allocators would think this is available resource while we do not allocate
// after the sync, so it's very important to let the unused memory go back to the OS
crate::file_picker::hint_allocator_collect();
tracing::info!(
"Bigram index built in {:.2}s — {} dense columns for {} files",
start.elapsed().as_secs_f64(),
cols,
files.len(),
);
if !content_binary_vec.is_empty() {
tracing::info!(
"Bigram build detected {} content-binary files (not caught by extension)",
content_binary_vec.len(),
);
}
(index, content_binary_vec)
}
/// Process one file: read up to `BIGRAM_CONTENT_CAP` bytes, feed them
/// to the bigram builder (or record as binary / skipped).
///
/// `base_fd` is the parent-directory fd for the Unix `openat` fast
/// path, or `-1` to force the portable `std::fs::File::open` fallback.
#[inline]
#[allow(clippy::too_many_arguments)]
fn process_file(
file: &crate::types::FileItem,
file_idx: usize,
builder: &BigramIndexBuilder,
skip_builder: &BigramIndexBuilder,
base_fd: i32,
base_path: &std::path::Path,
arena: crate::simd_path::ArenaPtr,
budget: &crate::types::ContentCacheBudget,
) -> FileOutcome {
if file.is_binary() || file.size == 0 || file.size > budget.max_file_size {
return FileOutcome::Skipped;
}
// Zero-copy fast path: the warmup phase may have cached this file's
// content already. Avoid re-reading from disk.
if let Some(cached) = file.get_content(arena, base_path, budget) {
if crate::file_picker::detect_binary_content(cached) {
return FileOutcome::Binary;
}
let capped = &cached[..cached.len().min(BIGRAM_CONTENT_CAP)];
builder.add_file_content(skip_builder, file_idx, capped);
return FileOutcome::Indexed;
}
let want = (file.size as usize).min(BIGRAM_CONTENT_CAP);
let mut path_buf = [0u8; crate::simd_path::PATH_BUF_SIZE];
READ_BUF.with(|read_cell| {
let mut buf = read_cell.borrow_mut();
let filled = read_file_content(
file,
base_fd,
base_path,
arena,
&mut path_buf,
&mut buf[..want],
);
if filled == 0 {
return FileOutcome::Skipped;
}
let data = &buf[..filled];
if crate::file_picker::detect_binary_content(data) {
return FileOutcome::Binary;
}
builder.add_file_content(skip_builder, file_idx, data);
FileOutcome::Indexed
})
}
/// Read up to `buf.len()` bytes of `file`'s content into `buf`. Returns
/// the number of bytes actually read (0 on any error, so callers treat
/// failures as "skip").
#[inline]
fn read_file_content(
file: &crate::types::FileItem,
base_fd: i32,
base_path: &std::path::Path,
arena: crate::simd_path::ArenaPtr,
path_buf: &mut [u8; crate::simd_path::PATH_BUF_SIZE],
buf: &mut [u8],
) -> usize {
#[cfg(unix)]
{
read_file_content_unix(file, base_fd, base_path, arena, path_buf, buf)
}
#[cfg(not(unix))]
{
let _ = base_fd;
read_file_content_std(file, base_path, arena, path_buf, buf)
}
}
#[cfg(unix)]
fn read_file_content_unix(
file: &crate::types::FileItem,
base_fd: libc::c_int,
base_path: &std::path::Path,
arena: crate::simd_path::ArenaPtr,
path_buf: &mut [u8; crate::simd_path::PATH_BUF_SIZE],
buf: &mut [u8],
) -> usize {
let fd = if base_fd >= 0 {
let rel_cstr = file.write_relative_cstr(arena, path_buf);
// SAFETY: `rel_cstr` is NUL-terminated, `base_fd` is a valid
// directory descriptor owned by the caller.
unsafe { libc::openat(base_fd, rel_cstr.as_ptr(), libc::O_RDONLY) }
} else {
use std::os::unix::io::IntoRawFd;
let abs = file.write_absolute_path(arena, base_path, path_buf);
match std::fs::File::open(abs) {
Ok(f) => f.into_raw_fd(),
Err(_) => return 0,
}
};
if fd < 0 {
return 0;
}
let mut filled = 0usize;
while filled < buf.len() {
// SAFETY: `fd` is an owned descriptor, `buf[filled..]` is a
// valid writable slice for `buf.len() - filled` bytes.
let n = unsafe {
libc::read(
fd,
buf[filled..].as_mut_ptr() as *mut libc::c_void,
(buf.len() - filled) as libc::size_t,
)
};
if n <= 0 {
break;
}
filled += n as usize;
}
// SAFETY: matching close for the owned descriptor.
unsafe { libc::close(fd) };
filled
index
}
/// Open the base directory for the `openat` fast path. Returns `-1` on
@@ -858,33 +744,6 @@ fn open_base_dir_fd(base_path: &std::path::Path) -> libc::c_int {
}
}
/// Portable fallback (Windows + non-`openat` Unix): `std::fs::File` +
/// `Read::read` into `buf`. Used on Windows unconditionally, and on
/// Unix when the base directory fd could not be opened.
#[cfg(not(unix))]
fn read_file_content_std(
file: &crate::types::FileItem,
base_path: &std::path::Path,
arena: crate::simd_path::ArenaPtr,
path_buf: &mut [u8; crate::simd_path::PATH_BUF_SIZE],
buf: &mut [u8],
) -> usize {
use std::io::Read;
let abs = file.write_absolute_path(arena, base_path, path_buf);
let Ok(mut f) = std::fs::File::open(abs) else {
return 0;
};
let mut filled = 0usize;
while filled < buf.len() {
match f.read(&mut buf[filled..]) {
Ok(0) => break,
Ok(n) => filled += n,
Err(_) => return 0,
}
}
filled
}
#[cfg(test)]
mod tests {
use super::*;
crates/fff-core/src/file_picker.rs
+197 -264
View File
@@ -56,7 +56,7 @@ use std::ops::ControlFlow;
use std::path::{Path, PathBuf};
use std::sync::{
Arc, LazyLock,
atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering},
atomic::{AtomicBool, AtomicUsize, Ordering},
};
use std::thread::JoinHandle;
use std::time::SystemTime;
@@ -74,11 +74,10 @@ pub(crate) static BACKGROUND_THREAD_POOL: LazyLock<rayon::ThreadPool> = LazyLock
let total = std::thread::available_parallelism()
.map(|p| p.get())
.unwrap_or(4);
let bg_threads = total.saturating_sub(2).max(1);
info!(
"Background pool: {} threads (system has {})",
bg_threads, total
);
// benchmarks show that most of the work background tasks spend on waiting for syscalls,
// by halfing available parallelism we loose some performance, but it is mostly nothing
let bg_threads = (total / 2).max(2);
rayon::ThreadPoolBuilder::new()
.num_threads(bg_threads)
.thread_name(|i| format!("fff-bg-{i}"))
@@ -138,7 +137,7 @@ pub(crate) struct FileSync {
indexable_count: usize,
base_count: usize,
/// Number of active present files that exists in the file system
live_count: usize,
pub(crate) live_count: usize,
/// Sorted directory table. `StableVec` so post-scan snapshots can keep
/// the allocation alive across a picker drop without copying, and so
/// concurrent readers observe a consistent view via the same shared
@@ -218,10 +217,11 @@ impl FileSync {
self.files.get_mut(index)
}
/// Find file index by path using binary search on the sorted base portion.
/// Find file index by path using binary search on the sorted base partitions,
/// falling back to a linear scan of the overflow region.
/// `path` must be an absolute path under `base_path`.
#[inline]
fn find_file_index(&self, path: &Path, base_path: &Path) -> Result<usize, usize> {
fn find_file_index(&self, path: &Path, base_path: &Path) -> Option<usize> {
let arena = self.arena_base_ptr();
// Strip base_path prefix to get the relative path. On Windows this
@@ -233,11 +233,11 @@ impl FileSync {
Err(_) => {
#[cfg(windows)]
{
canonical_relative_path(path, base_path).ok_or(0usize)?
canonical_relative_path(path, base_path)?
}
#[cfg(not(windows))]
{
return Err(0);
return None;
}
}
};
@@ -254,49 +254,49 @@ impl FileSync {
// Binary search dirs to find the parent directory index.
// Dir items store the relative path including trailing '/' (e.g. "src/components/").
let mut dir_buf = [0u8; crate::simd_path::PATH_BUF_SIZE];
let dir_idx = match self
let dir_idx = self
.dirs
.binary_search_by(|d| d.read_relative_path(arena, &mut dir_buf).cmp(dir_rel))
{
Ok(idx) => idx as u32,
Err(_) => return Err(0), // directory not found
};
.ok();
// Binary search files by (parent_dir, filename). Base files live in
// two internally-sorted partitions — indexable first, then
// unindexable — so we try each half in turn. Two O(log n) searches
// with short-circuit on the first hit.
let cmp_key = |f: &FileItem| {
f.parent_dir_index().cmp(&dir_idx).then_with(|| {
let fname = f.file_name(arena);
fname.as_str().cmp(filename)
})
};
// Binary search base files by (parent_dir, filename). Base files live in
// two internally-sorted partitions — indexable first, then unindexable —
// so we try each half in turn. Two O(log n) searches with short-circuit.
if let Some(dir_idx) = dir_idx {
let dir_idx = dir_idx as u32;
let cmp_key = |f: &FileItem| {
f.parent_dir_index.cmp(&dir_idx).then_with(|| {
let fname = f.file_name(arena);
fname.as_str().cmp(filename)
})
};
if self.indexable_count > 0
&& let Ok(pos) = self.files[..self.indexable_count].binary_search_by(cmp_key)
{
return Ok(pos);
if self.indexable_count > 0
&& let Ok(pos) = self.files[..self.indexable_count].binary_search_by(cmp_key)
{
return Some(pos);
}
if self.indexable_count < self.base_count
&& let Ok(rel_pos) =
self.files[self.indexable_count..self.base_count].binary_search_by(cmp_key)
{
return Some(self.indexable_count + rel_pos);
}
}
if self.indexable_count < self.base_count
&& let Ok(rel_pos) =
self.files[self.indexable_count..self.base_count].binary_search_by(cmp_key)
{
return Ok(self.indexable_count + rel_pos);
// Overflow region: linear scan by full relative path.
if self.base_count < self.files.len() {
let overflow_arena = self.overflow_arena_ptr();
if let Some(pos) = self.files[self.base_count..]
.iter()
.position(|f| f.relative_path_eq(overflow_arena, rel_path))
{
return Some(self.base_count + pos);
}
}
Err(0)
}
/// Find a file in the overflow portion by relative path (linear scan).
/// Returns the absolute index into `files` (i.e. `base_count + position`).
fn find_overflow_index(&self, relative_path: &str) -> Option<usize> {
let overflow_arena = self.overflow_arena_ptr();
self.files[self.base_count..]
.iter()
.position(|f| f.relative_path_eq(overflow_arena, relative_path))
.map(|pos| self.base_count + pos)
None
}
/// Tombstone every file that matches `predicate`. No shift: the
@@ -339,11 +339,6 @@ impl FileItem {
Self::new_with_metadata(path, base_path, git_status, metadata.as_ref())
}
pub fn delete(&mut self) {
self.set_deleted(true);
self.git_status = Some(Status::INDEX_DELETED); // this is not needed but cool
}
/// Create a FileItem using pre-fetched metadata to avoid a redundant stat syscall.
/// Returns `(FileItem, relative_path)`. The FileItem's `path` field is
/// empty; callers must populate it via `set_path` or `build_chunked_path_store_and_assign`.
@@ -754,12 +749,11 @@ impl FilePicker {
{
let mut guard = shared_picker.write()?;
*guard = Some(picker);
// by dropping the old picker if it exists we triggering
// it's internal `cancelled` flag flip which will automatically clean
// any thread that might be capturing the reference safely & unsfaely
}
// `ScanJob::spawn` flips `scanning=true` synchronously before handing
// off to the worker thread, so callers that invoke `wait_for_scan`
// immediately after `new_with_shared_state` are guaranteed to see
// the scan in progress.
ScanJob::new_initial(
shared_picker,
shared_frecency,
@@ -1200,33 +1194,6 @@ impl FilePicker {
)
}
#[doc(hidden)]
pub fn grep_original(
&self,
query: &FFFQuery<'_>,
options: &GrepSearchOptions,
) -> GrepResult<'_> {
let arena = self.arena_base_ptr();
let overflow_arena = self.sync_data.overflow_arena_ptr();
let cancel = options
.abort_signal
.as_deref()
.unwrap_or(&self.signals.cancelled);
grep_search(
self.get_files(),
query,
options,
self.cache_budget(),
self.sync_data.bigram_index.as_deref(),
None,
cancel,
&self.base_path,
arena,
overflow_arena,
)
}
// Returns an ongoing or finisshed scan progress
pub fn get_scan_progress(&self) -> ScanProgress {
let scanned_count = self.scanned_files_count.load(Ordering::Relaxed);
@@ -1239,9 +1206,12 @@ impl FilePicker {
}
}
pub(crate) fn set_bigram_index(&mut self, index: BigramFilter, overlay: BigramOverlay) {
pub(crate) fn set_bigram_index(&mut self, index: BigramFilter) {
self.sync_data.bigram_index = Some(Arc::new(index));
self.sync_data.bigram_overlay = Some(Arc::new(parking_lot::RwLock::new(overlay)));
// once the index is reset automatically reset the overaly
self.sync_data.bigram_overlay = Some(Arc::new(parking_lot::RwLock::new(
BigramOverlay::new(self.sync_data.indexable_count),
)));
}
pub(crate) fn scan_signals(&self) -> crate::scan::ScanSignals {
@@ -1273,7 +1243,8 @@ impl FilePicker {
if self
.signals
.post_scan_indexing_active
.load(Ordering::Acquire)
.compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
.is_err()
{
tracing::error!(
"Can not acquire post scan unsafe snapshot, someone already acquired it"
@@ -1281,10 +1252,6 @@ impl FilePicker {
return None;
}
self.signals
.post_scan_indexing_active
.store(true, Ordering::Release);
Some(PostScanUnsafeSnapshot {
files: self.sync_data.files.clone(),
dirs: self.sync_data.dirs.clone(),
@@ -1293,7 +1260,7 @@ impl FilePicker {
indexable_count: self.sync_data.indexable_count,
base_path: self.base_path.clone(),
budget: Arc::clone(&self.cache_budget),
// to clear automatically
cancelled: Arc::clone(&self.signals.cancelled),
post_scan_flag: Arc::clone(&self.signals.post_scan_indexing_active),
})
}
@@ -1323,6 +1290,7 @@ impl FilePicker {
let bp = self.base_path.clone();
let arena = self.arena_base_ptr();
let frecency = shared_frecency.read()?;
status_cache
.into_iter()
.try_for_each(|(path, status)| -> Result<(), Error> {
@@ -1335,7 +1303,7 @@ impl FilePicker {
// interior-mutable atomic score, so `&self` access is
// enough — no write aliasing against Arc clones.
let score = file.access_frecency_score as i32;
let dir_idx = file.parent_dir_index() as usize;
let dir_idx = file.parent_dir_index as usize;
if let Some(dir) = self.sync_data.dirs.get(dir_idx) {
dir.update_frecency_if_larger(score);
}
@@ -1358,13 +1326,9 @@ impl FilePicker {
) -> Result<(), Error> {
let path = file_path.as_ref();
let arena = self.arena_base_ptr();
let rel = self.to_relative_path(path);
let rel_ref: &str = rel.as_deref().unwrap_or("");
let index = self
.sync_data
.find_file_index(path, &self.base_path)
.ok()
.or_else(|| self.sync_data.find_overflow_index(rel_ref));
let index = self.sync_data.find_file_index(path, &self.base_path);
if let Some(index) = index
&& let Some(file) = self.sync_data.get_file_mut(index)
{
@@ -1372,7 +1336,7 @@ impl FilePicker {
// Update parent dir frecency inline (atomic, &self access).
let score = file.access_frecency_score as i32;
let dir_idx = file.parent_dir_index() as usize;
let dir_idx = file.parent_dir_index as usize;
if let Some(dir) = self.sync_data.dirs.get(dir_idx) {
dir.update_frecency_if_larger(score);
}
@@ -1384,19 +1348,12 @@ impl FilePicker {
pub fn get_file_by_path(&self, path: impl AsRef<Path>) -> Option<&FileItem> {
self.sync_data
.find_file_index(path.as_ref(), &self.base_path)
.ok()
.and_then(|index| self.sync_data.files().get(index))
}
pub fn get_mut_file_by_path(&mut self, path: impl AsRef<Path>) -> Option<&mut FileItem> {
let path = path.as_ref();
let rel = self.to_relative_path(path);
let rel_ref: &str = rel.as_deref().unwrap_or("");
let index = self
.sync_data
.find_file_index(path, &self.base_path)
.ok()
.or_else(|| self.sync_data.find_overflow_index(rel_ref));
let index = self.sync_data.find_file_index(path, &self.base_path);
index.and_then(|i| self.sync_data.get_file_mut(i))
}
@@ -1407,13 +1364,14 @@ impl FilePicker {
pub fn handle_create_or_modify(&mut self, path: impl AsRef<Path> + Debug) -> Option<&FileItem> {
let path = path.as_ref();
if let Ok(idx) = self.sync_data.find_file_index(path, &self.base_path) {
return self.handle_file_modify(path, FileSlot::Base(idx));
}
if let Some(idx) = self.sync_data.find_file_index(path, &self.base_path) {
let slot = if idx < self.sync_data.base_count {
FileSlot::Base(idx)
} else {
FileSlot::Overflow(idx)
};
let relative_path = self.to_relative_path(path)?;
if let Some(idx) = self.sync_data.find_overflow_index(&relative_path) {
return self.handle_file_modify(path, FileSlot::Overflow(idx));
return self.handle_file_modify(path, slot);
}
self.add_new_file(path)
@@ -1423,16 +1381,22 @@ impl FilePicker {
fn handle_file_modify(&mut self, path: &Path, slot: FileSlot) -> Option<&FileItem> {
let overlay = self.sync_data.bigram_overlay.as_ref().map(Arc::clone);
let pos = slot.index();
let file = self.sync_data.get_file_mut(pos)?;
let metadata = std::fs::metadata(path)
.inspect_err(|e| {
tracing::error!(
?e,
"File market for modification doesn't exists or not accessible"
)
})
.ok()?;
// this is the only way to actually know if the file is on disk, we CAN NOT
// rely on the watcher to proive the latest state of the file, do the actual check
let metadata = match std::fs::metadata(path) {
Ok(m) => {
self.untombstone_file(pos);
m
}
Err(_) => {
self.tombstone_file(pos);
return None;
}
};
let file = self.sync_data.get_file_mut(pos)?;
let size = metadata.len();
let modified_time = metadata
@@ -1441,11 +1405,6 @@ impl FilePicker {
.and_then(|t| t.duration_since(SystemTime::UNIX_EPOCH).ok())
.map(|d| d.as_secs());
let was_deleted = file.is_deleted();
if was_deleted {
file.set_deleted(false);
}
file.update_metadata(&self.cache_budget, modified_time, Some(size));
// only base-region entries participate in the bigram overlay
@@ -1462,12 +1421,6 @@ impl FilePicker {
}
}
// Increment after dropping the mutable file reference so the
// reborrow for the return doesn't conflict.
if was_deleted {
self.sync_data.live_count += 1;
}
self.sync_data.files().get(pos)
}
@@ -1516,35 +1469,46 @@ impl FilePicker {
self.sync_data.files.last()
}
/// Tombstone a file instead of removing it, keeping base indices stable.
fn tombstone_file(&mut self, index: usize) {
let file = &mut self.sync_data.files[index];
if file.is_deleted() {
return;
}
file.set_deleted(true);
file.invalidate_mmap(&self.cache_budget);
file.git_status = None;
// Only base-region files participate in the bigram overlay
if index < self.sync_data.base_count
&& let Some(ref overlay) = self.sync_data.bigram_overlay
{
overlay.write().delete_file(index);
}
self.sync_data.live_count -= 1;
}
fn untombstone_file(&mut self, index: usize) {
let file = &mut self.sync_data.files[index];
if !file.is_deleted() {
return;
}
file.set_deleted(false);
self.sync_data.live_count += 1;
}
/// Marks file as deleted, make sure that if you call this yourself these changes can be reverted
/// by the internal mechanics if the file actually exists on the disk, use only if you know that
/// the file going to be disapperaed or if you do not have the watcher installed
pub fn remove_file_by_path(&mut self, path: impl AsRef<Path>) -> bool {
let path = path.as_ref();
match self.sync_data.find_file_index(path, &self.base_path) {
Ok(index) => {
let file = &mut self.sync_data.files[index];
file.set_deleted(true);
file.git_status = None;
file.invalidate_mmap(&self.cache_budget);
if let Some(ref overlay) = self.sync_data.bigram_overlay {
overlay.write().delete_file(index);
}
self.sync_data.live_count -= 1;
true
}
Err(_) => {
let rel = self.to_relative_path(path);
let rel_ref: &str = rel.as_deref().unwrap_or("");
if let Some(abs_pos) = self.sync_data.find_overflow_index(rel_ref) {
let file = &mut self.sync_data.files[abs_pos];
file.set_deleted(true);
file.git_status = None;
file.invalidate_mmap(&self.cache_budget);
self.sync_data.live_count -= 1;
true
} else {
false
}
}
if let Some(index) = self.sync_data.find_file_index(path, &self.base_path) {
self.tombstone_file(index);
true
} else {
false
}
}
@@ -1584,6 +1548,12 @@ impl FilePicker {
self.signals.scanning.load(Ordering::Relaxed)
}
pub fn is_post_scan_active(&self) -> bool {
self.signals
.post_scan_indexing_active
.load(Ordering::Acquire)
}
/// Return a clone of the watcher-ready flag so callers can poll it without
/// holding a lock on the picker.
pub fn watcher_signal(&self) -> Arc<AtomicBool> {
@@ -1638,6 +1608,14 @@ fn canonical_relative_path(path: &Path, base: &Path) -> Option<String> {
rel.to_str().map(str::to_owned)
}
impl Drop for FilePicker {
fn drop(&mut self) {
// Cancel any in-flight ScanJob bound to this picker's signals so
// it cannot mutate the replacement picker after a swap.
self.signals.cancelled.store(true, Ordering::Release);
}
}
#[derive(Debug, Clone, Copy)]
enum FileSlot {
Base(usize),
@@ -1671,6 +1649,7 @@ pub(crate) struct PostScanUnsafeSnapshot {
pub base_count: usize,
pub indexable_count: usize,
pub base_path: PathBuf,
pub cancelled: Arc<AtomicBool>,
post_scan_flag: Arc<AtomicBool>,
}
@@ -1698,76 +1677,6 @@ pub struct ScanProgress {
pub is_warmup_complete: bool,
}
/// Pre-populate mmap caches for the most valuable files so the first grep
/// search doesn't pay the mmap creation + page fault cost.
///
/// All files are collected once, then an O(n) `select_nth_unstable_by`
/// partitions the top [`MAX_CACHED_CONTENT_FILES`] highest-frecency eligible
/// files to the front (binary / empty files are pushed to the end by the
/// comparator). The selected prefix is warmed in parallel via rayon.
///
/// Files beyond the budget are still available via temporary mmaps on first
/// grep access, so correctness is unaffected.
#[tracing::instrument(skip(files), name = "warmup_mmaps", level = Level::DEBUG)]
pub(crate) fn warmup_mmaps(
files: &[FileItem],
budget: &ContentCacheBudget,
base_path: &Path,
arena: ArenaPtr,
) {
let max_files = budget.max_files;
let max_bytes = budget.max_bytes;
let max_file_size = budget.max_file_size;
// Single collect — no pre-filter. The comparator in select_nth pushes
// ineligible files (binary, empty) to the tail automatically.
let mut all: Vec<&FileItem> = files.iter().collect();
// O(n) partial sort: top max_files eligible-by-frecency files land in
// all[..max_files]. Ineligible files compare as "lowest priority" so
// they naturally sink past the partition boundary.
if all.len() > max_files {
all.select_nth_unstable_by(max_files, |a, b| {
let a_ok = !a.is_binary() && a.size > 0;
let b_ok = !b.is_binary() && b.size > 0;
match (a_ok, b_ok) {
(true, false) => std::cmp::Ordering::Less,
(false, true) => std::cmp::Ordering::Greater,
(false, false) => std::cmp::Ordering::Equal,
(true, true) => b.total_frecency_score().cmp(&a.total_frecency_score()),
}
});
}
let to_warm = &all[..all.len().min(max_files)];
let warmed_bytes = AtomicU64::new(0);
let budget_exhausted = AtomicBool::new(false);
BACKGROUND_THREAD_POOL.install(|| {
to_warm.par_iter().for_each(|file| {
if budget_exhausted.load(Ordering::Relaxed) {
return;
}
if file.is_binary() || file.size == 0 || file.size > max_file_size {
return;
}
// Byte budget.
let prev_bytes = warmed_bytes.fetch_add(file.size, Ordering::Relaxed);
if prev_bytes + file.size > max_bytes {
budget_exhausted.store(true, Ordering::Relaxed);
return;
}
if let Some(content) = file.get_content(arena, base_path, budget) {
let _ = std::hint::black_box(content.first());
}
});
});
}
impl FileSync {
pub(crate) fn discover_git_workdir(base_path: &Path) -> Option<PathBuf> {
let git_workdir = Repository::discover(base_path)
@@ -1787,7 +1696,7 @@ impl FileSync {
std::thread::spawn(move || {
GitStatusCache::read_git_status(
Some(git_workdir.as_path()),
&mut crate::git::default_status_options(),
&mut crate::git::initial_scan_status_options(),
)
})
}
@@ -1795,6 +1704,7 @@ impl FileSync {
/// Returns files immediately (searchable) and a handle to the in-progress
/// git status computation. This avoids blocking on `git status` which can
/// take 10+ seconds on very large repos (e.g. chromium).
#[tracing::instrument(skip_all, name = "walk_filesystem", level = Level::INFO)]
pub(crate) fn walk_filesystem(
base_path: &Path,
git_workdir: Option<PathBuf>,
@@ -1834,6 +1744,7 @@ impl FileSync {
// no chunking, no HashMap, just Vec::push under the Mutex.
let pairs = parking_lot::Mutex::new(Vec::<(FileItem, String)>::new());
let walker_span = tracing::info_span!("walker_run").entered();
walker.run(|| {
let pairs = &pairs;
let counter = Arc::clone(synced_files_count);
@@ -1867,6 +1778,7 @@ impl FileSync {
ignore::WalkState::Continue
})
});
drop(walker_span);
let mut pairs = pairs.into_inner();
info!(
@@ -1907,7 +1819,7 @@ impl FileSync {
let _ = file.update_frecency_scores(frecency, arena, base_path, mode);
let score = file.access_frecency_score as i32;
if score > 0 {
let dir_idx = file.parent_dir_index() as usize;
let dir_idx = file.parent_dir_index as usize;
if let Some(dir) = dirs_ref.get(dir_idx) {
dir.update_frecency_if_larger(score);
}
@@ -1917,25 +1829,19 @@ impl FileSync {
}
drop(frecency);
// Re-sort by (indexable-first, parent_dir, filename). Indexable base
// files come first so the bigram builder can size its column bitsets to
// just the indexable subset. Within each partition files stay sorted by
// (parent_dir, filename) — `find_file_index` does two binary searches
// (one per partition) to preserve O(log n) lookups.
//
// "Indexable" = can possibly contribute bigrams: not binary-by-extension,
// non-zero size, not larger than the bigram/mmap cap. The cap matches
// `ContentCacheBudget::max_file_size` default (10 MB) — any file above
// that is skipped by `build_bigram_index` anyway.
const BIGRAM_ELIGIBLE_MAX_SIZE: u64 = 10 * 1024 * 1024;
let is_indexable =
|f: &FileItem| !f.is_binary() && f.size > 0 && f.size <= BIGRAM_ELIGIBLE_MAX_SIZE;
// un-indexable files that are binary or not fitting the size cap has to beplaced in the end
let is_indexable = |f: &FileItem| {
!f.is_binary()
&& f.size > 0
&& f.size <= crate::bigram_filter::MAX_INDEXABLE_FILE_SIZE as u64
};
BACKGROUND_THREAD_POOL.install(|| {
files.par_sort_unstable_by(|a, b| {
// Sort indexables first (true < false when we invert with !).
(!is_indexable(a))
.cmp(&!is_indexable(b))
.then_with(|| a.parent_dir_index().cmp(&b.parent_dir_index()))
// this just makes it faster in terms of allocation - we store the dir indexes
.then_with(|| a.parent_dir_index.cmp(&b.parent_dir_index))
.then_with(|| a.file_name(arena).cmp(&b.file_name(arena)))
});
});
@@ -1982,11 +1888,38 @@ impl FileSync {
}
}
/// Pre-populate mmap caches for cold tail files so the first grep search
/// doesn't pay the mmap creation + page fault cost.
#[tracing::instrument(skip(files), name = "warmup_mmaps", level = Level::DEBUG)]
pub(crate) fn warmup_mmaps(
files: &[FileItem],
budget: &ContentCacheBudget,
base_path: &Path,
arena: ArenaPtr,
) {
// for most of the use cases mmaps limit would be significantly smaller than arepo
for file in files.iter() {
if file.is_likely_hot()
|| file.is_binary()
|| file.size == 0
|| file.size > budget.max_file_size
{
continue;
}
let _ = file.get_cached_content(arena, base_path, budget);
if budget.is_exhausted() {
break;
}
}
}
/// This does both thing (yes sorry all the OOP morons)
/// in one go: populates files chunked storage and creates new directories
fn populates_dirs_files_chunked_storage<'a>(
pairs: &'a mut [(FileItem, String)],
builder: &mut crate::simd_path::ChunkedPathStoreBuilder,
chunk_storage: &mut crate::simd_path::ChunkedPathStoreBuilder,
) -> Vec<DirItem> {
let mut dirs: Vec<DirItem> = Vec::new();
@@ -1999,7 +1932,7 @@ fn populates_dirs_files_chunked_storage<'a>(
let dir_part: &'a str = &rel[..file.path.filename_offset as usize];
if !prev_dir_valid || prev_dir != dir_part {
let dir_string = builder.add_dir_immediate(dir_part);
let dir_string = chunk_storage.add_dir_immediate(dir_part);
// Compute last-segment offset: for "src/components/" -> 4 (points to "components/")
let last_seg = if dir_part.is_empty() {
@@ -2019,10 +1952,8 @@ fn populates_dirs_files_chunked_storage<'a>(
prev_dir_valid = true;
}
let cs = builder.add_file_immediate(rel, file.path.filename_offset);
file.set_path(cs);
file.set_parent_dir(current_dir_idx);
file.path = chunk_storage.add_file_immediate(rel, file.path.filename_offset);
file.parent_dir_index = current_dir_idx;
}
dirs
@@ -2031,7 +1962,8 @@ fn populates_dirs_files_chunked_storage<'a>(
/// Fast extension-based binary detection. Avoids opening files during scan.
/// Covers the vast majority of binary files in typical repositories.
#[inline]
fn is_known_binary_extension(path: &Path) -> bool {
#[doc(hidden)]
pub fn is_known_binary_extension(path: &Path) -> bool {
let Some(ext) = path.extension().and_then(|e| e.to_str()) else {
return false;
};
@@ -2040,7 +1972,7 @@ fn is_known_binary_extension(path: &Path) -> bool {
ext,
// Images
"png" | "jpg" | "jpeg" | "gif" | "bmp" | "ico" | "webp" | "tiff" | "tif" | "avif" |
"heic" | "psd" | "icns" | "cur" | "raw" | "cr2" | "nef" | "dng" |
"heic" | "psd" | "icns" | "cur" | "raw" | "cr2" | "nef" | "dng" | "tga" |
// Video/Audio
"mp4" | "avi" | "mov" | "wmv" | "mkv" | "mp3" | "wav" | "flac" | "ogg" | "m4a" |
"aac" | "webm" | "flv" | "mpg" | "mpeg" | "wma" | "opus" | "pcm" | "reapeaks" |
@@ -2049,10 +1981,10 @@ fn is_known_binary_extension(path: &Path) -> bool {
"cab" | "cpio" | "jsonlz4" |
// Packages/Installers
"deb" | "rpm" | "apk" | "dmg" | "msi" | "iso" | "nupkg" | "whl" | "egg" |
"snap" | "appimage" | "flatpak" | "crx" | "pak" |
"appimage" | "flatpak" | "crx" | "pak" |
// Executables/Libraries
"exe" | "dll" | "so" | "dylib" | "o" | "a" | "lib" | "bin" | "elf" |
// Documents
// Documents (binary office formats)
"pdf" | "doc" | "docx" | "xls" | "xlsx" | "ppt" | "pptx" |
// Databases
"db" | "sqlite" | "sqlite3" | "mdb" |
@@ -2064,18 +1996,19 @@ fn is_known_binary_extension(path: &Path) -> bool {
// Compiled/Runtime
"class" | "pyc" | "pyo" | "wasm" | "dex" | "jar" | "war" |
// OCaml / Swift / Objective-C build artefacts
"cmi" | "cmt" | "cmti" | "cmx" | "cof" | "cot" | "cop" | "nib" |
"cmi" | "cmt" | "cmti" | "cmx" | "cof" | "cop" | "nib" |
"swiftdeps" | "swiftdeps~" | "swiftdoc" | "swiftmodule" | "swiftsourceinfo" |
// ML/Data Science
"npy" | "npz" | "pkl" | "pickle" | "h5" | "hdf5" | "pt" | "pth" | "onnx" |
"safetensors" | "tfrecord" |
// 3D/Game
"glb" | "fbx" | "blend" | "blp" | "tga" |
// Game engines / Unity-Unreal side-files
"meta" | "dat" | "tfx" | "dia" | "journal" | "toc" | "thm" | "pfl" |
"shadow" | "scan" | "flm" | "bcmap" | "userinfo" |
// 3D/Game assets
"glb" | "fbx" | "blend" | "blp" |
// Compressed-text formats (gzip/binary on disk)
"dia" | "tfx" | "flm" | "bcmap" | "journal" |
// Protobuf wire format
"pb" |
// Data/serialized
"parquet" | "arrow" | "pb" |
"parquet" | "arrow" |
// IDE/OS metadata
"DS_Store" | "suo"
)
crates/fff-core/src/git.rs
+15 -7
View File
@@ -5,7 +5,6 @@ use std::{
fmt::Debug,
path::{Path, PathBuf},
};
use tracing::debug;
pub(crate) fn default_status_options() -> StatusOptions {
let mut opts = StatusOptions::new();
@@ -16,6 +15,20 @@ pub(crate) fn default_status_options() -> StatusOptions {
opts
}
/// Status options for the initial scan / rescan.
///
/// Skips `include_unmodified` because every `FileItem` starts with
/// `git_status: None` (== clean), so a missing cache entry already means
/// "clean" — no need to ask libgit2 to enumerate every tracked path.
/// Saves seconds on huge dirty trees (e.g. chromium with 400k+ entries).
pub(crate) fn initial_scan_status_options() -> StatusOptions {
let mut opts = StatusOptions::new();
opts.include_untracked(true)
.recurse_untracked_dirs(true)
.exclude_submodules(true);
opts
}
#[derive(Debug, Clone, Default)]
pub(crate) struct GitStatusCache(AHashMap<PathBuf, Status>);
@@ -79,7 +92,7 @@ impl GitStatusCache {
}
}
#[tracing::instrument(skip(repo), level = tracing::Level::DEBUG)]
#[tracing::instrument(skip(repo), fields(paths_count = paths.len()), level = tracing::Level::DEBUG)]
pub fn git_status_for_paths<TPath: AsRef<Path> + Debug>(
repo: &Repository,
paths: &[TPath],
@@ -111,11 +124,6 @@ impl GitStatusCache {
}
let git_status_cache = Self::read_status_impl(repo, &mut status_options)?;
debug!(
status_len = git_status_cache.statuses_len(),
"Multiple files git status"
);
Ok(git_status_cache)
}
}
crates/fff-core/src/grep.rs
+20 -9
View File
@@ -1547,7 +1547,7 @@ fn fuzzy_grep_search<'a>(
abort_signal: &AtomicBool,
base_path: &Path,
arena: crate::simd_path::ArenaPtr,
_overflow_arena: crate::simd_path::ArenaPtr,
overflow_arena: crate::simd_path::ArenaPtr,
) -> GrepResult<'a> {
// max_typos controls how many *needle* characters can be unmatched.
// A transposition (e.g. "shcema" → "schema") costs ~1 typo with
@@ -1655,7 +1655,12 @@ fn fuzzy_grep_search<'a>(
return None;
}
let file_bytes = file.get_content_for_search(buf, arena, base_path, budget)?;
let file_arena = if file.is_overflow() {
overflow_arena
} else {
arena
};
let file_bytes = file.get_content_for_search(buf, file_arena, base_path, budget)?;
// File-level prefilter: check if enough distinct needle chars
// exist anywhere in the file bytes. Uses memchr for speed.
@@ -2155,9 +2160,15 @@ pub(crate) fn grep_search<'a>(
return true;
}
// we use ptr offsets to avoid additional allocations and keep the index
let file_idx =
unsafe { (*f as *const FileItem).offset_from(base_ptr) as usize };
// Files past the bigram boundary (unindexable base files)
// are not tracked by the bigram filter — always search them.
if file_idx >= overflow_start {
return true;
}
BigramFilter::is_candidate(candidates, file_idx)
});
}
@@ -2273,6 +2284,11 @@ fn strip_file_path_constraints<'a>(
mod tests {
use super::*;
use crate::bigram_filter::BigramIndexBuilder;
use crate::file_picker::{FilePicker, FilePickerOptions};
use std::io::Write;
use std::sync::atomic::AtomicBool;
#[test]
fn test_unescaped_newline_detection() {
// Single \n → multiline
@@ -2511,11 +2527,6 @@ mod tests {
/// unconditionally appended by the overflow loop, producing duplicates.
#[test]
fn test_grep_no_duplicates_with_overflow_trailing_bits() {
use crate::bigram_filter::{BigramIndexBuilder, BigramOverlay};
use crate::file_picker::{FilePicker, FilePickerOptions};
use std::io::Write;
use std::sync::atomic::AtomicBool;
let dir = tempfile::tempdir().unwrap();
// Match the picker's internal dunce-canonicalize so paths passed to
// on_create_or_modify resolve back to the same base_path on Windows.
@@ -2555,7 +2566,7 @@ mod tests {
}
let mut index = consec_builder.compress(Some(0));
index.set_skip_index(skip_builder.compress(Some(0)));
picker.set_bigram_index(index, BigramOverlay::new(base_count));
picker.set_bigram_index(index);
// Add three overflow files (new after the bigram index was built),
// all containing "unicorn".
crates/fff-core/src/scan.rs
+72 -101
View File
@@ -1,36 +1,12 @@
//! Unified scan-phase orchestrator.
//!
//! Every (re)index code path — initial scan, FFI-triggered rescan,
//! watcher overflow rescan — goes through [`ScanJob::run`]. The
//! orchestrator owns the *sequence* of a scan:
//!
//! 1. walk filesystem off-lock
//! 2. swap `sync_data` under a brief write
//! 3. apply git status + frecency off-lock
//! 4. (optional, initial scan only) spawn the filesystem watcher
//! 5. (optional) post-scan: auto-size cache budget, warmup, bigram
//!
//! The picker write lock is held only in step 2 and step 5's index
//! install — both O(µs-ms), never seconds. Every other FFI caller on
//! the nvim main thread keeps running.
//!
//! ## Entry points
//!
//! - [`ScanJob::spawn`] — fire-and-forget from `SharedPicker` state.
//! Used by the watcher overflow path and by FFI (`scan_files`).
//! - [`ScanJob::spawn_initial`] — same, but takes explicit config for
//! the very first scan, before the `FilePicker` struct lives inside
//! the shared handle.
use std::path::PathBuf;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use rayon::prelude::*;
use tracing::{error, info};
use crate::FileSync;
use crate::background_watcher::BackgroundWatcher;
use crate::bigram_filter::BigramOverlay;
use crate::bigram_filter::build_bigram_index;
use crate::error::Error;
use crate::file_picker::{BACKGROUND_THREAD_POOL, FFFMode, warmup_mmaps};
@@ -38,7 +14,6 @@ use crate::git::GitStatusCache;
use crate::shared::{SharedFilePicker, SharedFrecency};
use crate::simd_path::ArenaPtr;
use crate::types::ContentCacheBudget;
use rayon::prelude::*;
#[derive(Clone, Default)]
pub(crate) struct ScanSignals {
@@ -57,7 +32,7 @@ pub(crate) struct ScanSignals {
}
/// Which optional phases a scan should run.
#[derive(Clone, Copy, Default)]
#[derive(Clone, Copy, Default, Debug)]
pub(crate) struct ScanConfig {
pub(crate) warmup: bool,
pub(crate) content_indexing: bool,
@@ -199,7 +174,12 @@ impl ScanJob {
return;
}
let live_count = sync.live_count;
picker.commit_new_sync(sync);
if config.auto_cache_budget && !picker.has_explicit_cache_budget() {
picker.set_cache_budget(ContentCacheBudget::new_for_repo(live_count));
}
} else {
error!("failed to install scan results into picker");
return;
@@ -216,14 +196,38 @@ impl ScanJob {
rescubscribe_watcher_post_scan(&shared_picker);
}
// 3. Apply git status from the parallel index. This is very fast and doesn't lock
if !signals.cancelled.load(Ordering::Acquire)
&& let Some(status_handle) = status_handle
let mut snapshot = if !signals.cancelled.load(Ordering::Acquire) {
shared_picker.read().ok().and_then(|guard| {
guard
.as_ref()
.and_then(|picker| unsafe { picker.post_scan_snapshot() })
})
} else {
None
};
// 3. Post-scan warmup + bigram build — runs in parallel with the
// git-status thread to overlap the two expensive phases.
if (config.warmup || config.content_indexing)
&& !signals.cancelled.load(Ordering::Acquire)
&& let Some(snap) = snapshot.as_ref()
{
apply_git_status_and_frecency(&shared_picker, &shared_frecency, status_handle, mode);
Self::run_post_scan(&shared_picker, &signals, &config, snap);
}
// 4. Install filesystem watcher (initial scan only).
// 4. Join and git status, this HAS to be done after the post scan
if !signals.cancelled.load(Ordering::Acquire)
&& let Some(status_handle) = status_handle
&& let Some(snapshot) = snapshot.as_mut()
// THIS DOES WAIT for potentially very long status query
&& let Ok(Some(git_status)) = status_handle.join()
{
apply_git_status_and_frecency(git_status, &shared_frecency, mode, snapshot);
}
drop(snapshot); // SNAPSHOT SHOULD NOT BE USED AFTER THIS POINT
// 5. Install filesystem watcher (initial scan only).
if config.install_watcher && config.watch && !signals.cancelled.load(Ordering::Acquire) {
let shared_picker: &SharedFilePicker = &shared_picker;
let shared_frecency: &SharedFrecency = &shared_frecency;
@@ -247,12 +251,6 @@ impl ScanJob {
};
}
// 5. Post-scan warmup + bigram build.
if (config.warmup || config.content_indexing) && !signals.cancelled.load(Ordering::Acquire)
{
Self::run_post_scan(&shared_picker, &signals, &config);
}
// 6. Drain any rescan that arrived while we were busy.
// if user initiated a new rescan we had no way to cancel current post scan, so do it again
if !signals.cancelled.load(Ordering::Acquire)
@@ -276,29 +274,13 @@ impl ScanJob {
}
}
#[tracing::instrument(skip_all)]
fn run_post_scan(shared_picker: &SharedFilePicker, signals: &ScanSignals, config: &ScanConfig) {
// Auto-scale the cache budget before we take the files snapshot
if config.auto_cache_budget
&& !signals.cancelled.load(Ordering::Acquire)
&& let Ok(mut guard) = shared_picker.write()
&& let Some(picker) = guard.as_mut()
&& !picker.has_explicit_cache_budget()
{
let file_count = picker.get_files().len();
picker.set_cache_budget(ContentCacheBudget::new_for_repo(file_count));
}
// we do unsafe dirty capturing here because we GUARANTEE that the files vec is not moving anywhere
let Some(unsafe_snapshot) = shared_picker.read().ok().and_then(|guard| {
guard
.as_ref()
.and_then(|picker| unsafe { picker.post_scan_snapshot() })
}) else {
tracing::error!("Failed to commit post scan reindexing job");
return;
};
#[tracing::instrument(skip_all, fields(warmup = ?config.warmup, indexing = ?config.content_indexing))]
fn run_post_scan(
shared_picker: &SharedFilePicker,
signals: &ScanSignals,
config: &ScanConfig,
unsafe_snapshot: &crate::file_picker::PostScanUnsafeSnapshot,
) {
let arena = unsafe_snapshot
.arena
.as_ref()
@@ -307,25 +289,29 @@ impl ScanJob {
let budget: &ContentCacheBudget = &unsafe_snapshot.budget;
let files: &[crate::types::FileItem] = &unsafe_snapshot.files[..unsafe_snapshot.base_count];
if config.warmup && !signals.cancelled.load(Ordering::Acquire) {
warmup_mmaps(files, budget, &unsafe_snapshot.base_path, arena);
if signals.cancelled.load(Ordering::Acquire) {
return;
}
if config.content_indexing && !signals.cancelled.load(Ordering::Acquire) {
// unified bigram and warmup_mmaps in one go, it's important to reuse open files as much as possible
if config.content_indexing {
let indexable_files = &files[..unsafe_snapshot.indexable_count.min(files.len())];
let (index, content_binary) =
build_bigram_index(indexable_files, budget, &unsafe_snapshot.base_path, arena);
let index = build_bigram_index(
indexable_files,
budget,
&unsafe_snapshot.base_path,
arena,
config.warmup, // can be optionally skipped
);
if let Ok(mut guard) = shared_picker.write()
&& let Some(picker) = guard.as_mut()
{
for &idx in &content_binary {
if let Some(file) = picker.get_file_mut(idx) {
file.set_binary(true);
}
}
picker.set_bigram_index(index, BigramOverlay::new(unsafe_snapshot.indexable_count));
picker.set_bigram_index(index);
}
} else if config.warmup {
// Warmup-only: no bigram indexing, just fill the mmap cache.
warmup_mmaps(files, budget, &unsafe_snapshot.base_path, arena);
}
}
}
@@ -378,32 +364,17 @@ fn rescubscribe_watcher_post_scan(shared_picker: &SharedFilePicker) {
});
}
#[tracing::instrument(
level = "debug",
skip_all,
fields(file_count = tracing::field::Empty, dirty_count = tracing::field::Empty),
)]
fn apply_git_status_and_frecency(
shared_picker: &SharedFilePicker,
git_cache: GitStatusCache,
shared_frecency: &SharedFrecency,
git_handle: std::thread::JoinHandle<Option<GitStatusCache>>,
mode: FFFMode,
unsafe_snapshot: &mut crate::file_picker::PostScanUnsafeSnapshot,
) {
let git_cache = match git_handle.join() {
Ok(Some(cache)) => cache,
Ok(None) => return,
Err(_) => {
error!("Git status thread panicked");
return;
}
};
// Safety:
// apply_git_status_and_frecency is not causing any reallocas, does only sparse
// disjoint updates which is absolutely safe
let Some(mut unsafe_snapshot) = shared_picker.read().ok().and_then(|guard| {
guard
.as_ref()
.and_then(|picker| unsafe { picker.post_scan_snapshot() })
}) else {
return;
};
let frecency = shared_frecency.read().ok();
let frecency_ref = frecency.as_ref().and_then(|f| f.as_ref());
@@ -425,6 +396,10 @@ fn apply_git_status_and_frecency(
BACKGROUND_THREAD_POOL.install(|| {
files.par_iter_mut().for_each(|file| {
if unsafe_snapshot.cancelled.load(Ordering::Relaxed) {
return;
}
let mut buf = [0u8; crate::simd_path::PATH_BUF_SIZE];
let absolute_path =
file.write_absolute_path(arena, &unsafe_snapshot.base_path, &mut buf);
@@ -437,18 +412,14 @@ fn apply_git_status_and_frecency(
let score = file.access_frecency_score as i32;
if score > 0 {
let dir_idx = file.parent_dir_index() as usize;
let dir_idx = file.parent_dir_index as usize;
if let Some(dir) = dirs.get(dir_idx) {
dir.update_frecency_if_larger(score);
}
}
});
});
drop(frecency);
info!(
"SCAN: Applied git status to {} files ({} dirty)",
unsafe_snapshot.base_count,
git_cache.statuses_len(),
);
let span = tracing::Span::current();
span.record("dirty_count", git_cache.statuses_len());
}
crates/fff-core/src/shared.rs
+23 -25
View File
@@ -135,10 +135,29 @@ impl SharedFilePicker {
true
}
/// Block until both the filesystem walk and post-scan indexing are
/// idle. When both `scanning` and `post_scan_indexing_active` are
/// false, no snapshot holds Arc clones of the backing buffers, so
/// the picker can be safely torn down.
/// Block until the background file watcher is ready.
/// Returns `true` if watcher ready, `false` on timeout.
pub fn wait_for_watcher(&self, timeout: Duration) -> bool {
let watch_ready_signal = {
let guard = self.0.picker.read();
match &*guard {
Some(picker) => Arc::clone(&picker.signals.watcher_ready),
None => return true,
}
};
let start = std::time::Instant::now();
while !watch_ready_signal.load(std::sync::atomic::Ordering::Acquire) {
if start.elapsed() >= timeout {
return false;
}
std::thread::sleep(Duration::from_millis(10));
}
true
}
/// Blocks until both the filesystem walk and post-scan indexing are done.
/// Returns true once scanning=false AND post_scan_indexing_active=false.
pub fn wait_for_indexing_complete(&self, timeout: Duration) -> bool {
let (scanning, post_scan_active) = {
let guard = self.0.picker.read();
@@ -165,27 +184,6 @@ impl SharedFilePicker {
}
}
/// Block until the background file watcher is ready.
/// Returns `true` if watcher ready, `false` on timeout.
pub fn wait_for_watcher(&self, timeout: Duration) -> bool {
let watch_ready_signal = {
let guard = self.0.picker.read();
match &*guard {
Some(picker) => Arc::clone(&picker.signals.watcher_ready),
None => return true,
}
};
let start = std::time::Instant::now();
while !watch_ready_signal.load(std::sync::atomic::Ordering::Acquire) {
if start.elapsed() >= timeout {
return false;
}
std::thread::sleep(Duration::from_millis(10));
}
true
}
/// Trigger a full filesystem rescan without blocking the caller.
/// Performs a safe async rescan. Guarantees only single active rescan per picker.
/// If many rescans requested the last one guaranteed to be finished.
crates/fff-core/src/types.rs
+176 -130
View File
@@ -1,7 +1,8 @@
use std::io::Read;
use std::path::{Path, PathBuf};
#[cfg(not(target_os = "windows"))]
use std::sync::OnceLock;
use std::sync::atomic::{AtomicI32, AtomicU64, AtomicUsize, Ordering};
use std::sync::atomic::{AtomicI32, AtomicU8, AtomicU64, AtomicUsize, Ordering};
use crate::constraints::Constrainable;
use crate::query_tracker::QueryMatchEntry;
@@ -49,33 +50,6 @@ impl FileSliceExt for [FileItem] {
}
}
/// Cached file contents — mmap on Unix, heap buffer on Windows.
///
/// On Windows, memory-mapped files hold the file handle open and prevent
/// editors from saving (writing/replacing) those files. Reading into a
/// `Vec<u8>` releases the handle immediately after the read completes.
///
/// The `Buffer` variant is also used on Unix for temporary (uncached) reads
/// where the mmap/munmap syscall overhead exceeds the cost of a heap copy.
#[derive(Debug)]
#[allow(dead_code)] // variants are conditionally used per platform
pub enum FileContent {
#[cfg(not(target_os = "windows"))]
Mmap(memmap2::Mmap),
Buffer(Vec<u8>),
}
impl std::ops::Deref for FileContent {
type Target = [u8];
fn deref(&self) -> &[u8] {
match self {
#[cfg(not(target_os = "windows"))]
FileContent::Mmap(m) => m,
FileContent::Buffer(b) => b,
}
}
}
pub struct FileItemFlags;
impl FileItemFlags {
@@ -238,23 +212,25 @@ pub struct FileItem {
pub modification_frecency_score: i16,
pub git_status: Option<git2::Status>,
pub(crate) path: crate::simd_path::ChunkedString,
parent_dir: u32,
flags: u8,
content: OnceLock<FileContent>,
pub(crate) parent_dir_index: u32,
flags: AtomicU8,
#[cfg(not(target_os = "windows"))]
content: OnceLock<memmap2::Mmap>,
}
impl Clone for FileItem {
fn clone(&self) -> Self {
Self {
path: self.path.clone(),
parent_dir: self.parent_dir,
parent_dir_index: self.parent_dir_index,
size: self.size,
modified: self.modified,
access_frecency_score: self.access_frecency_score,
modification_frecency_score: self.modification_frecency_score,
git_status: self.git_status,
flags: self.flags,
flags: AtomicU8::new(self.flags.load(Ordering::Relaxed)),
// on clone we have to reset the content lock
#[cfg(not(target_os = "windows"))]
content: OnceLock::new(),
}
}
@@ -278,13 +254,14 @@ impl FileItem {
Self {
path,
parent_dir: u32::MAX,
parent_dir_index: u32::MAX,
size,
modified,
access_frecency_score: 0,
modification_frecency_score: 0,
git_status,
flags,
flags: AtomicU8::new(flags),
#[cfg(not(target_os = "windows"))]
content: OnceLock::new(),
}
}
@@ -300,14 +277,6 @@ impl FileItem {
self.path = path;
}
pub(crate) fn parent_dir_index(&self) -> u32 {
self.parent_dir
}
pub(crate) fn set_parent_dir(&mut self, idx: u32) {
self.parent_dir = idx;
}
pub fn dir_str(&self, arena: impl FFFStringStorage) -> String {
let mut s = String::with_capacity(64);
self.path.write_dir_to(arena.arena_for(self), &mut s);
@@ -413,55 +382,164 @@ impl FileItem {
}
#[inline]
pub fn is_binary(&self) -> bool {
self.flags & FileItemFlags::BINARY != 0
pub(crate) fn is_likely_hot(&self) -> bool {
self.access_frecency_score > 0 || self.git_status.is_some()
}
/// Reads a fixed bytes count from the file optimized for quick speed of opening
#[inline]
pub(crate) fn read_trimmed_into_buf(
&self,
base_fd: i32,
base_path: &Path,
arena: ArenaPtr,
path_buf: &mut [u8; PATH_BUF_SIZE],
buf: &mut [u8],
) -> usize {
#[cfg(unix)]
{
self.read_into_buf_unix(base_fd, base_path, arena, path_buf, buf)
}
#[cfg(not(unix))]
{
let _ = base_fd;
self.read_into_buf_std(base_path, arena, path_buf, buf)
}
}
#[cfg(unix)]
fn read_into_buf_unix(
&self,
base_fd: libc::c_int,
base_path: &Path,
arena: ArenaPtr,
path_buf: &mut [u8; PATH_BUF_SIZE],
buf: &mut [u8],
) -> usize {
let fd = if base_fd >= 0 {
let relative_path = self.write_relative_cstr(arena, path_buf);
// SAFETY: `relative_path` is NUL-terminated, `base_fd` is a
// valid directory descriptor owned by the caller.
unsafe { libc::openat(base_fd, relative_path.as_ptr(), libc::O_RDONLY) }
} else {
use std::os::unix::io::IntoRawFd;
let abs = self.write_absolute_path(arena, base_path, path_buf);
match std::fs::File::open(abs) {
Ok(f) => f.into_raw_fd(),
Err(e) => {
tracing::error!(?e, "Failed to fopen file");
return 0;
}
}
};
if fd < 0 {
return 0;
}
let mut filled = 0usize;
while filled < buf.len() {
// SAFETY: `fd` is an owned descriptor, `buf[filled..]` is a
// valid writable slice for `buf.len() - filled` bytes.
let n = unsafe {
libc::read(
fd,
buf[filled..].as_mut_ptr() as *mut libc::c_void,
(buf.len() - filled) as libc::size_t,
)
};
if n <= 0 {
break;
}
filled += n as usize;
}
// SAFETY: matching close for the owned descriptor.
unsafe { libc::close(fd) };
filled
}
#[cfg(not(unix))]
fn read_into_buf_std(
&self,
base_path: &Path,
arena: ArenaPtr,
path_buf: &mut [u8; PATH_BUF_SIZE],
buf: &mut [u8],
) -> usize {
let abs = self.write_absolute_path(arena, base_path, path_buf);
let Ok(mut f) = std::fs::File::open(abs) else {
return 0;
};
let mut filled = 0usize;
while filled < buf.len() {
match f.read(&mut buf[filled..]) {
Ok(0) => break,
Ok(n) => filled += n,
Err(_) => return 0,
}
}
filled
}
#[inline]
pub fn set_binary(&mut self, val: bool) {
pub fn is_binary(&self) -> bool {
self.flags.load(Ordering::Relaxed) & FileItemFlags::BINARY != 0
}
#[inline]
pub fn set_binary(&self, val: bool) {
if val {
self.flags |= FileItemFlags::BINARY;
self.flags
.fetch_or(FileItemFlags::BINARY, Ordering::Relaxed);
} else {
self.flags &= !FileItemFlags::BINARY;
self.flags
.fetch_and(!FileItemFlags::BINARY, Ordering::Relaxed);
}
}
#[inline]
pub fn is_deleted(&self) -> bool {
self.flags & FileItemFlags::DELETED != 0
self.flags.load(Ordering::Relaxed) & FileItemFlags::DELETED != 0
}
#[inline]
pub fn set_deleted(&mut self, val: bool) {
#[doc(hidden)]
/// Don't use it, use FilePicker::delete_file
pub fn set_deleted(&self, val: bool) {
if val {
self.flags |= FileItemFlags::DELETED;
self.flags
.fetch_or(FileItemFlags::DELETED, Ordering::Relaxed);
} else {
self.flags &= !FileItemFlags::DELETED;
self.flags
.fetch_and(!FileItemFlags::DELETED, Ordering::Relaxed);
}
}
#[inline]
pub fn is_overflow(&self) -> bool {
self.flags & FileItemFlags::OVERFLOW != 0
self.flags.load(Ordering::Relaxed) & FileItemFlags::OVERFLOW != 0
}
#[inline]
pub fn set_overflow(&mut self, val: bool) {
pub fn set_overflow(&self, val: bool) {
if val {
self.flags |= FileItemFlags::OVERFLOW;
self.flags
.fetch_or(FileItemFlags::OVERFLOW, Ordering::Relaxed);
} else {
self.flags &= !FileItemFlags::OVERFLOW;
self.flags
.fetch_and(!FileItemFlags::OVERFLOW, Ordering::Relaxed);
}
}
}
impl FileItem {
/// Invalidate the cached content so the next `get_content()` call creates a fresh one.
/// Invalidate the cached mmap content, has to be called every time the file is updated.
///
/// Call this when the background watcher detects that the file has been modified.
/// On Unix, a file that is truncated while mapped can cause SIGBUS. On Windows,
/// the stale buffer simply won't reflect the new contents. In both cases,
/// invalidating ensures a fresh read on the next access.
#[cfg(not(target_os = "windows"))]
pub fn invalidate_mmap(&mut self, budget: &ContentCacheBudget) {
if self.content.get().is_some() {
budget.cached_count.fetch_sub(1, Ordering::Relaxed);
@@ -471,6 +549,9 @@ impl FileItem {
self.content = OnceLock::new();
}
#[cfg(target_os = "windows")]
pub fn invalidate_mmap(&mut self, _: &ContentCacheBudget) {}
pub fn update_metadata(
&mut self,
budget: &ContentCacheBudget,
@@ -497,7 +578,23 @@ impl FileItem {
/// of the budget should use [`get_content_for_search`].
///
/// After the first call, this is lock-free (just an atomic load + pointer deref).
pub(crate) fn get_content(
///
/// On Windows we never back this cache — `memmap2` would require a full
/// `std::fs::read` heap copy and the OS page cache already absorbs repeat
/// reopens. Returning `None` keeps callers on the scratch-read slow path
/// and avoids duplicating every indexed file on the heap.
#[cfg(target_os = "windows")]
pub(crate) fn get_cached_content(
&self,
_arena: ArenaPtr,
_base_path: &Path,
_budget: &ContentCacheBudget,
) -> Option<&[u8]> {
None
}
#[cfg(not(target_os = "windows"))]
pub(crate) fn get_cached_content(
&self,
arena: ArenaPtr,
base_path: &Path,
@@ -507,8 +604,11 @@ impl FileItem {
return Some(content);
}
let max_file_size = budget.max_file_size;
if self.size == 0 || self.size > max_file_size {
// Skip caching when mmap can't pay for itself. Files under one page
// worth of bytes waste kernel VM structures and a per-file syscall
// pair — the chunked `read_into_buf` fallback is cheaper for them
// and hits the OS page cache on repeat reads anyway.
if self.size < MMAP_THRESHOLD || self.size > budget.max_file_size {
return None;
}
@@ -521,11 +621,14 @@ impl FileItem {
return None;
}
let content = load_file_content(&self.absolute_path(arena, base_path), self.size)?;
let result = self.content.get_or_init(|| content);
let path = self.absolute_path(arena, base_path);
let file = std::fs::File::open(&path).ok()?;
// SAFETY: the mmap is backed by the kernel page cache and reflects
// file updates; the only risk is SIGBUS on a concurrent truncate,
// which the watcher mitigates by invalidating on modification.
let mmap = unsafe { memmap2::Mmap::map(&file) }.ok()?;
let result = self.content.get_or_init(|| mmap);
// Bump counters. Slight over-count under races is fine — the budget
// is a soft limit and the overshoot is bounded by rayon thread count.
budget.cached_count.fetch_add(1, Ordering::Relaxed);
budget.cached_bytes.fetch_add(self.size, Ordering::Relaxed);
@@ -546,7 +649,7 @@ impl FileItem {
budget: &ContentCacheBudget,
) -> Option<&'a [u8]> {
// Fast path: persistent cache hit (zero-copy).
if let Some(cached) = self.get_content(arena, base_path, budget) {
if let Some(cached) = self.get_cached_content(arena, base_path, budget) {
return Some(cached);
}
@@ -569,36 +672,13 @@ impl FileItem {
}
/// Files smaller than one page waste the remainder when mmapped.
/// Unused on Windows where `load_file_content` does not mmap.
/// Files smaller than one page waste the remainder when mmapped. Unused
/// on Windows where the persistent content cache is disabled.
#[cfg(all(not(target_os = "windows"), target_arch = "aarch64"))]
const MMAP_THRESHOLD: u64 = 16 * 1024;
#[cfg(all(not(target_os = "windows"), not(target_arch = "aarch64")))]
const MMAP_THRESHOLD: u64 = 4 * 1024;
fn load_file_content(path: &Path, size: u64) -> Option<FileContent> {
#[cfg(not(target_os = "windows"))]
{
if size < MMAP_THRESHOLD {
let data = std::fs::read(path).ok()?;
Some(FileContent::Buffer(data))
} else {
let file = std::fs::File::open(path).ok()?;
// SAFETY: The mmap is backed by the kernel page cache and automatically
// reflects file modifications. The only risk is SIGBUS if the file is
// truncated while mapped.
let mmap = unsafe { memmap2::Mmap::map(&file) }.ok()?;
Some(FileContent::Mmap(mmap))
}
}
#[cfg(target_os = "windows")]
{
let _ = size;
let data = std::fs::read(path).ok()?;
Some(FileContent::Buffer(data))
}
}
impl Constrainable for FileItem {
#[inline]
fn write_file_name(&self, arena: ArenaPtr, out: &mut String) {
@@ -748,6 +828,12 @@ impl ContentCacheBudget {
}
}
// Byte budget
pub fn is_exhausted(&self) -> bool {
self.cached_count.load(Ordering::Relaxed) >= self.max_files
|| self.cached_bytes.load(Ordering::Relaxed) >= self.max_bytes
}
pub fn new_for_repo(file_count: usize) -> Self {
let max_files = if file_count > 50_000 {
5_000
@@ -810,43 +896,3 @@ impl Default for ContentCacheBudget {
Self::new_for_repo(30_000)
}
}
#[cfg(test)]
impl FileItem {
/// Leaks a single-file arena so the pointer stays valid forever.
pub fn new_for_test(
rel_path: &str,
size: u64,
modified: u64,
git_status: Option<git2::Status>,
is_binary: bool,
) -> Self {
let (item, _arena) =
Self::new_for_test_with_arena(rel_path, size, modified, git_status, is_binary);
item
}
pub(crate) fn new_for_test_with_arena(
rel_path: &str,
size: u64,
modified: u64,
git_status: Option<git2::Status>,
is_binary: bool,
) -> (Self, ArenaPtr) {
let filename_start = rel_path
.rfind(std::path::is_separator)
.map(|i| i + 1)
.unwrap_or(0) as u16;
let mut item = Self::new_raw(filename_start, size, modified, git_status, is_binary);
let paths = [rel_path.to_string()];
let (store, strings) = crate::simd_path::build_chunked_path_store_from_strings(
&paths,
std::slice::from_ref(&item),
);
let cs = strings.into_iter().next().unwrap();
let arena = store.as_arena_ptr();
item.set_path(cs);
std::mem::forget(store);
(item, arena)
}
}
crates/fff-core/tests/bigram_overlay_coherence_test.rs
-19
View File
@@ -284,12 +284,6 @@ fn bigram_overlay_coherence_proves_contribution_for_modified_base() {
let with_overlay = grep_count(picker, unique);
assert_eq!(with_overlay, 1, "overlay should find the new token");
let without_overlay = grep_without_overlay_count(picker, unique);
assert_eq!(
without_overlay, 0,
"without overlay, bigram should exclude the file (stale bigrams)"
);
}
stop_picker(&shared_picker);
@@ -806,14 +800,6 @@ fn bigram_overlay_coherence_rescan_after_git_commit() {
with >= 1,
"post-rescan: edited token {token} should be findable"
);
// The content is now in the base index, so it should be
// findable even without the overlay.
let without = grep_without_overlay_count(picker, token);
assert!(
without >= 1,
"post-rescan: {token} should be in base index (without overlay: {without})"
);
}
for token in &new_tokens {
@@ -1339,11 +1325,6 @@ fn grep_count(picker: &FilePicker, query: &str) -> usize {
picker.grep(&parsed, &grep_opts()).matches.len()
}
fn grep_without_overlay_count(picker: &FilePicker, query: &str) -> usize {
let parsed = parse_grep_query(query);
picker.grep_original(&parsed, &grep_opts()).matches.len()
}
/// Wait for scanning to finish (no bigram requirement).
/// Use after `trigger_rescan` which replaces sync_data but does not
/// rebuild the bigram index.
crates/fff-core/tests/bigram_overlay_integration.rs
-15
View File
@@ -144,21 +144,6 @@ fn modified_file_findable_via_overlay() {
);
}
// Prove the overlay is actually doing something: without it, the bigram
// index would filter out beta.txt and the search would miss the needle.
{
let guard = shared_picker.read().unwrap();
let picker = guard.as_ref().unwrap();
let parsed = parse_grep_query("UNIQUE_NEEDLE");
let opts = grep_opts();
let result = picker.grep_original(&parsed, &opts);
assert_eq!(
result.matches.len(),
0,
"Without overlay, bigram prefiltering should exclude the modified file"
);
}
// Cleanup: stop background watcher.
if let Ok(mut guard) = shared_picker.write() {
if let Some(ref mut picker) = *guard {
crates/fff-core/tests/fuzz_file_operations.rs
+90
View File
@@ -768,3 +768,93 @@ fn git_init_and_commit(dir: &Path) {
git_run(dir, &["add", "-A"]);
git_run(dir, &["commit", "-m", "initial"]);
}
/// Proves that dropping the picker while post-scan (warmup + bigram build)
/// is actively iterating raw pointers does NOT segfault. The Drop impl
/// sets `cancelled`, waits for `post_scan_indexing_active` to clear, and
/// only then frees the backing Vec.
///
/// Runs 10 iterations to exercise the race window reliably.
#[test]
fn drop_during_post_scan_does_not_crash() {
let mut caught_active = 0u32;
for round in 0..10 {
let tmp = TempDir::new().unwrap();
let base = tmp.path();
// Create enough files so bigram build takes measurable time
for i in 0..2000 {
let dir = base.join(format!("d_{:02}", i % 20));
fs::create_dir_all(&dir).unwrap();
let content = format!(
"fn func_{i}() {{ let x = {i}; println!(\"{{x}}\"); }}\n\
const T_{i}: &str = \"TOKEN_{i}\";\n"
);
fs::write(dir.join(format!("f_{i:04}.rs")), content).unwrap();
}
git_init_and_commit(base);
let shared_picker = SharedFilePicker::default();
FilePicker::new_with_shared_state(
shared_picker.clone(),
SharedFrecency::noop(),
FilePickerOptions {
base_path: base.to_string_lossy().to_string(),
enable_mmap_cache: true,
enable_content_indexing: true,
watch: false,
mode: FFFMode::Neovim,
..Default::default()
},
)
.unwrap();
// Wait for scan but NOT for bigram — drop while post-scan is active
shared_picker.wait_for_scan(Duration::from_secs(10));
// Poll until post_scan_indexing_active is true (bigram started)
let deadline = std::time::Instant::now() + Duration::from_secs(5);
let mut was_active = false;
loop {
if let Ok(guard) = shared_picker.read() {
if let Some(picker) = guard.as_ref() {
if picker.is_post_scan_active() {
was_active = true;
break;
}
}
}
if std::time::Instant::now() > deadline {
break;
}
std::thread::sleep(Duration::from_millis(1));
}
if was_active {
caught_active += 1;
}
// Drop the picker while post_scan_indexing_active is set.
// Take it out of the shared handle first, then drop outside the lock —
// Drop spins until post-scan finishes, which needs the write lock for
// bigram install, so we can't hold it during Drop.
let old_picker = shared_picker.write().unwrap().take();
drop(old_picker); // Drop fires here — spins until post-scan exits
assert!(
shared_picker.read().unwrap().is_none(),
"round {round}: picker should be None after drop"
);
}
// At least some rounds must have caught the post-scan active window
assert!(
caught_active > 0,
"Test didn't catch post_scan_indexing_active=true in any round. \
The test is not exercising the race. ({caught_active}/10)"
);
eprintln!("Caught post-scan active in {caught_active}/10 rounds");
}
crates/fff-core/tests/fuzz_git_watcher_stress.rs
+79 -3
View File
@@ -912,6 +912,73 @@ fn grep_plain_matches(shared: &SharedFilePicker, query: &str) -> Vec<String> {
.collect()
}
/// Run live grep (fuzzy mode) and return matched file paths.
/// Exercises the `fuzzy_grep_search` code path which resolves content
/// via arena pointers — the path that was silently broken for overflow
/// files before the overflow_arena fix.
fn grep_fuzzy_matches(shared: &SharedFilePicker, query: &str) -> Vec<String> {
let guard = match shared.read() {
Ok(g) => g,
Err(_) => return Vec::new(),
};
let Some(picker) = guard.as_ref() else {
return Vec::new();
};
let parsed = parse_grep_query(query);
let opts = GrepSearchOptions {
max_file_size: 10 * 1024 * 1024,
max_matches_per_file: 200,
smart_case: true,
file_offset: 0,
page_limit: 500,
mode: GrepMode::Fuzzy,
time_budget_ms: 0,
before_context: 0,
after_context: 0,
classify_definitions: false,
trim_whitespace: false,
abort_signal: None,
};
let result = picker.grep(&parsed, &opts);
result
.files
.iter()
.map(|f| normalize(f.relative_path(picker)))
.collect()
}
/// Run live grep (regex mode) and return matched file paths.
fn grep_regex_matches(shared: &SharedFilePicker, query: &str) -> Vec<String> {
let guard = match shared.read() {
Ok(g) => g,
Err(_) => return Vec::new(),
};
let Some(picker) = guard.as_ref() else {
return Vec::new();
};
let parsed = parse_grep_query(query);
let opts = GrepSearchOptions {
max_file_size: 10 * 1024 * 1024,
max_matches_per_file: 200,
smart_case: true,
file_offset: 0,
page_limit: 500,
mode: GrepMode::Regex,
time_budget_ms: 0,
before_context: 0,
after_context: 0,
classify_definitions: false,
trim_whitespace: false,
abort_signal: None,
};
let result = picker.grep(&parsed, &opts);
result
.files
.iter()
.map(|f| normalize(f.relative_path(picker)))
.collect()
}
/// Report from [`probe_real_queries`]. `None` means "nothing to probe this
/// round" (empty live set). `Some(Err)` means a probe disagreed with truth
/// — convergence should not treat this as success.
@@ -952,12 +1019,21 @@ fn probe_real_queries(shared: &SharedFilePicker, live: &[Live]) -> ProbeOutcome
}
}
// --- Grep probe: search for the content marker ---
// --- Grep probe: search for the content marker using a randomly
// rotated grep strategy. Each round picks one of PlainText / Fuzzy /
// Regex so over many rounds all three code paths get exercised,
// including the overflow-arena resolution that was previously broken
// in fuzzy grep.
if let Some(marker) = extract_marker(&target.abs) {
let matches = grep_plain_matches(shared, &marker);
let probe_round = PROBE_COUNTER.load(Ordering::Relaxed);
let (mode_name, matches) = match probe_round % 3 {
0 => ("plain", grep_plain_matches(shared, &marker)),
1 => ("fuzzy", grep_fuzzy_matches(shared, &marker)),
_ => ("regex", grep_regex_matches(shared, &marker)),
};
if !matches.contains(&target.relative) {
return Some(Err(format!(
"grep({marker:?}) did not return expected live file {:?}\n\
"grep[{mode_name}]({marker:?}) did not return expected live file {:?}\n\
got {} matched files; first few: {:?}",
target.relative,
matches.len(),
crates/fff-core/tests/fuzz_real_repos.proptest-regressions
+7
View File
File diff suppressed because one or more lines are too long
crates/fff-core/tests/fuzz_real_repos.rs
+740
View File
@@ -0,0 +1,740 @@
//! Proptest-driven fuzz test against real GitHub repos with a live watcher.
//!
//! Clones real repository, runs the simulated close to real user sereies of file system ewvents and
//! verifies that fff can still find the correct files. Test cases are randomized and preserved
//! using proptest
//!
//! Run:
//! ```sh
//! RUSTFLAGS="--cfg stress" cargo test -p fff-search --test fuzz_real_repos -- --nocapture
//! ```
//!
//! Increase coverage:
//! ```sh
//! FFF_FUZZ_CASES=4 FFF_FUZZ_MAX_OPS=60 \
//! RUSTFLAGS="--cfg stress" cargo test -p fff-search --test fuzz_real_repos -- --nocapture
//! ```
#![cfg(stress)]
use std::fs;
use std::path::{Path, PathBuf};
use std::process::Command;
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use proptest::prelude::*;
use proptest::test_runner::{Config as ProptestConfig, FileFailurePersistence};
use fff_search::file_picker::{FFFMode, FilePicker, is_known_binary_extension};
use fff_search::grep::{GrepMode, GrepSearchOptions, parse_grep_query};
use fff_search::{FilePickerOptions, SharedFilePicker, SharedFrecency};
const REPO_POOL: &[(&str, &str)] = &[
("dmtrKovalenko/fff", "fff"),
("BurntSushi/ripgrep", "ripgrep"),
("sharkdp/fd", "fd"),
("ogham/exa", "exa"),
("casey/just", "just"),
("ajeetdsouza/zoxide", "zoxide"),
("helix-editor/helix", "helix"),
("astral-sh/ruff", "ruff"),
("biomejs/biome", "biome"),
("denoland/deno_lint", "deno_lint"),
("nickel-lang/nickel", "nickel"),
("typst/typst", "typst"),
("gleam-lang/gleam", "gleam"),
("pretzelhammer/rust-blog", "rust-blog"),
("tokio-rs/mini-redis", "mini-redis"),
];
const CACHE_DIR: &str = "/tmp/fff_fuzz_repos";
/// Fixed settle time for watcher event propagation.
const WATCHER_SETTLE: Duration = Duration::from_millis(100);
/// Maximum time to wait for watcher to process all pending events.
const CONVERGE_TIMEOUT: Duration = Duration::from_secs(30);
fn fuzz_cases() -> u32 {
std::env::var("FFF_FUZZ_CASES")
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(2)
}
fn fuzz_max_ops() -> usize {
std::env::var("FFF_FUZZ_MAX_OPS")
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(30)
}
fn fuzz_min_ops() -> usize {
std::env::var("FFF_FUZZ_MIN_OPS")
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(15)
}
fn ensure_repo_cloned(repo_url: &str, local_name: &str) -> PathBuf {
let cache = PathBuf::from(CACHE_DIR);
fs::create_dir_all(&cache).unwrap();
let repo_path = cache.join(local_name);
if repo_path.join(".git").exists() {
return repo_path;
}
let full_url = format!("https://github.com/{}.git", repo_url);
eprintln!(" Cloning {} ...", full_url);
let out = Command::new("git")
.args(["clone", "--depth=1", "--single-branch", &full_url])
.arg(&repo_path)
.output()
.expect("git clone failed");
assert!(
out.status.success(),
"git clone {} failed: {}",
full_url,
String::from_utf8_lossy(&out.stderr)
);
repo_path
}
fn copy_repo_to_workdir(cached: &Path, workdir: &Path) {
let out = Command::new("cp")
.args(["-r"])
.arg(cached)
.arg(workdir)
.output()
.expect("cp -r failed");
assert!(
out.status.success(),
"cp -r failed: {}",
String::from_utf8_lossy(&out.stderr)
);
}
fn collect_text_files(base: &Path) -> Vec<PathBuf> {
// Use `git ls-files` without --cached to get only files that are both
// tracked AND not gitignored. Files like Cargo.lock that are committed
// but in .gitignore would appear with --cached but the fff picker skips
// them during walk (respects .gitignore), causing false test failures.
let out = Command::new("git")
.args(["ls-files", "--others", "--exclude-standard", "-z"])
.current_dir(base)
.output()
.unwrap();
// Get tracked files that aren't ignored
let tracked = Command::new("git")
.args(["ls-files", "-z"])
.current_dir(base)
.output()
.unwrap();
// Check which tracked files are actually ignored
let ignored_check = Command::new("git")
.args(["check-ignore", "--stdin", "-z"])
.stdin(std::process::Stdio::piped())
.stdout(std::process::Stdio::piped())
.current_dir(base)
.spawn();
let mut ignored_set: std::collections::HashSet<String> = std::collections::HashSet::new();
if let Ok(mut child) = ignored_check {
use std::io::Write;
if let Some(ref mut stdin) = child.stdin {
let _ = stdin.write_all(&tracked.stdout);
}
if let Ok(output) = child.wait_with_output() {
for path in output.stdout.split(|&b| b == 0) {
if !path.is_empty() {
if let Ok(s) = std::str::from_utf8(path) {
ignored_set.insert(s.to_string());
}
}
}
}
}
// Combine: tracked non-ignored non-binary files
let mut files: Vec<PathBuf> = Vec::new();
for path in tracked.stdout.split(|&b| b == 0) {
if path.is_empty() {
continue;
}
let Ok(s) = std::str::from_utf8(path) else {
continue;
};
if ignored_set.contains(s) {
continue;
}
let full = base.join(s);
if full.is_file() && !is_known_binary_extension(&full) {
files.push(full);
}
}
files
}
/// Edit a file by injecting a marker line at a deterministic position,
/// preserving the rest of the content. Returns the original line that was
/// replaced so it can be restored on revert.
fn inject_marker(path: &Path, marker: &str, seed: u32) -> Option<String> {
let content = fs::read_to_string(path).ok()?;
let lines: Vec<&str> = content.lines().collect();
if lines.is_empty() {
fs::write(path, format!("// {marker}\n")).ok()?;
return Some(String::new());
}
// Pick a stable line position based on seed and file length
let line_idx = seed as usize % lines.len();
let original_line = lines[line_idx].to_string();
let mut result = String::with_capacity(content.len() + marker.len() + 10);
for (i, line) in lines.iter().enumerate() {
if i == line_idx {
result.push_str(&format!("// {marker}"));
} else {
result.push_str(line);
}
result.push('\n');
}
fs::write(path, &result).ok()?;
Some(original_line)
}
/// Revert a file by restoring the original line at the same position
/// where inject_marker placed the marker.
fn revert_marker(path: &Path, marker: &str, original_line: &str) {
let Ok(content) = fs::read_to_string(path) else {
return;
};
let marker_line = format!("// {marker}");
let result: String = content
.lines()
.map(|l| if l == marker_line { original_line } else { l })
.collect::<Vec<_>>()
.join("\n")
+ "\n";
let _ = fs::write(path, result);
}
// ═══════════════════════════════════════════════════════════════════════════
// Search helpers
// ═══════════════════════════════════════════════════════════════════════════
fn grep_opts(mode: GrepMode) -> GrepSearchOptions {
GrepSearchOptions {
max_file_size: 10 * 1024 * 1024,
max_matches_per_file: 200,
smart_case: true,
file_offset: 0,
page_limit: 500,
mode,
time_budget_ms: 5000,
before_context: 0,
after_context: 0,
classify_definitions: false,
trim_whitespace: false,
abort_signal: None,
}
}
fn grep_finds(picker: &FilePicker, query: &str, mode: GrepMode) -> bool {
let parsed = parse_grep_query(query);
let result = picker.grep(&parsed, &grep_opts(mode));
!result.matches.is_empty()
}
fn grep_file_list(picker: &FilePicker, query: &str, mode: GrepMode) -> Vec<String> {
let parsed = parse_grep_query(query);
let result = picker.grep(&parsed, &grep_opts(mode));
result
.files
.iter()
.map(|f| f.relative_path(picker))
.collect()
}
// ═══════════════════════════════════════════════════════════════════════════
// Infrastructure
// ═══════════════════════════════════════════════════════════════════════════
fn wait_for_bigram(sp: &SharedFilePicker) {
let deadline = Instant::now() + Duration::from_secs(120);
loop {
std::thread::sleep(Duration::from_millis(50));
let ready = sp
.read()
.ok()
.map(|g| {
g.as_ref()
.map_or(false, |p| !p.is_scan_active() && p.bigram_index().is_some())
})
.unwrap_or(false);
if ready {
return;
}
assert!(
Instant::now() < deadline,
"Timed out waiting for bigram index"
);
}
}
fn epoch_secs() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_secs()
}
struct TrackedFile {
relative: String,
marker: String,
/// The original line content that was replaced, for revert
original_line: String,
is_created: bool,
last_write_sec: u64,
}
fn run_scenario(ops: &[Op]) {
// Stream fff logs at info+ level by default. Override with RUST_LOG.
let _ = tracing_subscriber::fmt()
.with_env_filter(
tracing_subscriber::EnvFilter::try_from_default_env()
.unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("warn,fff_search=info")),
)
.with_test_writer()
.try_init();
// Allow forcing a specific repo via env for reproduction
let repo_idx = std::env::var("FFF_FUZZ_REPO_IDX")
.ok()
.and_then(|v| v.parse::<usize>().ok())
.unwrap_or_else(|| ops.len() % REPO_POOL.len());
let (repo_url, local_name) = REPO_POOL[repo_idx];
eprintln!("=== fuzz_real_repos: repo={repo_url} ops={} ===", ops.len());
let scenario_start = Instant::now();
let cached = ensure_repo_cloned(repo_url, local_name);
let tmp = tempfile::TempDir::new().unwrap();
let workdir = tmp.path().join(local_name);
copy_repo_to_workdir(&cached, &workdir);
// Ensure target/ is gitignored
let gitignore = workdir.join(".gitignore");
let mut gi = fs::read_to_string(&gitignore).unwrap_or_default();
if !gi.contains("target/") {
gi.push_str("\ntarget/\n");
fs::write(&gitignore, &gi).unwrap();
}
let shared_picker = SharedFilePicker::default();
FilePicker::new_with_shared_state(
shared_picker.clone(),
SharedFrecency::noop(),
FilePickerOptions {
base_path: workdir.to_string_lossy().to_string(),
enable_mmap_cache: true,
enable_content_indexing: true,
watch: true,
mode: FFFMode::Neovim,
..Default::default()
},
)
.expect("FilePicker init");
let t0 = Instant::now();
wait_for_bigram(&shared_picker);
let bigram_ms = t0.elapsed().as_secs_f64() * 1000.0;
{
let guard = shared_picker.read().unwrap();
let picker = guard.as_ref().unwrap();
let file_count = picker.get_files().len();
eprintln!(" indexed {file_count} files, bigram ready in {bigram_ms:.0}ms");
}
// Advance mtime past scan timestamp
std::thread::sleep(Duration::from_millis(1100));
let mut tracked: Vec<TrackedFile> = Vec::new();
let mut dead_markers: Vec<String> = Vec::new();
let mut ignored_markers: Vec<String> = Vec::new();
let mut ops_since_verify: usize = 0;
let mut text_files: Option<Vec<PathBuf>> = None;
for (op_idx, op) in ops.iter().enumerate() {
match op {
Op::CreateFile { seed } => {
let name = format!("fff_fuzz_new_{seed:08x}.rs");
let marker = format!("FFF_FUZZ_NEW_{seed:08x}");
// Marker appears only once on its own line
let content = format!("// {marker}\nfn placeholder() {{}}\n");
fs::write(workdir.join(&name), content).unwrap();
tracked.push(TrackedFile {
relative: name,
marker,
original_line: String::new(),
is_created: true,
last_write_sec: epoch_secs(),
});
ops_since_verify += 1;
}
Op::EditTracked { seed } => {
if tracked.is_empty() {
continue;
}
let idx = *seed as usize % tracked.len();
if tracked[idx].last_write_sec >= epoch_secs() {
std::thread::sleep(Duration::from_millis(1100));
}
let new_marker = format!("FFF_FUZZ_EDIT_{seed:08x}");
let path = workdir.join(&tracked[idx].relative);
// Replace the line containing our old marker with the new one
let old_marker_line = format!("// {}", tracked[idx].marker);
let content = fs::read_to_string(&path).unwrap_or_default();
let new_content = content
.lines()
.map(|l| {
if l == old_marker_line {
format!("// {new_marker}")
} else {
l.to_string()
}
})
.collect::<Vec<_>>()
.join("\n")
+ "\n";
fs::write(&path, new_content).unwrap();
dead_markers.push(tracked[idx].marker.clone());
tracked[idx].marker = new_marker;
tracked[idx].last_write_sec = epoch_secs();
ops_since_verify += 1;
}
Op::EditRandom { seed } => {
let files = text_files.get_or_insert_with(|| collect_text_files(&workdir));
if files.is_empty() {
continue;
}
let target = &files[*seed as usize % files.len()];
let relative = target
.strip_prefix(&workdir)
.unwrap()
.to_string_lossy()
.to_string();
if let Some(t) = tracked.iter().find(|t| t.relative == relative) {
if t.last_write_sec >= epoch_secs() {
std::thread::sleep(Duration::from_millis(1100));
}
}
let marker = format!("FFF_FUZZ_RAND_{seed:08x}");
// If already tracked, replace old marker line
if let Some(pos) = tracked.iter().position(|t| t.relative == relative) {
let old_marker_line = format!("// {}", tracked[pos].marker);
let content = fs::read_to_string(target).unwrap_or_default();
let new_content = content
.lines()
.map(|l| {
if l == old_marker_line {
format!("// {marker}")
} else {
l.to_string()
}
})
.collect::<Vec<_>>()
.join("\n")
+ "\n";
fs::write(target, new_content).unwrap();
dead_markers.push(tracked[pos].marker.clone());
tracked[pos].marker = marker;
tracked[pos].last_write_sec = epoch_secs();
} else {
// First edit: inject marker at a deterministic line
let original = inject_marker(target, &marker, *seed).unwrap_or_default();
tracked.push(TrackedFile {
relative,
marker,
original_line: original,
is_created: false,
last_write_sec: epoch_secs(),
});
}
ops_since_verify += 1;
}
Op::DeleteTracked => {
if tracked.is_empty() {
continue;
}
let removed = tracked.swap_remove(0);
let abs = workdir.join(&removed.relative);
if abs.exists() {
if removed.is_created {
fs::remove_file(&abs).ok();
} else {
let _ = Command::new("git")
.args(["rm", "-f", &removed.relative])
.current_dir(&workdir)
.output();
}
}
dead_markers.push(removed.marker);
text_files = None; // invalidate cache after deletion
ops_since_verify += 1;
}
Op::RevertTracked => {
// Revert a non-created tracked file using `git checkout`
// (restores original content, marker disappears)
let revertable = tracked.iter().position(|t| !t.is_created);
let Some(idx) = revertable else { continue };
if tracked[idx].last_write_sec >= epoch_secs() {
std::thread::sleep(Duration::from_millis(1100));
}
let _ = Command::new("git")
.args(["checkout", "--", &tracked[idx].relative])
.current_dir(&workdir)
.output();
let reverted = tracked.swap_remove(idx);
dead_markers.push(reverted.marker);
text_files = None; // invalidate cache after revert
ops_since_verify += 1;
}
Op::IgnoredBurst { count, seed } => {
let dir = workdir.join("target/debug/build");
fs::create_dir_all(&dir).unwrap();
for i in 0..*count {
let marker = format!("FFF_IGN_{seed:08x}_{i}");
fs::write(
dir.join(format!("ign_{seed:08x}_{i}.rs")),
format!("// {marker}\nfn {marker}() {{}}\n"),
)
.unwrap();
ignored_markers.push(marker);
}
ops_since_verify += 1;
}
Op::Verify => {
if tracked.is_empty() && dead_markers.is_empty() {
continue;
}
// Poll until the watcher has propagated all pending events:
// all live markers findable, all dead markers gone, no ignored leaks.
let modes = [
(GrepMode::PlainText, "Plain"),
(GrepMode::Regex, "Regex"),
(GrepMode::Fuzzy, "Fuzzy"),
];
let (mode, mode_name) = modes[op_idx % modes.len()];
let deadline = Instant::now() + CONVERGE_TIMEOUT;
let mut last_failure: Option<String> = None;
loop {
std::thread::sleep(WATCHER_SETTLE);
// Write trigger to force a watcher batch
let trigger = workdir.join("fff_fuzz_trigger.rs");
let _ = fs::write(&trigger, format!("// trigger {}\n", op_idx));
std::thread::sleep(WATCHER_SETTLE);
let mut all_ok = true;
// Check live markers (drop lock between each grep)
for tf in &tracked {
let guard = shared_picker.read().unwrap();
let picker = guard.as_ref().unwrap();
let found = grep_finds(picker, &tf.marker, mode);
drop(guard);
if !found {
last_failure = Some(format!(
"{mode_name} grep for {:?} in {:?} not found\n\
is_created={} exists={} on_disk_has_marker={}",
tf.marker,
tf.relative,
tf.is_created,
workdir.join(&tf.relative).exists(),
fs::read_to_string(workdir.join(&tf.relative))
.map(|c| c.contains(&tf.marker))
.unwrap_or(false),
));
all_ok = false;
break;
}
}
// Check dead markers (only sample a few per iteration to
// avoid holding the lock too long with many dead markers)
if all_ok {
let sample_size = dead_markers.len().min(20);
for dead in dead_markers.iter().take(sample_size) {
let guard = shared_picker.read().unwrap();
let picker = guard.as_ref().unwrap();
let found = grep_finds(picker, dead, GrepMode::PlainText);
drop(guard);
if found {
last_failure = Some(format!("dead marker {dead:?} still findable"));
all_ok = false;
break;
}
}
}
// Check ignored markers (sample first 5)
if all_ok {
for ig in ignored_markers.iter().take(5) {
let guard = shared_picker.read().unwrap();
let picker = guard.as_ref().unwrap();
let files = grep_file_list(picker, ig, GrepMode::PlainText);
drop(guard);
if !files.is_empty() {
last_failure =
Some(format!("ignored marker {ig:?} found in {files:?}"));
all_ok = false;
break;
}
}
}
if all_ok {
eprintln!(
" op[{op_idx}] verify OK: {mode_name} mode, {} live, {} dead, {} ignored",
tracked.len(),
dead_markers.len(),
ignored_markers.len(),
);
break;
}
if Instant::now() >= deadline {
panic!(
"op[{op_idx}] verify TIMEOUT after {CONVERGE_TIMEOUT:?}:\n {}\n ops_since_last_verify={}",
last_failure.unwrap_or_default(),
ops_since_verify,
);
}
}
ops_since_verify = 0;
}
}
}
// Final convergence: poll until everything is consistent
let deadline = Instant::now() + CONVERGE_TIMEOUT;
loop {
std::thread::sleep(WATCHER_SETTLE);
let guard = shared_picker.read().unwrap();
let picker = guard.as_ref().unwrap();
let live_ok = tracked
.iter()
.all(|tf| grep_finds(picker, &tf.marker, GrepMode::PlainText));
let dead_ok = dead_markers
.iter()
.all(|d| !grep_finds(picker, d, GrepMode::PlainText));
drop(guard);
if live_ok && dead_ok {
break;
}
assert!(
Instant::now() < deadline,
"final verify TIMEOUT: live_ok={live_ok} dead_ok={dead_ok}"
);
}
// Teardown
shared_picker.wait_for_indexing_complete(Duration::from_secs(30));
if let Ok(mut guard) = shared_picker.write() {
if let Some(mut picker) = guard.take() {
picker.stop_background_monitor();
}
}
eprintln!(
" PASSED: {} ops, {} tracked, {} dead, {} ignored ({:.1}s)",
ops.len(),
tracked.len(),
dead_markers.len(),
ignored_markers.len(),
scenario_start.elapsed().as_secs_f64(),
);
}
// ================
// Proptest harness
// =================
//
fn proptest_config() -> ProptestConfig {
ProptestConfig {
cases: fuzz_cases(),
max_shrink_iters: 0,
fork: false,
failure_persistence: Some(Box::new(FileFailurePersistence::Direct(concat!(
env!("CARGO_MANIFEST_DIR"),
"/tests/fuzz_real_repos.proptest-regressions",
)))),
..ProptestConfig::default()
}
}
#[derive(Debug, Clone)]
enum Op {
/// Create a new file with a unique marker
CreateFile { seed: u32 },
/// Edit a tracked file, replacing the marker line with a new marker
EditTracked { seed: u32 },
/// Edit a random repo file, injecting a marker at a deterministic line
EditRandom { seed: u32 },
/// Delete a tracked file
DeleteTracked,
/// Revert a tracked edit, restoring the original line (marker disappears)
RevertTracked,
/// Burst of writes into ignored directory
IgnoredBurst { count: u8, seed: u32 },
/// Search verification round (no mutation)
Verify,
}
fn op_strategy() -> impl Strategy<Value = Op> {
prop_oneof![
// Create new files — exercises overflow path
12 => any::<u32>().prop_map(|s| Op::CreateFile { seed: s }),
// Edit tracked files — exercises content invalidation
18 => any::<u32>().prop_map(|s| Op::EditTracked { seed: s }),
// Edit random repo files — exercises bigram overlay for base files
18 => any::<u32>().prop_map(|s| Op::EditRandom { seed: s }),
// Delete tracked files — exercises tombstoning
8 => Just(Op::DeleteTracked),
// Revert tracked edits — marker must disappear from search
10 => Just(Op::RevertTracked),
// Burst ignored writes — exercises .gitignore filtering under load
9 => (1u8..20, any::<u32>()).prop_map(|(c, s)| Op::IgnoredBurst { count: c, seed: s }),
// Explicit verification rounds
25 => Just(Op::Verify),
]
}
fn ops_strategy() -> impl Strategy<Value = Vec<Op>> {
let min = fuzz_min_ops();
let max = fuzz_max_ops();
prop::collection::vec(op_strategy(), min..=max)
}
proptest! {
#![proptest_config(proptest_config())]
#[test]
fn fuzz_real_repos_proptest(ops in ops_strategy()) {
run_scenario(&ops);
}
}
crates/fff-grep/src/lines.rs
+1 -1
View File
@@ -103,7 +103,7 @@ pub fn locate(bytes: &[u8], line_term: u8, range: Match) -> Match {
mod tests {
use super::*;
const SHERLOCK: &'static str = "\
const SHERLOCK: &str = "\
For the Doctor Watsons of this world, as opposed to the Sherlock
Holmeses, success in the province of detective work must always
be, to a very large extent, the result of luck. Sherlock Holmes
crates/fff-nvim/Cargo.toml
+7 -3
View File
@@ -78,17 +78,21 @@ criterion = { version = "0.5", features = ["html_reports"] }
rand = { version = "0.8", features = ["small_rng"] }
[[bench]]
name = "indexing_and_search"
name = "fuzzy_search"
path = "benches/fuzzy_search_bench.rs"
harness = false
[[bench]]
name = "grep_bench"
path = "benches/grep_bench.rs"
harness = false
[[bench]]
name = "query_tracker_bench"
name = "query_tracker"
path = "benches/query_tracker_bench.rs"
harness = false
[[bench]]
name = "post_scan_bench"
name = "scan"
path = "benches/scan_bench.rs"
harness = false
crates/fff-nvim/benches/indexing_and_search.rs → crates/fff-nvim/benches/fuzzy_search_bench.rs
-53
View File
@@ -145,58 +145,6 @@ fn setup_once() -> Result<(SharedFilePicker, SharedFrecency), String> {
Ok((shared_picker, shared_frecency))
}
/// Benchmark for indexing the big-repo directory
fn bench_indexing(c: &mut Criterion) {
init_tracing();
let big_repo_path = PathBuf::from("./big-repo");
if !big_repo_path.exists() {
eprintln!(
"./big-repo directory does not exist. Run git clone https://github.com/torvalds/linux.git big-repo"
);
return;
}
let canonical_path = match fff::path_utils::canonicalize(&big_repo_path) {
Ok(p) => p,
Err(e) => {
eprintln!("⚠ Failed to canonicalize path: {}", e);
return;
}
};
let mut group = c.benchmark_group("indexing");
group.sample_size(10);
group.measurement_time(Duration::from_secs(20));
group.bench_function("index_big_repo", |b| {
b.iter(|| {
let sp = SharedFilePicker::default();
let sf = SharedFrecency::default();
let start = std::time::Instant::now();
init_file_picker_internal(black_box(&canonical_path.to_string_lossy()), &sp, &sf)
.expect("Failed to init FilePicker");
match wait_for_scan_completion(&sp, 120) {
Ok(file_count) => {
let elapsed = start.elapsed();
eprintln!(" ✓ Indexed {} files in {:?}", file_count, elapsed);
cleanup_shared_state(&sp);
file_count
}
Err(e) => {
eprintln!(" ✗ Error: {}", e);
cleanup_shared_state(&sp);
0
}
}
});
});
group.finish();
}
/// Benchmark for searching with various query patterns
fn bench_search_queries(c: &mut Criterion) {
let (sp, _sf) = match setup_once() {
@@ -715,7 +663,6 @@ fn bench_grep_search(c: &mut Criterion) {
criterion_group!(
benches,
bench_indexing,
bench_search_queries,
bench_search_thread_scaling,
bench_search_result_limits,
crates/fff-nvim/benches/post_scan_bench.rs → crates/fff-nvim/benches/scan_bench.rs
+27 -2
View File
@@ -3,10 +3,30 @@ use criterion::{Criterion, criterion_group, criterion_main};
use fff::file_picker::{FFFMode, FilePicker};
use fff::{FilePickerOptions, SharedFilePicker, SharedFrecency};
use std::path::PathBuf;
use std::sync::Once;
use std::time::{Duration, Instant};
const WAIT_TIMEOUT: Duration = Duration::from_secs(300);
static TRACING_INIT: Once = Once::new();
fn init_tracing() {
TRACING_INIT.call_once(|| {
use tracing_subscriber::EnvFilter;
use tracing_subscriber::fmt::format::FmtSpan;
let _ = tracing_subscriber::fmt()
.with_env_filter(
EnvFilter::try_from_default_env()
.unwrap_or_else(|_| EnvFilter::new("warn,fff_search=info")),
)
.with_span_events(FmtSpan::CLOSE)
.with_target(true)
.with_writer(std::io::stderr)
.try_init();
});
}
fn resolve_repo() -> Option<PathBuf> {
if let Ok(env_path) = std::env::var("FFF_BENCH_REPO") {
let p = PathBuf::from(env_path);
@@ -14,7 +34,9 @@ fn resolve_repo() -> Option<PathBuf> {
return fff::path_utils::canonicalize(&p).ok();
}
}
let default = PathBuf::from("./big-repo");
// Resolve relative to the workspace root (two levels up from this crate).
let workspace_root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../..");
let default = workspace_root.join("big-repo");
if default.exists() {
return fff::path_utils::canonicalize(&default).ok();
}
@@ -56,7 +78,7 @@ fn wait_for_post_scan(sp: &SharedFilePicker, timeout: Duration) -> bool {
.and_then(|guard| {
guard
.as_ref()
.map(|p| !p.is_scan_active() && p.bigram_index().is_some())
.map(|p| !p.is_scan_active() && !p.is_post_scan_active())
})
.unwrap_or(false);
if ready {
@@ -104,6 +126,7 @@ fn cleanup(sp: SharedFilePicker) {
}
fn bench_full_init(c: &mut Criterion) {
init_tracing();
let Some(repo) = resolve_repo() else {
eprintln!("skip: set FFF_BENCH_REPO or clone a repo to ./big-repo");
return;
@@ -133,6 +156,7 @@ fn bench_full_init(c: &mut Criterion) {
}
fn bench_post_scan_only(c: &mut Criterion) {
init_tracing();
let Some(repo) = resolve_repo() else {
return;
};
@@ -161,6 +185,7 @@ fn bench_post_scan_only(c: &mut Criterion) {
}
fn bench_walk_only(c: &mut Criterion) {
init_tracing();
let Some(repo) = resolve_repo() else {
return;
};
crates/fff-nvim/src/bin/test_watcher.rs
+3 -7
View File
@@ -35,11 +35,9 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
ctrlc::set_handler(move || {
println!("\n🛑 Received interrupt signal, shutting down...");
if let Ok(mut guard) = picker_for_cleanup.write() {
if let Some(mut picker) = guard.take() {
picker.stop_background_monitor();
println!("🧹 FilePicker cleaned up");
}
guard.take();
}
println!("🧹 FilePicker cleaned up");
r.store(false, Ordering::SeqCst);
std::process::exit(0);
})?;
@@ -200,9 +198,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
// Clean up before exit
if let Ok(mut guard) = shared_picker.write() {
if let Some(mut picker) = guard.take() {
picker.stop_background_monitor();
}
guard.take();
}
Ok(())
crates/fff-nvim/src/lib.rs
+17 -41
View File
@@ -81,41 +81,6 @@ pub fn init_file_picker(_: &Lua, base_path: String) -> LuaResult<bool> {
Ok(true)
}
fn reinit_file_picker_internal(path: &Path) -> Result<(), Error> {
// Cancel and stop the old picker's watcher under the write lock.
// `stop_background_monitor` is non-blocking (signals the debouncer
// to exit on its next tick without joining), so it's safe under
// the lock. In-flight watcher handlers finish naturally once we
// release the guard.
{
let mut guard = FILE_PICKER.write()?;
if let Some(ref mut picker) = *guard {
// Signal cancellation BEFORE stopping the watcher so any
// orphaned scan/post-scan threads discard their results
// instead of racing with the new picker.
picker.cancel();
picker.stop_background_monitor();
}
// Don't take() the picker here — leave the old one in place so
// searches still work until new_with_shared_state replaces it.
}
// Create new picker — this atomically replaces the old one via write lock
FilePicker::new_with_shared_state(
FILE_PICKER.clone(),
FRECENCY.clone(),
fff::FilePickerOptions {
base_path: path.to_string_lossy().to_string(),
enable_mmap_cache: true,
enable_content_indexing: true,
mode: FFFMode::Neovim,
..Default::default()
},
)?;
Ok(())
}
pub fn restart_index_in_path(_: &Lua, new_path: String) -> LuaResult<()> {
let path = std::path::PathBuf::from(&new_path);
if !path.exists() {
@@ -148,10 +113,11 @@ pub fn restart_index_in_path(_: &Lua, new_path: String) -> LuaResult<()> {
Ok(g) => g,
Err(_) => return,
};
if let Some(ref picker) = *guard
&& picker.base_path() == canonical_path
{
::tracing::info!(?canonical_path, "restart_index_in_path: same dir, skipping");
tracing::info!(?canonical_path, "restart_index_in_path: same dir, skipping");
return;
}
}
@@ -160,14 +126,25 @@ pub fn restart_index_in_path(_: &Lua, new_path: String) -> LuaResult<()> {
?canonical_path,
"restart_index_in_path: calling reinit_file_picker_internal"
);
if let Err(e) = reinit_file_picker_internal(&canonical_path) {
::tracing::error!(
// this will AUTOMATICALLY drop the old picker within a write lock inside the implementation
// that will stop all the ongoing work and drop all the workeres
if let Err(e) = FilePicker::new_with_shared_state(
FILE_PICKER.clone(),
FRECENCY.clone(),
fff::FilePickerOptions {
base_path: canonical_path.to_string_lossy().to_string(),
enable_mmap_cache: true,
enable_content_indexing: true,
mode: FFFMode::Neovim,
..Default::default()
},
) {
tracing::error!(
?e,
?canonical_path,
"Failed to index directory after changing"
);
} else {
::tracing::info!(?canonical_path, "Successfully reindexed directory");
}
});
@@ -519,7 +496,6 @@ pub fn cleanup_file_picker(_: &Lua, _: ()) -> LuaResult<bool> {
if let Some(picker) = file_picker.take() {
drop(picker);
::tracing::info!("FilePicker cleanup completed");
Ok(true)
} else {
Ok(false)
packages/pi-fff/src/index.ts
+30 -67
View File
@@ -256,9 +256,7 @@ function createFffMentionProvider(
const query = prefix.startsWith('@"') ? prefix.slice(2) : prefix.slice(1);
const items = await getItems(query, options.signal);
return options.signal.aborted || items.length === 0
? null
: { items, prefix };
return options.signal.aborted || items.length === 0 ? null : { items, prefix };
},
applyCompletion(_lines, cursorLine, cursorCol, item, prefix) {
const currentLine = _lines[cursorLine] || "";
@@ -267,11 +265,7 @@ function createFffMentionProvider(
const newLine = before + item.value + after;
const newCursorCol = cursorCol - prefix.length + item.value.length;
return {
lines: [
..._lines.slice(0, cursorLine),
newLine,
..._lines.slice(cursorLine + 1),
],
lines: [..._lines.slice(0, cursorLine), newLine, ..._lines.slice(cursorLine + 1)],
cursorLine,
cursorCol: newCursorCol,
};
@@ -381,22 +375,20 @@ export default function fffExtension(pi: ExtensionAPI) {
const result = f.mixedSearch(query, { pageSize: MENTION_MAX_RESULTS });
if (!result.ok) return [];
return result.value.items
.slice(0, MENTION_MAX_RESULTS)
.map((mixed: MixedItem) => {
if (mixed.type === "directory") {
return {
value: buildAtCompletionValue(mixed.item.relativePath),
label: mixed.item.dirName,
description: mixed.item.relativePath,
};
}
return result.value.items.slice(0, MENTION_MAX_RESULTS).map((mixed: MixedItem) => {
if (mixed.type === "directory") {
return {
value: buildAtCompletionValue(mixed.item.relativePath),
label: mixed.item.fileName,
label: mixed.item.dirName,
description: mixed.item.relativePath,
};
});
}
return {
value: buildAtCompletionValue(mixed.item.relativePath),
label: mixed.item.fileName,
description: mixed.item.relativePath,
};
});
}
// Editor wrapper that injects FFF @-mention autocomplete alongside base provider.
@@ -423,12 +415,8 @@ export default function fffExtension(pi: ExtensionAPI) {
if (mentionResult) return mentionResult;
// Fall back to base provider
return (
this.baseProvider?.getSuggestions(
lines,
cursorLine,
cursorCol,
options,
) ?? null
this.baseProvider?.getSuggestions(lines, cursorLine, cursorCol, options) ??
null
);
},
applyCompletion: (lines, cursorLine, cursorCol, item, prefix) => {
@@ -480,14 +468,12 @@ export default function fffExtension(pi: ExtensionAPI) {
});
pi.registerFlag("fff-frecency-db", {
description:
"Path to the frecency database (overrides FFF_FRECENCY_DB env)",
description: "Path to the frecency database (overrides FFF_FRECENCY_DB env)",
type: "string",
});
pi.registerFlag("fff-history-db", {
description:
"Path to the query history database (overrides FFF_HISTORY_DB env)",
description: "Path to the query history database (overrides FFF_HISTORY_DB env)",
type: "string",
});
@@ -517,20 +503,15 @@ export default function fffExtension(pi: ExtensionAPI) {
context: any,
maxLines = 15,
) => {
const text =
(context.lastComponent as Text | undefined) ?? new Text("", 0, 0);
const output =
result.content?.find((c) => c.type === "text")?.text?.trim() ?? "";
const text = (context.lastComponent as Text | undefined) ?? new Text("", 0, 0);
const output = result.content?.find((c) => c.type === "text")?.text?.trim() ?? "";
if (!output) {
text.setText(theme.fg("muted", "No output"));
return text;
}
const lines = output.split("\n");
const displayLines = lines.slice(
0,
options.expanded ? lines.length : maxLines,
);
const displayLines = lines.slice(0, options.expanded ? lines.length : maxLines);
let content = `\n${displayLines.map((line: string) => theme.fg("toolOutput", line)).join("\n")}`;
if (lines.length > displayLines.length) {
content += theme.fg(
@@ -602,8 +583,7 @@ export default function fffExtension(pi: ExtensionAPI) {
// as a valid regex, otherwise plain literal. The fuzzy fallback below
// only kicks in for plain mode — regex queries are intentional.
const hasRegexSyntax =
params.pattern !==
params.pattern.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
params.pattern !== params.pattern.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
let mode: GrepMode = hasRegexSyntax ? "regex" : "plain";
if (mode === "regex") {
try {
@@ -675,14 +655,10 @@ export default function fffExtension(pi: ExtensionAPI) {
let output = formatGrepOutput(result);
const notices: string[] = [];
if (result.regexFallbackError) {
notices.push(
`Invalid regex: ${result.regexFallbackError}, used literal match`,
);
notices.push(`Invalid regex: ${result.regexFallbackError}, used literal match`);
}
if (result.nextCursor) {
notices.push(
`Continue with cursor="${storeCursor(result.nextCursor)}"`,
);
notices.push(`Continue with cursor="${storeCursor(result.nextCursor)}"`);
}
if (notices.length > 0) output += `\n\n[${notices.join(". ")}]`;
@@ -698,8 +674,7 @@ export default function fffExtension(pi: ExtensionAPI) {
},
renderCall(args, theme, context) {
const text =
(context.lastComponent as Text | undefined) ?? new Text("", 0, 0);
const text = (context.lastComponent as Text | undefined) ?? new Text("", 0, 0);
const pattern = args?.pattern ?? "";
const path = args?.path ?? ".";
let content =
@@ -795,8 +770,7 @@ export default function fffExtension(pi: ExtensionAPI) {
// shown so far there's another page to fetch.
const shownSoFar = pageIndex * effectiveLimit + result.items.length;
const hasMore =
result.items.length >= effectiveLimit &&
result.totalMatched > shownSoFar;
result.items.length >= effectiveLimit && result.totalMatched > shownSoFar;
const notices: string[] = [];
if (formatted.weak && formatted.shownCount > 0)
@@ -830,8 +804,7 @@ export default function fffExtension(pi: ExtensionAPI) {
},
renderCall(args, theme, context) {
const text =
(context.lastComponent as Text | undefined) ?? new Text("", 0, 0);
const text = (context.lastComponent as Text | undefined) ?? new Text("", 0, 0);
const pattern = args?.pattern ?? "";
const path = args?.path ?? ".";
let content =
@@ -864,9 +837,7 @@ export default function fffExtension(pi: ExtensionAPI) {
constraints: Type.Optional(
Type.String({ description: "File filter, e.g. '*.{ts,tsx} !test/'" }),
),
context: Type.Optional(
Type.Number({ description: "Context lines before+after" }),
),
context: Type.Optional(Type.Number({ description: "Context lines before+after" })),
limit: Type.Optional(
Type.Number({
description: `Max matches (default ${DEFAULT_GREP_LIMIT})`,
@@ -932,8 +903,7 @@ export default function fffExtension(pi: ExtensionAPI) {
},
renderCall(args, theme, context) {
const text =
(context.lastComponent as Text | undefined) ?? new Text("", 0, 0);
const text = (context.lastComponent as Text | undefined) ?? new Text("", 0, 0);
const patterns = args?.patterns ?? [];
const constraints = args?.constraints;
let content =
@@ -955,8 +925,7 @@ export default function fffExtension(pi: ExtensionAPI) {
// --- commands ---
pi.registerCommand("fff-mode", {
description:
"Show or set FFF mode: /fff-mode [tools-and-ui | tools-only | override]",
description: "Show or set FFF mode: /fff-mode [tools-and-ui | tools-only | override]",
handler: async (args, ctx) => {
const arg = (args || "").trim();
@@ -965,19 +934,13 @@ export default function fffExtension(pi: ExtensionAPI) {
const mode = getMode();
const flag = pi.getFlag("fff-mode") ?? "unset";
const env = process.env.PI_FFF_MODE ?? "unset";
ctx.ui.notify(
`Current mode: '${mode}'\nFlag: ${flag}, Env: ${env}`,
"info",
);
ctx.ui.notify(`Current mode: '${mode}'\nFlag: ${flag}, Env: ${env}`, "info");
return;
}
// Validate and set mode
if (!VALID_MODES.includes(arg as FffMode)) {
ctx.ui.notify(
`Usage: /fff-mode [${VALID_MODES.join(" | ")}]`,
"warning",
);
ctx.ui.notify(`Usage: /fff-mode [${VALID_MODES.join(" | ")}]`, "warning");
return;
}
packages/pi-fff/test/query.test.ts
+9 -3
View File
@@ -48,7 +48,9 @@ describe("path constraint normalization", () => {
});
test("converts absolute in-workspace file path to repo-relative", () => {
expect(normalizePathConstraint("/tmp/workspace/src/main.rs", cwd)).toBe("src/main.rs");
expect(normalizePathConstraint("/tmp/workspace/src/main.rs", cwd)).toBe(
"src/main.rs",
);
expect(buildQuery("/tmp/workspace/src/main.rs", "needle", undefined, cwd)).toBe(
"src/main.rs needle",
);
@@ -56,10 +58,14 @@ describe("path constraint normalization", () => {
test("converts absolute in-workspace directory (without trailing slash) to repo-relative", () => {
expect(normalizePathConstraint("/tmp/workspace/src", cwd)).toBe("src/");
expect(buildQuery("/tmp/workspace/src", "needle", undefined, cwd)).toBe("src/ needle");
expect(buildQuery("/tmp/workspace/src", "needle", undefined, cwd)).toBe(
"src/ needle",
);
});
test("converts absolute in-workspace glob path to repo-relative glob", () => {
expect(normalizePathConstraint("/tmp/workspace/src/**/*.ts", cwd)).toBe("src/**/*.ts");
expect(normalizePathConstraint("/tmp/workspace/src/**/*.ts", cwd)).toBe(
"src/**/*.ts",
);
});
});
tests/test_cd_during_post_scan.lua
+46
View File
@@ -0,0 +1,46 @@
--- Reproducer for SIGSEGV when :cd is issued during post-scan.
--- Run with: nvim --headless -l tests/test_cd_during_post_scan.lua
---
--- Uses ~/dev/chromium (large repo) so bigram build takes 5-10s,
--- then immediately reinits on the fff source dir.
-- Setup runtimepath so fff.rust can be found
local script_path = debug.getinfo(1, 'S').source:sub(2)
local plugin_dir = vim.fn.fnamemodify(script_path, ':h:h')
vim.opt.runtimepath:prepend(plugin_dir)
local fff_rust = require('fff.rust')
local big_repo = vim.fn.expand('~/dev/chromium')
if vim.fn.isdirectory(big_repo) ~= 1 then
print('SKIP: ~/dev/chromium not found')
os.exit(0)
end
print('Init picker on ' .. big_repo .. ' (500K+ files, slow bigram)...')
local ok = fff_rust.init_file_picker(big_repo)
assert(ok, 'init_file_picker failed')
-- Wait for scan to finish but NOT bigram (bigram is the slow part ~5-10s)
vim.wait(100, function() return false end)
fff_rust.wait_for_initial_scan(120000)
print('Scan done. Immediately reinit on fff source (simulates :cd)...')
fff_rust.restart_index_in_path(plugin_dir)
-- The reinit waits for chromium's post-scan to finish (Drop spin-wait),
-- then installs the new picker. Give it enough time.
local deadline = vim.uv.hrtime() + 30e9 -- 30s
while true do
vim.wait(500, function() return false end)
local ok, result = pcall(fff_rust.fuzzy_search_files, 'lib', 2, nil, 100, 3, 0, 10)
if ok and result and #result.items > 0 then
print('PASS: :cd during post-scan did not crash (' .. #result.items .. ' results found)')
break
end
if vim.uv.hrtime() > deadline then error('TIMEOUT: new picker never became available') end
end
-- Cleanup
pcall(fff_rust.stop_background_monitor)
pcall(fff_rust.cleanup_file_picker)