Compare commits
36 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 7b2864e4b1 | |||
| 2c9cae2e0f | |||
| a166255d8f | |||
| ffd8eb9c08 | |||
| ed6b40a591 | |||
| c4f6c600d6 | |||
| 787a9b1cb7 | |||
| 6398d32c0c | |||
| c6194b848d | |||
| b71b7cf4ac | |||
| cc289f0f93 | |||
| 2107f038dc | |||
| 9441cbc5ae | |||
| dd87489d6b | |||
| d4c416c2fc | |||
| b6f351d729 | |||
| e2cad2f09e | |||
| 4afad820b4 | |||
| 7af3a9dab6 | |||
| 031005e227 | |||
| 3a0ce85c54 | |||
| 695724eb4a | |||
| 086044f5f7 | |||
| 3298ed9f39 | |||
| 1eb913e509 | |||
| e453d007d5 | |||
| 9033efb60f | |||
| ea2ee6b341 | |||
| 84a9cd556f | |||
| 9ec35bd101 | |||
| edda2cbef8 | |||
| 2cf871210b | |||
| fbee146c44 | |||
| 6a239e9875 | |||
| fde8c52a29 | |||
| 84a9b50dd7 |
@@ -18,6 +18,10 @@ env:
|
||||
# Force Node 24 for all JS-based actions to avoid the libuv
|
||||
# process_title assertion crash on Windows (known Node 20 bug).
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
|
||||
# e2e only needs a working binary, so skip fat LTO (same settings as the `ci`
|
||||
# profile releases ship). Overriding release keeps artifacts in target/release.
|
||||
CARGO_PROFILE_RELEASE_LTO: thin
|
||||
CARGO_PROFILE_RELEASE_CODEGEN_UNITS: 16
|
||||
|
||||
jobs:
|
||||
lua-tests:
|
||||
@@ -32,7 +36,6 @@ jobs:
|
||||
- os: ubuntu-latest
|
||||
- os: macos-latest
|
||||
- os: windows-latest
|
||||
target: x86_64-pc-windows-msvc
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: oven-sh/setup-bun@v2
|
||||
@@ -49,20 +52,13 @@ jobs:
|
||||
cache-on-failure: false
|
||||
cache-key: "v2-lua-e2e"
|
||||
rustflags: ""
|
||||
target: ${{ matrix.target || '' }}
|
||||
|
||||
- name: Build Rust binary (Windows)
|
||||
if: matrix.target
|
||||
run: cargo build --release --target ${{ matrix.target }} -p fff-nvim --no-default-features --features zlob
|
||||
|
||||
- name: Copy binary to target/release (Windows)
|
||||
if: matrix.target
|
||||
- name: Build Rust binary
|
||||
shell: bash
|
||||
run: |
|
||||
cp target/${{ matrix.target }}/release/fff_nvim.dll target/release/fff_nvim.dll
|
||||
run: make build-e2e
|
||||
|
||||
- name: Verify Windows DLL has no unexpected dependencies
|
||||
if: matrix.target
|
||||
if: matrix.os == 'windows-latest'
|
||||
shell: pwsh
|
||||
run: |
|
||||
# Find dumpbin via vswhere (always available on GitHub Actions Windows runners)
|
||||
@@ -78,10 +74,6 @@ jobs:
|
||||
exit 1
|
||||
}
|
||||
|
||||
- name: Build Rust binary
|
||||
if: ${{ !matrix.target }}
|
||||
run: make build
|
||||
|
||||
- name: Install Neovim
|
||||
uses: rhysd/action-setup-vim@v1
|
||||
with:
|
||||
@@ -208,6 +200,10 @@ jobs:
|
||||
{ "name": "@ff-labs/fff-bin-linux-x64-musl", "version": "0.0.0" }
|
||||
JSON
|
||||
|
||||
# fff-node builds with bun; the npm `bun` package ships musl binaries
|
||||
- name: Install bun
|
||||
run: npm install -g bun
|
||||
|
||||
- name: Build fff-node
|
||||
working-directory: packages/fff-node
|
||||
run: npm run build
|
||||
|
||||
@@ -15,6 +15,8 @@ on:
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
MACOSX_DEPLOYMENT_TARGET: "13.0"
|
||||
CARGO_PROFILE_RELEASE_LTO: thin
|
||||
CARGO_PROFILE_RELEASE_CODEGEN_UNITS: 16
|
||||
|
||||
jobs:
|
||||
test:
|
||||
@@ -31,7 +33,11 @@ jobs:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Install Rust
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: actions-rust-lang/setup-rust-toolchain@v1.15.4
|
||||
with:
|
||||
cache: true
|
||||
cache-on-failure: true
|
||||
cache-key: "v1-rust-python"
|
||||
|
||||
- name: Setup uv
|
||||
uses: astral-sh/setup-uv@v5
|
||||
@@ -43,6 +49,8 @@ jobs:
|
||||
working-directory: packages/fff-python
|
||||
shell: bash
|
||||
run: |
|
||||
uv sync --all-extras
|
||||
# --no-install-project: uv would otherwise build the wheel via
|
||||
# maturin's PEP 517 backend, then `maturin develop` builds it again.
|
||||
uv sync --all-extras --no-install-project
|
||||
uv run maturin develop --release
|
||||
uv run pytest -v
|
||||
|
||||
@@ -612,7 +612,7 @@ jobs:
|
||||
|
||||
Update mcp via:
|
||||
```sh
|
||||
curl -fsSL https://raw.githubusercontent.com/dmtrKovalenko/fff.nvim/main/install-mcp.sh | bash
|
||||
curl -fsSL https://raw.githubusercontent.com/dmtrKovalenko/fff/main/install-mcp.sh | bash
|
||||
```
|
||||
|
||||
- name: Bump Homebrew formula (uses local checksums)
|
||||
@@ -710,6 +710,8 @@ jobs:
|
||||
node-version: "25"
|
||||
registry-url: "https://registry.npmjs.org"
|
||||
|
||||
- uses: oven-sh/setup-bun@v2
|
||||
|
||||
- name: Determine version
|
||||
id: version
|
||||
run: lua scripts/determine-version.lua
|
||||
@@ -747,6 +749,8 @@ jobs:
|
||||
make set-npm-version PKG=packages/fff-bun VERSION="$VERSION"
|
||||
|
||||
cd packages/fff-bun
|
||||
bun install --frozen-lockfile
|
||||
bun run build
|
||||
npm publish --tag "$TAG" --access public --provenance
|
||||
|
||||
- name: Publish Node.js package
|
||||
|
||||
@@ -47,7 +47,8 @@ jobs:
|
||||
components: rustfmt, clippy
|
||||
|
||||
- name: Run tests
|
||||
run: cargo test --no-default-features --features zlob --workspace --exclude fff-nvim
|
||||
# fff-python requires full python o3 machinery which is very slow
|
||||
run: cargo test --no-default-features --features zlob --workspace --exclude fff-nvim --exclude fff-python
|
||||
|
||||
stress-test:
|
||||
name: Fuzz Tests
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
-- luacheck configuration for fff.nvim
|
||||
-- luacheck configuration for fff
|
||||
-- https://luacheck.readthedocs.io/en/stable/config.html
|
||||
|
||||
-- Neovim globals
|
||||
|
||||
Generated
+15
-14
@@ -633,7 +633,7 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
|
||||
|
||||
[[package]]
|
||||
name = "fff-c"
|
||||
version = "0.10.1"
|
||||
version = "0.10.4"
|
||||
dependencies = [
|
||||
"fff-query-parser",
|
||||
"fff-search",
|
||||
@@ -643,7 +643,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "fff-grep"
|
||||
version = "0.10.1"
|
||||
version = "0.10.4"
|
||||
dependencies = [
|
||||
"bstr",
|
||||
"memchr",
|
||||
@@ -651,7 +651,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "fff-mcp"
|
||||
version = "0.10.1"
|
||||
version = "0.10.4"
|
||||
dependencies = [
|
||||
"clap",
|
||||
"fff-query-parser",
|
||||
@@ -662,8 +662,10 @@ dependencies = [
|
||||
"schemars",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tempfile",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -682,7 +684,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "fff-nvim"
|
||||
version = "0.10.1"
|
||||
version = "0.10.4"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"chrono",
|
||||
@@ -703,7 +705,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "fff-python"
|
||||
version = "0.10.1"
|
||||
version = "0.10.4"
|
||||
dependencies = [
|
||||
"fff-query-parser",
|
||||
"fff-search",
|
||||
@@ -713,7 +715,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "fff-query-parser"
|
||||
version = "0.10.1"
|
||||
version = "0.10.4"
|
||||
dependencies = [
|
||||
"criterion",
|
||||
"zlob",
|
||||
@@ -721,7 +723,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "fff-search"
|
||||
version = "0.10.1"
|
||||
version = "0.10.4"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"aho-corasick",
|
||||
@@ -925,15 +927,14 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "git2"
|
||||
version = "0.20.4"
|
||||
version = "0.21.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7b88256088d75a56f8ecfa070513a775dd9107f6530ef14919dac831af9cfe2b"
|
||||
checksum = "ddddbf932745a6be37109b6112d3ee09696106f848449069d3a57bba937ab82e"
|
||||
dependencies = [
|
||||
"bitflags 2.11.0",
|
||||
"libc",
|
||||
"libgit2-sys",
|
||||
"log",
|
||||
"url",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1333,9 +1334,9 @@ checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d"
|
||||
|
||||
[[package]]
|
||||
name = "libgit2-sys"
|
||||
version = "0.18.3+1.9.2"
|
||||
version = "0.18.7+1.9.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c9b3acc4b91781bb0b3386669d325163746af5f6e4f73e6d2d630e09a35f3487"
|
||||
checksum = "23c7391e4b9f4ffab1a624223cc1d7385ff9a678f490768add717de7ea2f4d89"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"libc",
|
||||
@@ -3236,9 +3237,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "zlob"
|
||||
version = "1.6.1"
|
||||
version = "1.6.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e41cb327ac1b7e7e0d4514658500cb5734cd655edbe4e5ffeda69955da9028ee"
|
||||
checksum = "f0f010d9b10495cc0037cb47cedc143c39c7fa2d59f752dca19036fd0234a210"
|
||||
dependencies = [
|
||||
"bindgen",
|
||||
"bitflags 2.11.0",
|
||||
|
||||
+4
-4
@@ -15,8 +15,8 @@ resolver = "2"
|
||||
module_inception = "allow"
|
||||
|
||||
[workspace.dependencies]
|
||||
fff-grep = { version = "0.10.1", path = "crates/fff-grep" }
|
||||
fff-query-parser = { version = "0.10.1", path = "crates/fff-query-parser", default-features = false }
|
||||
fff-grep = { version = "0.10.4", path = "crates/fff-grep" }
|
||||
fff-query-parser = { version = "0.10.4", path = "crates/fff-query-parser", default-features = false }
|
||||
|
||||
# Shared dependencies
|
||||
ahash = "0.8"
|
||||
@@ -26,7 +26,7 @@ ctrlc = "3.4.2"
|
||||
dirs = "5.0"
|
||||
dunce = "1.0"
|
||||
# git2 - base config without TLS (each crate adds platform-specific TLS)
|
||||
git2 = { version = "0.20.2", default-features = false, features = [
|
||||
git2 = { version = "0.21.0", default-features = false, features = [
|
||||
"vendored-libgit2",
|
||||
] }
|
||||
glidesort = "0.1"
|
||||
@@ -36,7 +36,7 @@ ignore = "0.4.22"
|
||||
memmap2 = "0.9"
|
||||
mimalloc = "0.1.47"
|
||||
signal-hook-registry = "1.4"
|
||||
zlob = { version = "=1.6.1" }
|
||||
zlob = { version = "=1.6.3" }
|
||||
|
||||
mlua = { version = "0.11.1", features = ["module", "luajit"] }
|
||||
neo_frizbee = { version = "0.11.0", features = ["match_end_col"] }
|
||||
|
||||
+8
-8
@@ -2,34 +2,34 @@
|
||||
# Maintained in-repo; auto-bumped by .github/workflows/release.yaml on stable releases.
|
||||
class FffMcp < Formula
|
||||
desc "Fast file search toolkit for AI agents (MCP server)"
|
||||
homepage "https://github.com/dmtrKovalenko/fff.nvim"
|
||||
homepage "https://github.com/dmtrKovalenko/fff"
|
||||
license "MIT"
|
||||
version "0.10.1"
|
||||
version "0.10.4"
|
||||
|
||||
LIVECHECK_REPO = "dmtrKovalenko/fff.nvim".freeze
|
||||
RELEASE_BASE = "https://github.com/dmtrKovalenko/fff.nvim/releases/download".freeze
|
||||
LIVECHECK_REPO = "dmtrKovalenko/fff".freeze
|
||||
RELEASE_BASE = "https://github.com/dmtrKovalenko/fff/releases/download".freeze
|
||||
|
||||
on_macos do
|
||||
on_arm do
|
||||
url "#{RELEASE_BASE}/v#{version}/fff-mcp-aarch64-apple-darwin"
|
||||
sha256 "eff6660a9c48e3e1972d557c1003e057e5ff99d6039f5f810671f2123093fdfc"
|
||||
sha256 "f55ce2945cd30fd252a4b1e7d8674ff7ce41b104cc009a8d3d52d08bfccca2c0"
|
||||
end
|
||||
|
||||
on_intel do
|
||||
url "#{RELEASE_BASE}/v#{version}/fff-mcp-x86_64-apple-darwin"
|
||||
sha256 "d3c8d70d47ab80afa2287e1b4546b23f494bb4604b80e5a706a0af714ee25674"
|
||||
sha256 "821aedd5873292841f32d7584eb68fea085b248e5c970ff5d4f6dca790e9b18c"
|
||||
end
|
||||
end
|
||||
|
||||
on_linux do
|
||||
on_arm do
|
||||
url "#{RELEASE_BASE}/v#{version}/fff-mcp-aarch64-unknown-linux-gnu"
|
||||
sha256 "ed628d57200e48298fcbeffc0f6d85fb572de02bb900bcc6fc7cc05e33ba2777"
|
||||
sha256 "3f266c0eded51a106badf1d47448a7f963762ccf3a12c5a4f06ee2b7adaf82d1"
|
||||
end
|
||||
|
||||
on_intel do
|
||||
url "#{RELEASE_BASE}/v#{version}/fff-mcp-x86_64-unknown-linux-gnu"
|
||||
sha256 "3541dad4663fc9e5b8b97ce8bd20084ddae08a3c6cb828fcfbcbb6ca8c411513"
|
||||
sha256 "b5dd3182028c8a7bc34ee964cf2d30694f8121d2837a8516cf4a99bcfa822127"
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@ SHELL := bash
|
||||
# string rather than the literal `-o` / `pipefail` tokens.
|
||||
.SHELLFLAGS := -o pipefail -euc
|
||||
|
||||
.PHONY: build build-c-lib install uninstall test test-rust test-c-smoke test-c-api test-lua test-lua-snap test-version test-bun test-node prepare-bun prepare-bun-packaged prepare-node set-npm-version header test-stress test-stress-seeded test-stress-random test-stress-regressions test-stress-repos test-node-stress sync-js-api sync-js-api-check bump-homebrew-formula bump-install-mcp-sh test-bun-compile
|
||||
.PHONY: build build-c-lib install uninstall test test-rust test-rescan test-rescan-known-defects rescan-probe test-c-smoke test-c-api test-lua test-lua-snap test-version test-bun test-node prepare-bun prepare-bun-packaged prepare-node set-npm-version header test-stress test-stress-seeded test-stress-random test-stress-regressions test-stress-repos test-node-stress sync-js-api sync-js-api-check bump-homebrew-formula bump-install-mcp-sh test-bun-compile
|
||||
|
||||
all: format test lint
|
||||
|
||||
@@ -45,6 +45,11 @@ sync-js-api-check:
|
||||
build:
|
||||
cargo build --release --no-default-features --features zlob
|
||||
|
||||
# Only the crates the e2e suites load (nvim lua tests + C/bun/node FFI tests),
|
||||
# skipping fff-python (pyo3) and fff-mcp (tokio/rmcp) which e2e never touches.
|
||||
build-e2e:
|
||||
cargo build --release -p fff-nvim -p fff-c --no-default-features --features zlob
|
||||
|
||||
build-c-lib:
|
||||
cargo build --release -p fff-c --no-default-features --features zlob
|
||||
|
||||
@@ -90,7 +95,26 @@ test-setup:
|
||||
fi
|
||||
|
||||
test-rust:
|
||||
cargo test --workspace --no-default-features --features zlob --exclude fff-nvim
|
||||
cargo test --workspace --no-default-features --features zlob --exclude fff-nvim --exclude fff-python
|
||||
|
||||
# Watcher rescan harness: asserts that editing, build output, git activity and
|
||||
# preview reads all stay on the incremental path instead of re-walking the tree.
|
||||
test-rescan:
|
||||
cargo test -p fff-search --no-default-features --features zlob \
|
||||
--lib --test rescan_regression -- rescan
|
||||
|
||||
# Live probe for watcher rescan requests and their causes.
|
||||
# Usage: make rescan-probe DIR=~/some/repo [SECONDS=120]
|
||||
rescan-probe:
|
||||
cargo run --release -p fff-nvim --bin rescan_probe \
|
||||
--no-default-features --features zlob,rescan-stats -- \
|
||||
$(or $(DIR),.) $(if $(SECONDS),--seconds $(SECONDS),)
|
||||
|
||||
# The same harness, restricted to cases that currently fail on purpose. Each
|
||||
# `#[ignore]` reason names the code that causes the unnecessary rescan.
|
||||
test-rescan-known-defects:
|
||||
cargo test --no-fail-fast -p fff-search --no-default-features --features zlob \
|
||||
--lib --test rescan_regression -- --ignored --nocapture
|
||||
|
||||
CC ?= cc
|
||||
CFLAGS ?= -O0 -g -Wall -Wextra -std=c99
|
||||
@@ -259,6 +283,9 @@ set-npm-version:
|
||||
pkg.optionalDependencies[dep] = '$(VERSION)'; \
|
||||
} \
|
||||
} \
|
||||
for (const dep of ['@ff-labs/fff-bun', '@ff-labs/fff-node']) { \
|
||||
if (pkg.dependencies?.[dep]) pkg.dependencies[dep] = '$(VERSION)'; \
|
||||
} \
|
||||
fs.writeFileSync('$(PKG)/package.json', JSON.stringify(pkg, null, 2) + '\n'); \
|
||||
"
|
||||
@echo "Set $(PKG) to $(VERSION)"
|
||||
@@ -283,7 +310,7 @@ lint: lint-rust lint-lua lint-ts
|
||||
|
||||
check: format lint
|
||||
|
||||
FFF_RELEASE_REPO ?= dmtrKovalenko/fff.nvim
|
||||
FFF_RELEASE_REPO ?= dmtrKovalenko/fff
|
||||
FFF_FORMULA_PATH ?= Formula/fff-mcp.rb
|
||||
FFF_INSTALL_SCRIPT_PATH ?= install-mcp.sh
|
||||
|
||||
|
||||
@@ -4,11 +4,14 @@
|
||||
<i>A file search toolkit for humans and AI agents. Really fast.</i>
|
||||
</p>
|
||||
|
||||
Typo-resistant path and content search, frecency-ranked file access, a background watcher, and a lightweight in-memory content index. Way faster than CLIs like ripgrep and fzf in any long-running process that searches more than once.
|
||||
Typo-resistant path and content search, frequency-ranked file access, a background watcher, and a lightweight in-memory content index. Way faster than CLIs like ripgrep and fzf in any long-running process that searches more than once.
|
||||
|
||||
Powers file search in [opencode](http://github.com/anomalyco/opencode/), [nushell](https://github.com/nushell/nushell), and many more amazing projects!
|
||||
|
||||
Originally started as [Neovim plugin](#neovim-plugin) people loved, but it turned out that plenty of AI harnesses and code editors need the same thing: accurate, fast file search as a library. That is what fff is.
|
||||
<p>
|
||||
<a href="https://trendshift.io/repositories/26711?utm_source=repository-badge&utm_medium=badge&utm_campaign=badge-repository-26711" target="_blank" rel="noopener noreferrer"><img src="https://trendshift.io/api/badge/repositories/26711" alt="dmtrKovalenko%2Ffff | Trendshift" width="250" height="55"/></a>
|
||||
</p>
|
||||
|
||||
---
|
||||
|
||||
@@ -34,7 +37,7 @@ curl -L https://dmtrkovalenko.dev/install-fff-mcp.sh | bash
|
||||
Windows (PowerShell):
|
||||
|
||||
```powershell
|
||||
irm https://raw.githubusercontent.com/dmtrKovalenko/fff.nvim/main/install-mcp.ps1 | iex
|
||||
irm https://raw.githubusercontent.com/dmtrKovalenko/fff/main/install-mcp.ps1 | iex
|
||||
```
|
||||
|
||||
The scripts live at [`install-mcp.sh`](./install-mcp.sh) and [`install-mcp.ps1`](./install-mcp.ps1) if you want to read them first. They print the exact wiring instructions for your client.
|
||||
@@ -46,7 +49,7 @@ brew install dmtrKovalenko/fff/fff-mcp
|
||||
brew upgrade fff-mcp # after new stable releases
|
||||
```
|
||||
|
||||
Formula lives in [`Formula/fff-mcp.rb`](./Formula/fff-mcp.rb) in this repo and is **auto-bumped on every stable release** (see `bump-homebrew-formula` in [`.github/workflows/release.yaml`](./.github/workflows/release.yaml)). Installs the prebuilt `fff-mcp` binary from [GitHub releases](https://github.com/dmtrKovalenko/fff.nvim/releases).
|
||||
Formula lives in [`Formula/fff-mcp.rb`](./Formula/fff-mcp.rb) in this repo and is **auto-bumped on every stable release** (see `bump-homebrew-formula` in [`.github/workflows/release.yaml`](./.github/workflows/release.yaml)). Installs the prebuilt `fff-mcp` binary from [GitHub releases](https://github.com/dmtrKovalenko/fff/releases).
|
||||
|
||||
### Codex setup
|
||||
|
||||
@@ -150,8 +153,9 @@ https://github.com/user-attachments/assets/5d0e1ce9-642c-4c44-aa88-01b05bb86abb
|
||||
#### lazy.nvim
|
||||
|
||||
```lua
|
||||
-- Package name changed from `fff.nvim` to `fff`. If you installed fff.nvim before, clean with `:Lazy clean`
|
||||
{
|
||||
'dmtrKovalenko/fff.nvim',
|
||||
'dmtrKovalenko/fff',
|
||||
build = function()
|
||||
-- downloads a prebuilt binary or falls back to cargo build
|
||||
require("fff.download").download_or_build_binary()
|
||||
@@ -184,13 +188,14 @@ https://github.com/user-attachments/assets/5d0e1ce9-642c-4c44-aa88-01b05bb86abb
|
||||
#### vim.pack
|
||||
|
||||
```lua
|
||||
vim.pack.add({ 'https://github.com/dmtrKovalenko/fff.nvim' })
|
||||
-- Package name changed from `fff.nvim` to `fff`. If you installed fff.nvim before, clean with `:packdel fff.nvim`
|
||||
vim.pack.add({ 'https://github.com/dmtrKovalenko/fff' })
|
||||
|
||||
vim.api.nvim_create_autocmd('PackChanged', {
|
||||
callback = function(ev)
|
||||
local name, kind = ev.data.spec.name, ev.data.kind
|
||||
if name == 'fff.nvim' and (kind == 'install' or kind == 'update') then
|
||||
if not ev.data.active then vim.cmd.packadd('fff.nvim') end
|
||||
if name == 'fff' and (kind == 'install' or kind == 'update') then
|
||||
if not ev.data.active then vim.cmd.packadd('fff') end
|
||||
require('fff.download').download_or_build_binary()
|
||||
end
|
||||
end,
|
||||
@@ -309,6 +314,11 @@ require('fff').setup({
|
||||
-- Border style for the picker windows. Leave unset (nil) to follow the
|
||||
-- global `vim.o.winborder`; set it to override fff's borders independently.
|
||||
border = nil, -- 'single' | 'double' | 'rounded' | 'solid' | 'shadow' | 'none'
|
||||
-- border = {
|
||||
-- { ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ' },
|
||||
-- { ' ', ' ', ' ', ' ', ' ' },
|
||||
-- },
|
||||
|
||||
flex = { size = 130, wrap = 'top' },
|
||||
min_list_height = 10, -- do not display anything except the list below this threshold
|
||||
show_scrollbar = true,
|
||||
@@ -606,7 +616,7 @@ cargo build --release -p fff-c --features zlob
|
||||
|
||||
The output is a `cdylib` (`libfff_c.so` / `libfff_c.dylib` / `fff_c.dll`). The header lives at [`crates/fff-c/include/fff.h`](./crates/fff-c/include/fff.h).
|
||||
|
||||
Prebuilt binaries for every version, including every commit on main, are on the [releases page](https://github.com/dmtrKovalenko/fff.nvim/releases). The same binaries also ship inside the `@ff-labs/fff-bin-*` npm packages.
|
||||
Prebuilt binaries for every version, including every commit on main, are on the [releases page](https://github.com/dmtrKovalenko/fff/releases). The same binaries also ship inside the `@ff-labs/fff-bin-*` npm packages.
|
||||
|
||||
### Install
|
||||
|
||||
@@ -783,7 +793,7 @@ FFF is a file search library, not a CLI. Ripgrep and fzf are great tools, but th
|
||||
|
||||
FFF keeps the index and the file cache resident in one long-lived process and exposes the same Rust core through four thin layers: a native crate (`fff-search`), a C library (`libfff_c`), a Node/Bun SDK (`@ff-labs/fff-node`), and an MCP server. You call `FileFinder.create()` once, then every subsequent search hits warm memory. On a 500k-file Chromium checkout, that is the difference between 3-9 **SECONDS** per ripgrep spawn and sub-10 ms per FFF query.
|
||||
|
||||
Algorithm for fuzzy matching is much more comprehensive than fzf's algorithm it is **typo-resistant** and we provide a query language with additional constraint parsing for prefiltering e.g. "\*.rs !test/ shcema" is a perfectly valid query for fff, but fzf wouldn't find anything even for a single typo in "shcema".
|
||||
Algorithm for fuzzy matching is much more comprehensive than fzf's algorithm. It is **typo-resistant** and we provide a query language with additional constraint parsing for prefiltering e.g. "\*.rs !test/ shcema" is a perfectly valid query for fff, but fzf wouldn't find anything even for a single typo in "shcema".
|
||||
|
||||
### Why a programmatic API matters
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "fff-c"
|
||||
version = "0.10.1"
|
||||
version = "0.10.4"
|
||||
edition = "2024"
|
||||
description = "Raw C api of FFF file finder"
|
||||
license = "MIT"
|
||||
@@ -19,6 +19,6 @@ zlob = ["fff/zlob", "fff-query-parser/zlob"]
|
||||
[dependencies]
|
||||
git2.workspace = true
|
||||
|
||||
fff = { package = "fff-search", path = "../fff-core", version = "0.10.1", default-features = false }
|
||||
fff-query-parser = { path = "../fff-query-parser", version = "0.10.1", default-features = false }
|
||||
fff = { package = "fff-search", path = "../fff-core", version = "0.10.4", default-features = false }
|
||||
fff-query-parser = { path = "../fff-query-parser", version = "0.10.4", default-features = false }
|
||||
serde_json = "1.0"
|
||||
|
||||
+3
-10
@@ -729,19 +729,12 @@ pub unsafe extern "C" fn fff_multi_grep(
|
||||
}
|
||||
};
|
||||
|
||||
let is_ai = picker.mode().is_ai();
|
||||
|
||||
// Parse constraints from the optional string (e.g. "*.rs /src/")
|
||||
let parsed_constraints = constraints_str.map(|c| {
|
||||
if is_ai {
|
||||
fff::QueryParser::new(fff_query_parser::AiGrepConfig).parse(c)
|
||||
} else {
|
||||
fff::grep::parse_grep_query(c)
|
||||
}
|
||||
});
|
||||
let parsed_constraints = constraints_str
|
||||
.map(|c| fff::QueryParser::new(fff_query_parser::AiGrepConfig).parse_constraints(c));
|
||||
|
||||
let constraint_refs: &[fff::Constraint<'_>] = match &parsed_constraints {
|
||||
Some(q) => &q.constraints,
|
||||
Some(constraints) => constraints,
|
||||
None => &[],
|
||||
};
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "fff-search"
|
||||
version = "0.10.1"
|
||||
version = "0.10.4"
|
||||
edition = "2024"
|
||||
license = "MIT"
|
||||
authors = ["Dmitriy Kovalenko <dmtr.kovalenko@outlook.com>"]
|
||||
@@ -41,6 +41,9 @@ harness = false
|
||||
default = ["ripgrep"]
|
||||
# Enable C FFI exports
|
||||
ffi = []
|
||||
# Count full rescans and their causes. Always on in debug builds; enable this
|
||||
# to keep the accounting in a release build (used by the rescan_probe binary).
|
||||
rescan-stats = []
|
||||
# Enables POC definition classification for grep result matched lines
|
||||
definitions = []
|
||||
# Pure-Rust filesystem walker + glob matcher (ignore + globset crates).
|
||||
|
||||
@@ -1,11 +1,5 @@
|
||||
//! Compare three glob-matching strategies for `match_glob_pattern` in constraints.rs:
|
||||
//!
|
||||
//! 1. Current: `zlob_match_paths` -> collect `as_ptr()` into AHashSet, filter paths
|
||||
//! by pointer to recover indices.
|
||||
//! 2. Free fn: `zlob_match_paths_indices` (added in zlob 1.4) — indices direct from C.
|
||||
//! 3. Compiled: `ZlobPattern::compile` + `match_indices` — same indices path, but with
|
||||
//! a precompiled pattern (reusable). For one-shot it should match (2); the win
|
||||
//! appears if the pattern is reused (chunked / repeated calls).
|
||||
//! Compare three glob-matching strategies for `match_glob_pattern` in constraints.rs
|
||||
//! need to make sure that my zlob changes are not affecting fff's internal globbing
|
||||
use ahash::AHashSet;
|
||||
use criterion::{BenchmarkId, Criterion, black_box, criterion_group, criterion_main};
|
||||
use zlob::{ZlobFlags, ZlobPattern, zlob_match_paths, zlob_match_paths_indices};
|
||||
|
||||
@@ -3,6 +3,15 @@ fn main() {
|
||||
// used by tests/fuzz_git_watcher_stress.rs
|
||||
println!("cargo::rustc-check-cfg=cfg(stress)");
|
||||
|
||||
// Full-rescan accounting. Debug builds get it for free; a release build has
|
||||
// to opt in with `--features rescan-stats` (what the rescan_probe needs).
|
||||
println!("cargo::rustc-check-cfg=cfg(rescan_stats)");
|
||||
if std::env::var("DEBUG").is_ok_and(|debug| debug != "false")
|
||||
|| std::env::var("CARGO_FEATURE_RESCAN_STATS").is_ok()
|
||||
{
|
||||
println!("cargo::rustc-cfg=rescan_stats");
|
||||
}
|
||||
|
||||
// When the `zlob` feature is enabled (Zig-compiled C library):
|
||||
// On Windows MSVC, explicitly link the C runtime libraries.
|
||||
// Zig-compiled static libraries don't emit /DEFAULTLIB directives for the
|
||||
|
||||
@@ -12,16 +12,25 @@ pub const MAX_INDEXABLE_FILE_SIZE: usize = 2 * 1024 * 1024;
|
||||
pub const MAX_CACHED_CONTENT_BYTES: u64 = 512 * 1024 * 1024;
|
||||
|
||||
/// Files below one page waste the remainder when mmapped, so the cache skips
|
||||
/// them and falls back to chunked reads. Unused on Windows (no content cache).
|
||||
/// them and falls back to chunked reads. Unused on Windows (no content cache)
|
||||
#[cfg(all(not(target_os = "windows"), target_arch = "aarch64"))]
|
||||
pub const MMAP_THRESHOLD: u64 = 16 * 1024;
|
||||
#[cfg(all(not(target_os = "windows"), not(target_arch = "aarch64")))]
|
||||
pub const MMAP_THRESHOLD: u64 = 4 * 1024;
|
||||
|
||||
/// Capacity reserved for files the watcher discovers after the initial scan;
|
||||
/// exceeding it forces a full rescan.
|
||||
/// Watcher overflow capacity reserved after the initial scan
|
||||
pub const MAX_OVERFLOW_FILES: usize = 1024;
|
||||
|
||||
/// Minimum delay between watcher-initiated rescans.
|
||||
pub const RESCAN_MIN_INTERVAL: std::time::Duration = std::time::Duration::from_secs(30);
|
||||
|
||||
/// Rescan delay for large indexes.
|
||||
pub const RESCAN_MIN_INTERVAL_LARGE_INDEX: std::time::Duration =
|
||||
std::time::Duration::from_secs(5 * 60);
|
||||
|
||||
/// Live-file count at which [`RESCAN_MIN_INTERVAL_LARGE_INDEX`] takes over.
|
||||
pub const LARGE_INDEX_FILE_COUNT: usize = 1_000_000;
|
||||
|
||||
/// Fresh-mmap threshold: files at or above this size get mmapped directly on
|
||||
/// cache miss instead of chunked reads into Vec. Empirically tuned per-platform.
|
||||
/// Only referenced on Unix; Windows uses the `std::fs::read` fallback so this
|
||||
|
||||
@@ -0,0 +1,250 @@
|
||||
use heed::{Env, EnvOpenOptions};
|
||||
use std::collections::HashMap;
|
||||
use std::fs;
|
||||
use std::ops::Deref;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{Arc, LazyLock, Mutex, MutexGuard, PoisonError, Weak};
|
||||
use std::thread;
|
||||
use std::time::Duration;
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
use crate::lmdb::DbHealth;
|
||||
|
||||
pub(crate) struct EnvSpec {
|
||||
pub label: &'static str,
|
||||
pub map_size: usize,
|
||||
pub max_dbs: u32,
|
||||
pub size_cap_bytes: u64,
|
||||
}
|
||||
|
||||
pub(crate) struct PooledEnv {
|
||||
env: Env,
|
||||
key: PathBuf,
|
||||
/// lmdb's env spec label
|
||||
label: &'static str,
|
||||
map_size: usize,
|
||||
max_dbs: u32,
|
||||
health: DbHealth,
|
||||
gc_started: AtomicBool,
|
||||
dbi_lock: Mutex<()>,
|
||||
}
|
||||
|
||||
impl Drop for PooledEnv {
|
||||
fn drop(&mut self) {
|
||||
let mut pool = POOL.lock().unwrap_or_else(PoisonError::into_inner);
|
||||
// Only remove a dead entry: begin_exclusive_destroy may have removed ours.
|
||||
if pool.get(&self.key).is_some_and(|w| w.strong_count() == 0) {
|
||||
pool.remove(&self.key);
|
||||
}
|
||||
// heed closes the env right after this body; a concurrent reopen of the
|
||||
// same path rides out that gap via env_closing_event in get_or_open.
|
||||
}
|
||||
}
|
||||
|
||||
// Cloneable handle to a process-shared LMDB env, derefs to `heed::Env`.
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct SharedEnv(Arc<PooledEnv>);
|
||||
|
||||
impl Deref for SharedEnv {
|
||||
type Target = Env;
|
||||
fn deref(&self) -> &Env {
|
||||
&self.0.env
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for SharedEnv {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_tuple("SharedEnv").field(&self.0.env).finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl SharedEnv {
|
||||
pub(crate) fn get_or_open(db_path: &Path, spec: &EnvSpec) -> Result<Self> {
|
||||
fs::create_dir_all(db_path).map_err(Error::CreateDir)?;
|
||||
let path = fs::canonicalize(db_path).map_err(|e| Error::EnvOpen {
|
||||
db: spec.label,
|
||||
source: heed::Error::Io(e),
|
||||
})?;
|
||||
|
||||
let mut close_waits = 0u32;
|
||||
let mut transient_retries = 0u32;
|
||||
|
||||
loop {
|
||||
let mut open_failed = false;
|
||||
|
||||
{
|
||||
let mut pool = POOL.lock().unwrap_or_else(PoisonError::into_inner);
|
||||
if let Some(existing) = pool.get(&path).and_then(Weak::upgrade) {
|
||||
drop(pool);
|
||||
if existing.label != spec.label
|
||||
|| existing.map_size != spec.map_size
|
||||
|| existing.max_dbs != spec.max_dbs
|
||||
{
|
||||
return Err(Error::EnvSpecMismatch {
|
||||
path,
|
||||
open_as: existing.label,
|
||||
requested_as: spec.label,
|
||||
});
|
||||
}
|
||||
return Ok(Self(existing));
|
||||
}
|
||||
|
||||
erase_if_oversized(&path, spec);
|
||||
let result = unsafe {
|
||||
let mut opts = EnvOpenOptions::new();
|
||||
opts.map_size(spec.map_size);
|
||||
if spec.max_dbs > 0 {
|
||||
opts.max_dbs(spec.max_dbs);
|
||||
}
|
||||
opts.open(&path)
|
||||
};
|
||||
|
||||
match result {
|
||||
Ok(env) => {
|
||||
let entry = Arc::new(PooledEnv {
|
||||
env,
|
||||
key: path.clone(),
|
||||
label: spec.label,
|
||||
map_size: spec.map_size,
|
||||
max_dbs: spec.max_dbs,
|
||||
health: DbHealth::new(),
|
||||
gc_started: AtomicBool::new(false),
|
||||
dbi_lock: Mutex::new(()),
|
||||
});
|
||||
pool.insert(path.clone(), Arc::downgrade(&entry));
|
||||
drop(pool);
|
||||
let shared = Self(entry);
|
||||
|
||||
match shared.clear_stale_readers() {
|
||||
Ok(cleared_count) if cleared_count > 0 => {
|
||||
tracing::info!(
|
||||
cleared_count,
|
||||
db = spec.label,
|
||||
"reclaimed stale LMDB reader slots at open"
|
||||
);
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(e) => {
|
||||
tracing::debug!("clear_stale_readers at open failed: {e}")
|
||||
}
|
||||
}
|
||||
|
||||
return Ok(shared);
|
||||
}
|
||||
Err(heed::Error::EnvAlreadyOpened) => open_failed = true,
|
||||
// special handling cause we know this happens randomly
|
||||
Err(e)
|
||||
if is_transient_env_open_error(&e)
|
||||
&& transient_retries < MAX_TRANSIENT_RETRIES =>
|
||||
{
|
||||
transient_retries += 1;
|
||||
tracing::debug!(
|
||||
path = %path.display(),
|
||||
transient_retries,
|
||||
error = ?e,
|
||||
"transient LMDB env open error, retrying"
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
return Err(Error::EnvOpen {
|
||||
db: spec.label,
|
||||
source: e,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if open_failed {
|
||||
close_waits += 1;
|
||||
if close_waits > MAX_CLOSE_WAITS {
|
||||
return Err(Error::EnvOpen {
|
||||
db: spec.label,
|
||||
source: heed::Error::EnvAlreadyOpened,
|
||||
});
|
||||
}
|
||||
|
||||
match heed::env_closing_event(&path) {
|
||||
Some(event) => {
|
||||
event.wait_timeout(CLOSE_WAIT);
|
||||
}
|
||||
None => thread::sleep(Duration::from_millis(2)),
|
||||
}
|
||||
} else {
|
||||
thread::sleep(TRANSIENT_RETRY_SLEEP);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn health(&self) -> &DbHealth {
|
||||
&self.0.health
|
||||
}
|
||||
|
||||
// First caller wins: GC runs once per opened env, not once per tracker.
|
||||
pub(crate) fn try_start_gc(&self) -> bool {
|
||||
!self.0.gc_started.swap(true, Ordering::AcqRel)
|
||||
}
|
||||
|
||||
// LMDB forbids mdb_dbi_open from concurrent txns in the same process.
|
||||
pub(crate) fn lock_dbi_open(&self) -> MutexGuard<'_, ()> {
|
||||
self.0
|
||||
.dbi_lock
|
||||
.lock()
|
||||
.unwrap_or_else(PoisonError::into_inner)
|
||||
}
|
||||
|
||||
pub(crate) fn destroy(&self) -> Result<Option<heed::EnvClosingEvent>> {
|
||||
let mut pool = POOL.lock().unwrap_or_else(PoisonError::into_inner);
|
||||
let holders = Arc::strong_count(&self.0);
|
||||
|
||||
if holders > 1 {
|
||||
return Err(Error::DbInUse {
|
||||
db: self.0.label,
|
||||
path: self.0.key.clone(),
|
||||
holders: holders - 1,
|
||||
});
|
||||
}
|
||||
|
||||
pool.remove(&self.0.key);
|
||||
Ok(heed::env_closing_event(&self.0.key))
|
||||
}
|
||||
}
|
||||
|
||||
static POOL: LazyLock<Mutex<HashMap<PathBuf, Weak<PooledEnv>>>> = LazyLock::new(Mutex::default);
|
||||
|
||||
const CLOSE_WAIT: Duration = Duration::from_millis(100);
|
||||
const MAX_CLOSE_WAITS: u32 = 100;
|
||||
const TRANSIENT_RETRY_SLEEP: Duration = Duration::from_millis(50);
|
||||
const MAX_TRANSIENT_RETRIES: u32 = 8;
|
||||
|
||||
// Concurrent mdb_env_open calls on the same path can race on macOS
|
||||
// this is for some reason fixable by simple retry of the open
|
||||
fn is_transient_env_open_error(err: &heed::Error) -> bool {
|
||||
match err {
|
||||
heed::Error::Io(io) => matches!(
|
||||
io.kind(),
|
||||
std::io::ErrorKind::InvalidInput | std::io::ErrorKind::NotFound
|
||||
),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn erase_if_oversized(db_path: &Path, spec: &EnvSpec) {
|
||||
let data = db_path.join("data.mdb");
|
||||
let Ok(meta) = fs::metadata(&data) else {
|
||||
return;
|
||||
};
|
||||
|
||||
if meta.len() <= spec.size_cap_bytes {
|
||||
return;
|
||||
}
|
||||
|
||||
tracing::error!(
|
||||
path = %db_path.display(),
|
||||
size = meta.len(),
|
||||
cap = spec.size_cap_bytes,
|
||||
"LMDB db exceeds size cap, erasing"
|
||||
);
|
||||
let _ = fs::remove_file(&data);
|
||||
let _ = fs::remove_file(db_path.join("lock.mdb"));
|
||||
}
|
||||
@@ -1,10 +1,11 @@
|
||||
use super::db_healthcheck::DbHealthChecker;
|
||||
use super::lmdb::{DbHealth, LmdbStore, is_map_full};
|
||||
use super::env_pool::SharedEnv;
|
||||
use crate::error::{Error, Result};
|
||||
use crate::file_picker::FFFMode;
|
||||
use crate::git::is_modified_status;
|
||||
use crate::lmdb::{DbHealth, LmdbStore, is_map_full};
|
||||
use heed::Database;
|
||||
use heed::types::{Bytes, SerdeBincode};
|
||||
use heed::{Database, Env};
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
use std::{collections::VecDeque, path::Path};
|
||||
|
||||
@@ -19,7 +20,7 @@ const AI_MAX_HISTORY_DAYS: f64 = 7.0; // Only consider accesses within 7 days
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct FrecencyTracker {
|
||||
env: Env,
|
||||
env: SharedEnv,
|
||||
db: Database<Bytes, SerdeBincode<VecDeque<u64>>>,
|
||||
health: DbHealth,
|
||||
}
|
||||
@@ -77,7 +78,7 @@ impl LmdbStore for FrecencyTracker {
|
||||
// MAP_SIZE so we don't hit MDB_MAP_FULL before the open-time erase fires.
|
||||
const SIZE_CAP_BYTES: u64 = 12 * 1024 * 1024;
|
||||
|
||||
fn env(&self) -> &Env {
|
||||
fn shared_env(&self) -> &SharedEnv {
|
||||
&self.env
|
||||
}
|
||||
|
||||
@@ -85,7 +86,7 @@ impl LmdbStore for FrecencyTracker {
|
||||
&self.health
|
||||
}
|
||||
|
||||
fn purge_stale_data(env: &Env) -> Result<()> {
|
||||
fn purge_stale_data(env: &SharedEnv) -> Result<()> {
|
||||
let (deleted, pruned) = Self::purge_stale_entries(env)?;
|
||||
if deleted > 0 || pruned > 0 {
|
||||
tracing::info!(deleted, pruned, "Frecency GC purged entries");
|
||||
@@ -121,7 +122,7 @@ impl FrecencyTracker {
|
||||
/// Removes entries where all timestamps are older than MAX_HISTORY_DAYS,
|
||||
/// and prunes stale timestamps from entries that still have recent ones.
|
||||
/// Returns (deleted_count, pruned_count).
|
||||
fn purge_stale_entries(env: &Env) -> Result<(usize, usize)> {
|
||||
fn purge_stale_entries(env: &SharedEnv) -> Result<(usize, usize)> {
|
||||
let now = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap()
|
||||
|
||||
@@ -1,12 +1,11 @@
|
||||
use heed::{Database, Env, EnvOpenOptions};
|
||||
use std::fs;
|
||||
use heed::{Database, Env};
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use std::sync::RwLock;
|
||||
use std::sync::atomic::{AtomicU8, Ordering};
|
||||
use std::thread;
|
||||
use std::time::Duration;
|
||||
|
||||
use super::env_pool::{EnvSpec, SharedEnv};
|
||||
use crate::error::{Error, Result};
|
||||
|
||||
pub(crate) fn is_map_full(err: &heed::Error) -> bool {
|
||||
@@ -85,9 +84,13 @@ pub(crate) fn spawn_lmdb_gc<T: LmdbStore>(shared: Arc<RwLock<Option<T>>>) {
|
||||
let Some(ref tracker) = *guard else {
|
||||
return; // destroyed before we started
|
||||
};
|
||||
let env = tracker.env();
|
||||
// Trackers attaching to an already-pooled env must not repeat the
|
||||
// GC; the first opener's run flips the shared health flag.
|
||||
if !tracker.shared_env().try_start_gc() {
|
||||
return;
|
||||
}
|
||||
|
||||
if let Err(e) = T::purge_stale_data(env) {
|
||||
if let Err(e) = T::purge_stale_data(tracker.shared_env()) {
|
||||
tracing::debug!("purge_stale_data failed: {e}");
|
||||
}
|
||||
|
||||
@@ -105,18 +108,6 @@ pub(crate) fn spawn_lmdb_gc<T: LmdbStore>(shared: Arc<RwLock<Option<T>>>) {
|
||||
}
|
||||
}
|
||||
|
||||
// Concurrent `mdb_env_open` calls on the same path can race on macOS
|
||||
// this is for some reason fixabtly by simple retry of the open
|
||||
fn is_transient_env_open_error(err: &heed::Error) -> bool {
|
||||
match err {
|
||||
heed::Error::Io(io) => matches!(
|
||||
io.kind(),
|
||||
std::io::ErrorKind::InvalidInput | std::io::ErrorKind::NotFound
|
||||
),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) trait LmdbStore: Sized + Send + Sync + 'static {
|
||||
/// Short label used to defferintiate different instances of this trait
|
||||
const LABEL: &'static str;
|
||||
@@ -127,85 +118,51 @@ pub(crate) trait LmdbStore: Sized + Send + Sync + 'static {
|
||||
/// Hard cap on `data.mdb` size.
|
||||
const SIZE_CAP_BYTES: u64;
|
||||
|
||||
/// Borrow the env in the read lock
|
||||
fn env(&self) -> &Env;
|
||||
/// Borrow the pooled env handle shared by every tracker of this path.
|
||||
fn shared_env(&self) -> &SharedEnv;
|
||||
/// Borrow the health flag from the tracker.
|
||||
fn health(&self) -> &DbHealth;
|
||||
|
||||
/// Borrow the raw heed env.
|
||||
fn env(&self) -> &Env {
|
||||
self.shared_env()
|
||||
}
|
||||
|
||||
/// Override to purge stale rows, compact, etc. Default no-op. Runs on
|
||||
/// the GC thread while a read lock is held against the shared handle,
|
||||
/// so destroy / re-init naturally wait for it.
|
||||
fn purge_stale_data(_env: &Env) -> Result<()> {
|
||||
fn purge_stale_data(_env: &SharedEnv) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Open the LMDB env. Returns env + a `DbHealth` starting in Pending;
|
||||
/// the GC thread spawned by `spawn_gc` flips it to Healthy. Write
|
||||
/// paths flip it to Degraded on MDB_MAP_FULL.
|
||||
/// Open (or join) the process-shared LMDB env for `db_path`. The health
|
||||
/// flag is per-env: the GC of the first opener flips it for everyone.
|
||||
#[tracing::instrument]
|
||||
fn open_env(db_path: &Path) -> Result<(Env, DbHealth)> {
|
||||
Self::erase_if_oversized(db_path);
|
||||
fs::create_dir_all(db_path).map_err(Error::CreateDir)?;
|
||||
let db = Self::LABEL;
|
||||
|
||||
const MAX_ATTEMPTS: u32 = 8;
|
||||
let mut attempt = 0u32;
|
||||
let env = loop {
|
||||
let result = unsafe {
|
||||
let mut opts = EnvOpenOptions::new();
|
||||
opts.map_size(Self::MAP_SIZE);
|
||||
if Self::MAX_DBS > 0 {
|
||||
opts.max_dbs(Self::MAX_DBS);
|
||||
}
|
||||
opts.open(db_path)
|
||||
};
|
||||
|
||||
match result {
|
||||
Ok(env) => break env,
|
||||
Err(e) if is_transient_env_open_error(&e) && attempt + 1 < MAX_ATTEMPTS => {
|
||||
attempt += 1;
|
||||
tracing::debug!(
|
||||
path = %db_path.display(),
|
||||
attempt,
|
||||
error = ?e,
|
||||
"transient LMDB env open error, retrying"
|
||||
);
|
||||
|
||||
thread::sleep(Duration::from_millis(50));
|
||||
}
|
||||
Err(e) => return Err(Error::EnvOpen { db, source: e }),
|
||||
}
|
||||
};
|
||||
|
||||
// Reclaim reader slots left behind by prior processes that died
|
||||
// without cleanup. Must run before we start any read txns (which
|
||||
// open_database_safe does) — otherwise we may hit MDB_READERS_FULL
|
||||
// on a fresh env just because lock.mdb still has stale entries
|
||||
// from a previous crash.
|
||||
//
|
||||
// This is the one LMDB maintenance call we run on the caller's
|
||||
// thread. If the lock file is genuinely wedged this will block
|
||||
// forever, but the alternative — never getting past init — is
|
||||
// worse and the bg-thread trick doesn't solve it anyway.
|
||||
match env.clear_stale_readers() {
|
||||
Ok(cleared) if cleared > 0 => {
|
||||
tracing::warn!(cleared, "reclaimed stale LMDB reader slots at open");
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(e) => tracing::debug!("clear_stale_readers at open failed: {e}"),
|
||||
}
|
||||
|
||||
Ok((env, DbHealth::new()))
|
||||
fn open_env(db_path: &Path) -> Result<(SharedEnv, DbHealth)> {
|
||||
let shared = SharedEnv::get_or_open(
|
||||
db_path,
|
||||
&EnvSpec {
|
||||
label: Self::LABEL,
|
||||
map_size: Self::MAP_SIZE,
|
||||
max_dbs: Self::MAX_DBS,
|
||||
size_cap_bytes: Self::SIZE_CAP_BYTES,
|
||||
},
|
||||
)?;
|
||||
let health = shared.health().clone();
|
||||
Ok((shared, health))
|
||||
}
|
||||
|
||||
/// Open or create a database without blocking on the LMDB writer mutex
|
||||
/// when the database already exists.
|
||||
fn open_database_safe<KC, DC>(env: &Env, name: Option<&str>) -> Result<Database<KC, DC>>
|
||||
fn open_database_safe<KC, DC>(env: &SharedEnv, name: Option<&str>) -> Result<Database<KC, DC>>
|
||||
where
|
||||
KC: 'static,
|
||||
DC: 'static,
|
||||
{
|
||||
let db = Self::LABEL;
|
||||
// mdb_dbi_open must not run from concurrent txns in this process.
|
||||
let _dbi_guard = env.lock_dbi_open();
|
||||
|
||||
let rtxn = env
|
||||
.read_txn()
|
||||
.map_err(|source| Error::DbStartReadTxn { db, source })?;
|
||||
@@ -237,23 +194,4 @@ pub(crate) trait LmdbStore: Sized + Send + Sync + 'static {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn erase_if_oversized(db_path: &Path) {
|
||||
let data = db_path.join("data.mdb");
|
||||
let Ok(meta) = fs::metadata(&data) else {
|
||||
return;
|
||||
};
|
||||
if meta.len() <= Self::SIZE_CAP_BYTES {
|
||||
return;
|
||||
}
|
||||
|
||||
tracing::error!(
|
||||
path = %db_path.display(),
|
||||
size = meta.len(),
|
||||
cap = Self::SIZE_CAP_BYTES,
|
||||
"LMDB db exceeds size cap, erasing"
|
||||
);
|
||||
let _ = fs::remove_file(&data);
|
||||
let _ = fs::remove_file(db_path.join("lock.mdb"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
pub(crate) mod env_pool;
|
||||
pub(crate) mod lmdb;
|
||||
|
||||
pub mod db_healthcheck;
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
use super::db_healthcheck::DbHealthChecker;
|
||||
use super::lmdb::{DbHealth, LmdbStore, is_map_full};
|
||||
use super::env_pool::SharedEnv;
|
||||
use crate::error::Error;
|
||||
use crate::lmdb::{DbHealth, LmdbStore, is_map_full};
|
||||
use heed::types::{Bytes, SerdeBincode};
|
||||
use heed::{Database, Env};
|
||||
use serde::{Deserialize, Serialize};
|
||||
@@ -27,7 +28,7 @@ struct HistoryEntry {
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct QueryTracker {
|
||||
env: Env,
|
||||
env: SharedEnv,
|
||||
// Database for (project_path, query) -> QueryMatchEntry mappings
|
||||
query_file_db: Database<Bytes, SerdeBincode<QueryMatchEntry>>,
|
||||
// Database for project_path -> VecDeque<HistoryEntry> mappings (file picker)
|
||||
@@ -92,7 +93,7 @@ impl LmdbStore for QueryTracker {
|
||||
const MAX_DBS: u32 = 16;
|
||||
const SIZE_CAP_BYTES: u64 = 8 * 1024 * 1024;
|
||||
|
||||
fn env(&self) -> &Env {
|
||||
fn shared_env(&self) -> &SharedEnv {
|
||||
&self.env
|
||||
}
|
||||
|
||||
|
||||
@@ -34,6 +34,22 @@ pub enum Error {
|
||||
#[source]
|
||||
source: heed::Error,
|
||||
},
|
||||
#[error(
|
||||
"LMDB env at {path} is already open as the '{open_as}' database with different options; requested by '{requested_as}'. Use a distinct path per database."
|
||||
)]
|
||||
EnvSpecMismatch {
|
||||
path: std::path::PathBuf,
|
||||
open_as: &'static str,
|
||||
requested_as: &'static str,
|
||||
},
|
||||
#[error(
|
||||
"The {db} database at {path} is still used by {holders} other tracker(s) in this process"
|
||||
)]
|
||||
DbInUse {
|
||||
db: &'static str,
|
||||
path: std::path::PathBuf,
|
||||
holders: usize,
|
||||
},
|
||||
#[error("Failed to create {db} database: {source}")]
|
||||
DbCreate {
|
||||
db: &'static str,
|
||||
|
||||
@@ -45,6 +45,7 @@ use crate::types::{
|
||||
ContentCacheBudget, DirItem, DirSearchResult, FileItem, MixedItemRef, MixedSearchResult,
|
||||
PaginationArgs, Score, ScoringContext, SearchResult,
|
||||
};
|
||||
use crate::walk::WalkOutput;
|
||||
use crate::watch::BackgroundWatcher;
|
||||
use fff_query_parser::FFFQuery;
|
||||
use git2::{Repository, Status};
|
||||
@@ -628,6 +629,10 @@ impl FilePicker {
|
||||
&self.base_path
|
||||
}
|
||||
|
||||
pub fn has_git_repo(&self) -> bool {
|
||||
self.sync_data.git_workdir.is_some()
|
||||
}
|
||||
|
||||
/// Ignore rules the walker assembled during the last scan (zlob backend
|
||||
/// only). The background watcher uses these to filter events without
|
||||
/// libgit2. `None` when the backend doesn't surface rules or no ignore
|
||||
@@ -2020,30 +2025,38 @@ impl FileSync {
|
||||
let is_git_repo = git_workdir.is_some();
|
||||
let bg_threads = BACKGROUND_THREAD_POOL.current_num_threads();
|
||||
|
||||
let mut walk_output = crate::walk::walk_collect_files(
|
||||
let WalkOutput {
|
||||
dirs: mut walked_dirs,
|
||||
mut pairs,
|
||||
ignore_rules,
|
||||
} = crate::walk::walk_collect_files(
|
||||
base_path,
|
||||
is_git_repo,
|
||||
follow_symlinks,
|
||||
bg_threads,
|
||||
synced_files_count,
|
||||
)?;
|
||||
let ignore_rules = walk_output.ignore_rules.take().map(Arc::new);
|
||||
let mut pairs = walk_output.pairs;
|
||||
let ignore_rules = ignore_rules.map(Arc::new);
|
||||
|
||||
// Sort by (dir_part, filename). This groups files by their directory
|
||||
// into contiguous runs so the linear dir-extraction pass below can
|
||||
// dedupe by comparing only against the previous dir.
|
||||
// group walked dirs and files with a dir part to the same order
|
||||
BACKGROUND_THREAD_POOL.install(|| {
|
||||
pairs.par_sort_unstable_by(|(a, path_a), (b, path_b)| {
|
||||
// SAFETY: `filename_offset` is always at a character boundary
|
||||
let (a_dir, a_file) = path_a.split_at(a.path.filename_offset as usize);
|
||||
let (b_dir, b_file) = path_b.split_at(b.path.filename_offset as usize);
|
||||
a_dir.cmp(b_dir).then_with(|| a_file.cmp(b_file))
|
||||
});
|
||||
rayon::join(
|
||||
|| {
|
||||
pairs.par_sort_unstable_by(|(a, path_a), (b, path_b)| {
|
||||
// SAFETY: `filename_offset` is always at a character boundary
|
||||
let (a_dir, a_file) = path_a.split_at(a.path.filename_offset as usize);
|
||||
let (b_dir, b_file) = path_b.split_at(b.path.filename_offset as usize);
|
||||
a_dir.cmp(b_dir).then_with(|| a_file.cmp(b_file))
|
||||
});
|
||||
},
|
||||
|| walked_dirs.par_sort_unstable(),
|
||||
);
|
||||
});
|
||||
walked_dirs.dedup();
|
||||
|
||||
let mut builder = crate::simd_path::ChunkedPathStoreBuilder::new(pairs.len());
|
||||
let dirs = populates_dirs_files_chunked_storage(&mut pairs, &mut builder);
|
||||
let dirs = populates_dirs_files_chunked_storage(&mut pairs, &walked_dirs, &mut builder);
|
||||
drop(walked_dirs);
|
||||
|
||||
let mut files: Vec<FileItem> = pairs.into_iter().map(|(file, _)| file).collect();
|
||||
let chunked_paths = builder.finish();
|
||||
@@ -2164,49 +2177,91 @@ pub(crate) fn warmup_mmaps(
|
||||
}
|
||||
|
||||
/// This does both thing (yes sorry all the OOP morons)
|
||||
/// in one go: populates files chunked storage and creates new directories
|
||||
/// in one go: populates files chunked storage and builds the dir table from
|
||||
/// `walked_dirs` (every dir the walker visited: sorted, '/'-terminated,
|
||||
/// deduped), merging file parents in a single lockstep sweep so dirs with no
|
||||
/// files (empty subtrees, pure ancestors) are indexed and searchable too.
|
||||
fn populates_dirs_files_chunked_storage<'a>(
|
||||
pairs: &'a mut [(FileItem, String)],
|
||||
walked_dirs: &[String],
|
||||
chunk_storage: &mut crate::simd_path::ChunkedPathStoreBuilder,
|
||||
) -> Vec<DirItem> {
|
||||
let mut dirs: Vec<DirItem> = Vec::new();
|
||||
let mut dirs: Vec<DirItem> = Vec::with_capacity(walked_dirs.len() + 1);
|
||||
let mut dir_iter = walked_dirs.iter().peekable();
|
||||
|
||||
// Root-level files sort first and their "" parent is never a walker dir.
|
||||
if pairs
|
||||
.first()
|
||||
.is_some_and(|(f, _)| f.path.filename_offset == 0)
|
||||
{
|
||||
push_dir_item(&mut dirs, chunk_storage, "");
|
||||
}
|
||||
|
||||
// Detects contiguous same-dir runs (pairs are sorted by dir) so the
|
||||
// merge below runs once per directory, not once per file.
|
||||
let mut prev_dir: &'a str = "";
|
||||
let mut prev_dir_valid = false;
|
||||
let mut current_dir_idx: u32 = 0;
|
||||
|
||||
for (file, rel) in pairs.iter_mut() {
|
||||
let rel: &'a str = rel;
|
||||
let dir_part: &'a str = &rel[..file.path.filename_offset as usize];
|
||||
|
||||
if !prev_dir_valid || prev_dir != dir_part {
|
||||
let dir_string = chunk_storage.add_dir_immediate(dir_part);
|
||||
if prev_dir != dir_part {
|
||||
// Flush walked dirs up to and including this file's parent,
|
||||
// keeping the table sorted for the find_dir_index binary search.
|
||||
while let Some(dir) = dir_iter.peek()
|
||||
&& dir.as_str() < dir_part
|
||||
{
|
||||
push_dir_item(&mut dirs, chunk_storage, dir);
|
||||
dir_iter.next();
|
||||
}
|
||||
|
||||
// Compute last-segment offset: for "src/components/" -> 4 (points to "components/")
|
||||
let last_seg = if dir_part.is_empty() {
|
||||
0
|
||||
} else {
|
||||
let trimmed = dir_part.trim_end_matches(std::path::is_separator);
|
||||
trimmed
|
||||
.rfind(std::path::is_separator)
|
||||
.map(|i| i + 1)
|
||||
.unwrap_or(0) as u16
|
||||
};
|
||||
match dir_iter.peek() {
|
||||
Some(dir) if dir.as_str() == dir_part => {
|
||||
push_dir_item(&mut dirs, chunk_storage, dir);
|
||||
dir_iter.next();
|
||||
}
|
||||
// Parents the walker reported with a non-dir kind
|
||||
// (e.g. followed symlinks) aren't in the list.
|
||||
_ => push_dir_item(&mut dirs, chunk_storage, dir_part),
|
||||
}
|
||||
|
||||
dirs.push(DirItem::new(dir_string, last_seg));
|
||||
current_dir_idx = (dirs.len() - 1) as u32;
|
||||
|
||||
prev_dir = dir_part;
|
||||
prev_dir_valid = true;
|
||||
}
|
||||
|
||||
file.path = chunk_storage.add_file_immediate(rel, file.path.filename_offset);
|
||||
file.parent_dir_index = current_dir_idx;
|
||||
}
|
||||
|
||||
for dir in dir_iter {
|
||||
push_dir_item(&mut dirs, chunk_storage, dir);
|
||||
}
|
||||
|
||||
dirs
|
||||
}
|
||||
|
||||
fn push_dir_item(
|
||||
dirs: &mut Vec<DirItem>,
|
||||
chunk_storage: &mut crate::simd_path::ChunkedPathStoreBuilder,
|
||||
dir_part: &str,
|
||||
) {
|
||||
let dir_string = chunk_storage.add_dir_immediate(dir_part);
|
||||
|
||||
// Compute last-segment offset: for "src/components/" -> 4 (points to "components/")
|
||||
let last_seg = if dir_part.is_empty() {
|
||||
0
|
||||
} else {
|
||||
let trimmed = dir_part.trim_end_matches(std::path::is_separator);
|
||||
trimmed
|
||||
.rfind(std::path::is_separator)
|
||||
.map(|i| i + 1)
|
||||
.unwrap_or(0) as u16
|
||||
};
|
||||
|
||||
dirs.push(DirItem::new(dir_string, last_seg));
|
||||
}
|
||||
|
||||
/// Fast extension-based binary detection. Avoids opening files during scan.
|
||||
/// Covers the vast majority of binary files in typical repositories.
|
||||
#[inline]
|
||||
@@ -2341,13 +2396,9 @@ mod tests {
|
||||
use super::*;
|
||||
|
||||
/// The watcher must watch every ancestor directory up to `base_path`,
|
||||
/// not just the immediate parents of indexed files. Intermediate dirs
|
||||
/// that contain only subdirectories (no direct files) are NOT in
|
||||
/// `sync_data.dirs` — yet they must still appear in `extract_watch_dirs`
|
||||
/// so Create events on new subdirectories below them fire.
|
||||
///
|
||||
/// Correctness regression guard for any refactor that replaces the
|
||||
/// ancestor walk with a direct `sync_data.dirs` iteration.
|
||||
/// not just the immediate parents of indexed files. The dir table is
|
||||
/// built from the walker's visited dirs, so pure ancestors (dirs that
|
||||
/// contain only subdirectories) must be present and emitted exactly once.
|
||||
#[test]
|
||||
fn extract_watch_dirs_includes_pure_ancestor_dirs() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
@@ -2361,17 +2412,6 @@ mod tests {
|
||||
// base/src/components/button.txt (src/components has a file)
|
||||
// base/src/routes/home.txt (src/routes has a file)
|
||||
// base/lib/deep/nested/util.txt (lib and lib/deep have no files)
|
||||
//
|
||||
// `sync_data.dirs` will only contain:
|
||||
// src/components/
|
||||
// src/routes/
|
||||
// lib/deep/nested/
|
||||
//
|
||||
// But the watcher also needs:
|
||||
// src/ (pure ancestor — no direct files)
|
||||
// lib/ (pure ancestor)
|
||||
// lib/deep/ (pure ancestor)
|
||||
// otherwise new siblings like `src/NewDir/x.txt` are missed.
|
||||
for rel in [
|
||||
"src/components/button.txt",
|
||||
"src/routes/home.txt",
|
||||
@@ -2429,6 +2469,97 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// Regression guard for #725: dirs that are EMPTY at scan time are merged
|
||||
/// into `sync_data.dirs` so they are searchable and get an inotify watch;
|
||||
/// files created in them later must be detected.
|
||||
#[test]
|
||||
fn for_each_dir_includes_empty_directories() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let base_buf = crate::path_utils::canonicalize(dir.path()).unwrap();
|
||||
let base = base_buf.as_path();
|
||||
|
||||
// Tree:
|
||||
// base/init.lua (file directly under base)
|
||||
// base/commands/ (empty at scan — the #725 repro)
|
||||
// base/src/main.rs (src is indexed)
|
||||
// base/src/plugins/extra/ (empty chain under an indexed dir)
|
||||
std::fs::create_dir_all(base.join("commands")).unwrap();
|
||||
std::fs::create_dir_all(base.join("src/plugins/extra")).unwrap();
|
||||
std::fs::write(base.join("init.lua"), b"x").unwrap();
|
||||
std::fs::write(base.join("src/main.rs"), b"x").unwrap();
|
||||
|
||||
let mut picker = FilePicker::new(FilePickerOptions {
|
||||
base_path: base.to_str().unwrap().into(),
|
||||
watch: false,
|
||||
..Default::default()
|
||||
})
|
||||
.unwrap();
|
||||
picker.collect_files().unwrap();
|
||||
|
||||
let mut watch_dirs: Vec<PathBuf> = Vec::new();
|
||||
picker.for_each_dir(|p| {
|
||||
watch_dirs.push(p.to_path_buf());
|
||||
std::ops::ControlFlow::Continue(())
|
||||
});
|
||||
let watch_set: std::collections::HashSet<PathBuf> = watch_dirs.iter().cloned().collect();
|
||||
|
||||
for rel in ["commands", "src/plugins", "src/plugins/extra", "src"] {
|
||||
assert!(
|
||||
watch_set.contains(&base.join(rel)),
|
||||
"expected {rel} in watch dirs, got {watch_set:?}",
|
||||
);
|
||||
}
|
||||
|
||||
// Dirs covered by indexed files must not be duplicated.
|
||||
assert_eq!(
|
||||
watch_dirs.len(),
|
||||
watch_set.len(),
|
||||
"duplicate watch dir emitted: {watch_dirs:?}",
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dir_table_merges_walked_dirs_with_file_parents() {
|
||||
let mut pairs: Vec<(FileItem, String)> = ["src/main.rs", "src/deep/lib.rs", "root.txt"]
|
||||
.iter()
|
||||
.map(|p| {
|
||||
let (item, rel) = FileItem::new(PathBuf::from(p), Path::new(""), None);
|
||||
(item, rel)
|
||||
})
|
||||
.collect();
|
||||
pairs.sort_by(|(a, pa), (b, pb)| {
|
||||
pa[..a.path.filename_offset as usize]
|
||||
.cmp(&pb[..b.path.filename_offset as usize])
|
||||
.then_with(|| pa.cmp(pb))
|
||||
});
|
||||
|
||||
// Sorted '/'-terminated walker output: file parents + an empty dir +
|
||||
// a sibling sharing a prefix with a file parent.
|
||||
let walked: Vec<String> = ["empty/", "src/", "src/deep/", "src/deeper/"]
|
||||
.iter()
|
||||
.map(|s| s.to_string())
|
||||
.collect();
|
||||
|
||||
let mut builder = crate::simd_path::ChunkedPathStoreBuilder::new(pairs.len());
|
||||
let dirs = populates_dirs_files_chunked_storage(&mut pairs, &walked, &mut builder);
|
||||
let store = builder.finish();
|
||||
let arena = store.as_arena_ptr();
|
||||
|
||||
let table: Vec<String> = dirs.iter().map(|d| d.relative_path(arena)).collect();
|
||||
// Sorted: "" (root files) first, all walked dirs present exactly once.
|
||||
assert_eq!(table, ["", "empty/", "src/", "src/deep/", "src/deeper/"]);
|
||||
|
||||
// Every file's parent_dir_index points at its own dir entry.
|
||||
for (file, _) in &pairs {
|
||||
let dir = &dirs[file.parent_dir_index as usize];
|
||||
let rel = file.relative_path(arena);
|
||||
assert!(
|
||||
rel.starts_with(&dir.relative_path(arena)),
|
||||
"file {rel} must live under its parent dir",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn common_dir_prefix_len_cases() {
|
||||
assert_eq!(common_dir_prefix_len("", ""), 0);
|
||||
|
||||
@@ -62,7 +62,7 @@ impl GitStatusCache {
|
||||
|
||||
let mut entries = AHashMap::with_capacity(statuses.len());
|
||||
for entry in &statuses {
|
||||
if let Some(entry_path) = entry.path() {
|
||||
if let Ok(entry_path) = entry.path() {
|
||||
// libgit2 returns entry paths with forward slashes on every platform
|
||||
// fff stores native paths - meaning we have forward slash issue on windows
|
||||
let full_path = crate::path_utils::normalize(repo_path.join(entry_path));
|
||||
|
||||
@@ -229,9 +229,19 @@ pub(crate) fn grep_search<'a>(
|
||||
return result;
|
||||
}
|
||||
|
||||
// Keep any explicit FilePath scope (AI mode `path/to/file.ext` prefix) so the
|
||||
// fallback can't leak matches outside the file the user pinned. Only the
|
||||
// swallowed operator/glob tokens are dropped. See issue #756.
|
||||
let scoped_constraints: fff_query_parser::ConstraintVec<'_> = query
|
||||
.constraints
|
||||
.iter()
|
||||
.filter(|c| matches!(c, fff_query_parser::Constraint::FilePath(_)))
|
||||
.cloned()
|
||||
.collect();
|
||||
|
||||
let literal_query = FFFQuery {
|
||||
raw_query: query.raw_query,
|
||||
constraints: Vec::new(),
|
||||
constraints: scoped_constraints,
|
||||
fuzzy_query: fff_query_parser::FuzzyQuery::Text(raw),
|
||||
location: None,
|
||||
};
|
||||
|
||||
@@ -461,3 +461,58 @@ fn test_grep_no_duplicates_with_overflow_trailing_bits() {
|
||||
result.matches.len()
|
||||
);
|
||||
}
|
||||
|
||||
/// Issue #756: an AI-mode regex query with an inline FilePath scope and
|
||||
/// top-level alternation. The regex fragments are swallowed as bogus Glob
|
||||
/// constraints, the constrained search finds nothing, and the literal/regex
|
||||
/// fallback must NOT drop the FilePath scope — otherwise the `|` branch leaks
|
||||
/// matches into files outside the pinned path.
|
||||
#[test]
|
||||
fn regex_fallback_keeps_file_path_scope_issue_756() {
|
||||
use fff_query_parser::{AiGrepConfig, QueryParser};
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let base = crate::path_utils::canonicalize(dir.path()).unwrap();
|
||||
std::fs::create_dir(base.join("scope")).unwrap();
|
||||
std::fs::write(
|
||||
base.join("scope").join("target.css"),
|
||||
"/* ---------- target ---------- */\n",
|
||||
)
|
||||
.unwrap();
|
||||
std::fs::write(
|
||||
base.join("outside.css"),
|
||||
"/* ---------- outside ---------- */\n",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let mut picker = FilePicker::new(FilePickerOptions {
|
||||
base_path: base.to_str().unwrap().into(),
|
||||
watch: false,
|
||||
..Default::default()
|
||||
})
|
||||
.unwrap();
|
||||
picker.collect_files().unwrap();
|
||||
|
||||
let options = crate::GrepSearchOptions {
|
||||
mode: super::GrepMode::Regex,
|
||||
smart_case: true,
|
||||
max_matches_per_file: 80,
|
||||
page_limit: 100,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let raw = r"scope/target.css ^/\* |^\s*/\* ----------";
|
||||
let query = QueryParser::new(AiGrepConfig).parse(raw);
|
||||
let result = picker.grep(&query, &options);
|
||||
let mut paths: Vec<String> = result
|
||||
.files
|
||||
.iter()
|
||||
.map(|f| f.relative_path(&picker))
|
||||
.collect();
|
||||
paths.sort();
|
||||
|
||||
assert_eq!(
|
||||
paths,
|
||||
vec!["scope/target.css"],
|
||||
"regex fallback must not leak outside the FilePath scope"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
use super::grep::{GrepContext, perform_grep};
|
||||
use super::prefilter::prefilter_with_filepath_retry;
|
||||
use super::prefilter::prefilter_files;
|
||||
use super::sink::{SinkState, debug_assert_newline_terminator};
|
||||
use super::types::{GrepResult, GrepSearchOptions};
|
||||
use crate::index::{BigramFilter, BigramOverlay, bigram_boundary, literal_candidates};
|
||||
@@ -121,7 +121,8 @@ pub(crate) fn multi_grep_search<'a>(
|
||||
let bigram_candidates = literal_candidates(bigram_index, bigram_overlay, patterns);
|
||||
let base_file_count = bigram_boundary(bigram_overlay, files.len());
|
||||
|
||||
let (files_to_search, filtered_file_count) = prefilter_with_filepath_retry(
|
||||
// Constraints are separate from patterns, so a miss must not broaden the search.
|
||||
let (files_to_search, filtered_file_count) = prefilter_files(
|
||||
files,
|
||||
constraints,
|
||||
bigram_candidates.as_deref(),
|
||||
|
||||
@@ -50,7 +50,7 @@ pub(super) fn prefilter_with_filepath_retry<'a>(
|
||||
/// Single pass prefilter that doesn't involve file reading
|
||||
/// allocates only amount of memory required for storing references of the FileItems have to be
|
||||
/// opened for grepping unaviodably, in the worst case allocates N * <word> memory if no prefilter needed
|
||||
fn prefilter_files<'a>(
|
||||
pub(crate) fn prefilter_files<'a>(
|
||||
files: &'a [FileItem],
|
||||
constraints: &[Constraint<'_>],
|
||||
bigram_candidates: Option<&[u64]>,
|
||||
|
||||
@@ -3,28 +3,53 @@ use std::path::Path;
|
||||
/// Directories excluded when walking a non-git root. Entries are `cfg`-gated
|
||||
/// so a single iteration covers standard + platform-specific overrides.
|
||||
pub(crate) const IGNORED_DIRS: &[&str] = &[
|
||||
// various dev tools that can be meet in the developer app
|
||||
"node_modules",
|
||||
"__pycache__",
|
||||
"venv",
|
||||
".venv",
|
||||
// Rust (glob-only patterns for non_git_repo_overrides; is_non_code_directory
|
||||
// matches the "target" component separately).
|
||||
"target/debug",
|
||||
"target/release",
|
||||
"target/rust-analyzer",
|
||||
"target/criterion",
|
||||
// Language package caches in non-git roots.
|
||||
"go/pkg/mod",
|
||||
".cargo/registry",
|
||||
".rustup/toolchains",
|
||||
".gradle/caches",
|
||||
".m2/repository",
|
||||
".npm/_cacache",
|
||||
".pub-cache",
|
||||
#[cfg(not(target_os = "windows"))]
|
||||
".local/state", // this contains tons of logs which generate too much watcher noise
|
||||
#[cfg(target_os = "macos")]
|
||||
"Library/Application Support",
|
||||
#[cfg(target_os = "macos")]
|
||||
"Library/Caches",
|
||||
// App-group sandbox storage — used by iMessage, Photos, Notes, Calendar,
|
||||
// Electron apps, etc. for SQLite-WAL, LevelDB, protobuf files. These are
|
||||
// almost entirely extension-less binary files (~80k on a typical $HOME)
|
||||
// that never need to appear in a fuzzy or grep search.
|
||||
#[cfg(target_os = "macos")]
|
||||
"Library/Group Containers",
|
||||
"Library/Containers", // sandboxed apps data
|
||||
#[cfg(target_os = "macos")]
|
||||
"Library/Containers",
|
||||
"Library/Group Containers", // random application data and networking
|
||||
#[cfg(target_os = "macos")]
|
||||
"Library/pnpm",
|
||||
#[cfg(target_os = "macos")]
|
||||
"Library/Metadata",
|
||||
#[cfg(target_os = "macos")]
|
||||
"Library/Developer/CoreSimulator",
|
||||
#[cfg(target_os = "macos")]
|
||||
"Library/Android",
|
||||
#[cfg(target_os = "macos")]
|
||||
"Library/Logs",
|
||||
#[cfg(target_os = "macos")]
|
||||
"Library/Daemon Containers",
|
||||
#[cfg(target_os = "macos")]
|
||||
"Library/Trial",
|
||||
#[cfg(target_os = "macos")]
|
||||
"Library/Preferences",
|
||||
#[cfg(target_os = "macos")]
|
||||
"Library/Messages",
|
||||
#[cfg(target_os = "macos")]
|
||||
"Library/IdentityServices",
|
||||
#[cfg(target_os = "windows")]
|
||||
"bin/Debug",
|
||||
#[cfg(target_os = "windows")]
|
||||
@@ -57,6 +82,10 @@ pub(crate) fn non_git_repo_overrides(base_path: &Path) -> Option<ignore::overrid
|
||||
pub(crate) fn is_non_code_directory(path: &Path) -> bool {
|
||||
let path_str = path.as_os_str().to_str().unwrap_or("");
|
||||
IGNORED_DIRS.iter().any(|&dir| {
|
||||
// Entries are gitignore patterns for the walkers; here they are matched
|
||||
// as substrings, so a leading `*` wildcard has to come off first.
|
||||
let dir = dir.strip_prefix('*').unwrap_or(dir);
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
let dir = dir.replace('/', std::path::MAIN_SEPARATOR_STR);
|
||||
#[cfg(target_os = "windows")]
|
||||
@@ -66,3 +95,42 @@ pub(crate) fn is_non_code_directory(path: &Path) -> bool {
|
||||
path_str.contains(dir)
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(all(test, target_os = "macos"))]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn home_machine_state_is_excluded_but_source_trees_are_not() {
|
||||
// Representative machine state from a home index.
|
||||
for rel in [
|
||||
"Library/pnpm/store/v3/files/00/abcdef",
|
||||
"Library/Preferences/com.apple.finder.plist",
|
||||
"Library/Messages/prewarm.db-shm",
|
||||
"Library/IdentityServices/TetraDB-identityservicesd.db-wal",
|
||||
"Library/Developer/CoreSimulator/Devices/X/data/f",
|
||||
"go/pkg/mod/github.com/x/y@v1/main.go",
|
||||
".cargo/registry/src/index.crates.io-1/serde-1.0/src/lib.rs",
|
||||
"Library/Android/sdk/platforms/android-34/data/x",
|
||||
".local/state/nvim/fff+123+456.log",
|
||||
] {
|
||||
assert!(
|
||||
is_non_code_directory(Path::new(rel)),
|
||||
"{rel} must not reach the index"
|
||||
);
|
||||
}
|
||||
|
||||
// Source trees under $HOME stay searchable.
|
||||
for rel in [
|
||||
"dev/chromium/third_party/blink/renderer/core/dom/node.cc",
|
||||
"dev/fff/crates/fff-core/src/lib.rs",
|
||||
"Documents/notes/todo.md",
|
||||
"dev/myproj/pkg/mod/thing.go",
|
||||
] {
|
||||
assert!(
|
||||
!is_non_code_directory(Path::new(rel)),
|
||||
"{rel} must stay searchable"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
//! # FFF Search — High-performance file finder core
|
||||
//!
|
||||
//! This crate provides the core search engine for [FFF (Fast File Finder)](https://github.com/dmtrKovalenko/fff.nvim).
|
||||
//! This crate provides the core search engine for [FFF (Fast File Finder)](https://github.com/dmtrKovalenko/fff).
|
||||
//! It includes filesystem indexing with real-time watching, fuzzy matching powered
|
||||
//! by [frizbee](https://docs.rs/neo_frizbee), frecency scoring backed by LMDB,
|
||||
//! and multi-mode grep search.
|
||||
@@ -135,6 +135,12 @@ pub use types::*;
|
||||
|
||||
pub mod constants;
|
||||
|
||||
/// Watcher rescan request accounting.
|
||||
pub mod rescan_stats;
|
||||
pub use rescan_stats::{RESCAN_STATS_ENABLED, RescanReason, RescanStats};
|
||||
|
||||
mod rescan_throttle;
|
||||
|
||||
// ==================================
|
||||
// these are public only for benchmarks, no backward compatibility guaranteed
|
||||
#[doc(hidden)]
|
||||
|
||||
@@ -0,0 +1,215 @@
|
||||
#[cfg(rescan_stats)]
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
/// Whether rescan accounting is compiled in.
|
||||
pub const RESCAN_STATS_ENABLED: bool = cfg!(rescan_stats);
|
||||
|
||||
/// Cause recorded for a filesystem rescan request.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
|
||||
pub enum RescanReason {
|
||||
/// Requested through the public API (refresh, directory change).
|
||||
Explicit,
|
||||
/// The kernel dropped events and asked us to re-read the subtree.
|
||||
KernelEventLoss,
|
||||
/// A `.gitignore`/`.ignore` changed, so the cached ignore rules are stale.
|
||||
IgnoreFileChanged,
|
||||
/// A single debounce batch touched more paths than we apply incrementally.
|
||||
EventBatchOverflow,
|
||||
/// The picker refused an incremental insert/update.
|
||||
IndexUpdateRejected,
|
||||
/// The post-scan overflow region ran out of slots.
|
||||
OverflowCapacity,
|
||||
}
|
||||
|
||||
impl RescanReason {
|
||||
pub const ALL: [RescanReason; 6] = [
|
||||
RescanReason::Explicit,
|
||||
RescanReason::KernelEventLoss,
|
||||
RescanReason::IgnoreFileChanged,
|
||||
RescanReason::EventBatchOverflow,
|
||||
RescanReason::IndexUpdateRejected,
|
||||
RescanReason::OverflowCapacity,
|
||||
];
|
||||
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
RescanReason::Explicit => "explicit",
|
||||
RescanReason::KernelEventLoss => "kernel_event_loss",
|
||||
RescanReason::IgnoreFileChanged => "ignore_file_changed",
|
||||
RescanReason::EventBatchOverflow => "event_batch_overflow",
|
||||
RescanReason::IndexUpdateRejected => "index_update_rejected",
|
||||
RescanReason::OverflowCapacity => "overflow_capacity",
|
||||
}
|
||||
}
|
||||
|
||||
const fn slot(self) -> usize {
|
||||
match self {
|
||||
RescanReason::Explicit => 0,
|
||||
RescanReason::KernelEventLoss => 1,
|
||||
RescanReason::IgnoreFileChanged => 2,
|
||||
RescanReason::EventBatchOverflow => 3,
|
||||
RescanReason::IndexUpdateRejected => 4,
|
||||
RescanReason::OverflowCapacity => 5,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for RescanReason {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.write_str(self.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
/// Snapshot of rescan requests grouped by reason.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
pub struct RescanStats {
|
||||
pub total: usize,
|
||||
/// Requests suppressed during the cooldown.
|
||||
pub throttled: usize,
|
||||
counts: [usize; RescanReason::ALL.len()],
|
||||
throttled_counts: [usize; RescanReason::ALL.len()],
|
||||
}
|
||||
|
||||
impl RescanStats {
|
||||
pub fn count(&self, reason: RescanReason) -> usize {
|
||||
self.counts[reason.slot()]
|
||||
}
|
||||
|
||||
pub fn count_throttled(&self, reason: RescanReason) -> usize {
|
||||
self.throttled_counts[reason.slot()]
|
||||
}
|
||||
|
||||
/// Admitted requests originating from watcher fallbacks.
|
||||
pub fn watcher_triggered(&self) -> usize {
|
||||
self.total - self.count(RescanReason::Explicit)
|
||||
}
|
||||
|
||||
/// Per-reason delta against an earlier snapshot.
|
||||
pub fn since(&self, earlier: &RescanStats) -> RescanStats {
|
||||
let mut counts = [0usize; RescanReason::ALL.len()];
|
||||
let mut throttled_counts = [0usize; RescanReason::ALL.len()];
|
||||
for slot in 0..RescanReason::ALL.len() {
|
||||
counts[slot] = self.counts[slot].saturating_sub(earlier.counts[slot]);
|
||||
throttled_counts[slot] =
|
||||
self.throttled_counts[slot].saturating_sub(earlier.throttled_counts[slot]);
|
||||
}
|
||||
|
||||
RescanStats {
|
||||
total: self.total.saturating_sub(earlier.total),
|
||||
throttled: self.throttled.saturating_sub(earlier.throttled),
|
||||
counts,
|
||||
throttled_counts,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for RescanStats {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "{} rescan(s)", self.total)?;
|
||||
let mut first = true;
|
||||
for reason in RescanReason::ALL {
|
||||
let count = self.count(reason);
|
||||
if count == 0 {
|
||||
continue;
|
||||
}
|
||||
f.write_str(if first { " [" } else { ", " })?;
|
||||
write!(f, "{reason}={count}")?;
|
||||
first = false;
|
||||
}
|
||||
if !first {
|
||||
f.write_str("]")?;
|
||||
}
|
||||
if self.throttled > 0 {
|
||||
write!(f, ", {} throttled", self.throttled)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(rescan_stats)]
|
||||
#[derive(Default)]
|
||||
pub(crate) struct RescanCounters {
|
||||
counters: [AtomicUsize; RescanReason::ALL.len()],
|
||||
throttled: [AtomicUsize; RescanReason::ALL.len()],
|
||||
}
|
||||
|
||||
#[cfg(rescan_stats)]
|
||||
impl RescanCounters {
|
||||
pub(crate) fn record(&self, reason: RescanReason) {
|
||||
self.counters[reason.slot()].fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub(crate) fn record_throttled(&self, reason: RescanReason) {
|
||||
self.throttled[reason.slot()].fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub(crate) fn snapshot(&self) -> RescanStats {
|
||||
let mut stats = RescanStats::default();
|
||||
for reason in RescanReason::ALL {
|
||||
let count = self.counters[reason.slot()].load(Ordering::Relaxed);
|
||||
stats.counts[reason.slot()] = count;
|
||||
stats.total += count;
|
||||
|
||||
let throttled = self.throttled[reason.slot()].load(Ordering::Relaxed);
|
||||
stats.throttled_counts[reason.slot()] = throttled;
|
||||
stats.throttled += throttled;
|
||||
}
|
||||
stats
|
||||
}
|
||||
|
||||
pub(crate) fn reset(&self) {
|
||||
for counter in self.counters.iter().chain(self.throttled.iter()) {
|
||||
counter.store(0, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Release builds retain the API without counter storage.
|
||||
#[cfg(not(rescan_stats))]
|
||||
#[derive(Default)]
|
||||
pub(crate) struct RescanCounters;
|
||||
|
||||
#[cfg(not(rescan_stats))]
|
||||
impl RescanCounters {
|
||||
pub(crate) fn record(&self, _reason: RescanReason) {}
|
||||
|
||||
pub(crate) fn record_throttled(&self, _reason: RescanReason) {}
|
||||
|
||||
pub(crate) fn snapshot(&self) -> RescanStats {
|
||||
RescanStats::default()
|
||||
}
|
||||
|
||||
pub(crate) fn reset(&self) {}
|
||||
}
|
||||
|
||||
#[cfg(all(test, rescan_stats))]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn counters_attribute_and_diff_per_reason() {
|
||||
let counters = RescanCounters::default();
|
||||
counters.record(RescanReason::Explicit);
|
||||
let baseline = counters.snapshot();
|
||||
|
||||
counters.record(RescanReason::IgnoreFileChanged);
|
||||
counters.record(RescanReason::IgnoreFileChanged);
|
||||
counters.record(RescanReason::OverflowCapacity);
|
||||
|
||||
let stats = counters.snapshot();
|
||||
assert_eq!(stats.total, 4);
|
||||
assert_eq!(stats.watcher_triggered(), 3);
|
||||
|
||||
let delta = stats.since(&baseline);
|
||||
assert_eq!(delta.total, 3);
|
||||
assert_eq!(delta.count(RescanReason::Explicit), 0);
|
||||
assert_eq!(delta.count(RescanReason::IgnoreFileChanged), 2);
|
||||
assert_eq!(
|
||||
delta.to_string(),
|
||||
"3 rescan(s) [ignore_file_changed=2, overflow_capacity=1]"
|
||||
);
|
||||
|
||||
counters.reset();
|
||||
assert_eq!(counters.snapshot(), RescanStats::default());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::time::Instant;
|
||||
|
||||
use crate::constants::{
|
||||
LARGE_INDEX_FILE_COUNT, RESCAN_MIN_INTERVAL, RESCAN_MIN_INTERVAL_LARGE_INDEX,
|
||||
};
|
||||
|
||||
const NEVER: u64 = u64::MAX;
|
||||
|
||||
// Drops watcher rescan requests inside the cooldown after the last scan.
|
||||
// A slightly stale index is fine: the next admitted event rescans everything.
|
||||
pub(crate) struct RescanThrottle {
|
||||
epoch: Instant,
|
||||
last_admitted: AtomicU64,
|
||||
}
|
||||
|
||||
impl Default for RescanThrottle {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
epoch: Instant::now(),
|
||||
last_admitted: AtomicU64::new(NEVER),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl RescanThrottle {
|
||||
/// Returns `true` if a rescan may start now and records it as the last scan
|
||||
pub(crate) fn admit(&self, live_files: usize, has_git_repo: bool) -> bool {
|
||||
let min_interval = if !has_git_repo && live_files >= LARGE_INDEX_FILE_COUNT {
|
||||
RESCAN_MIN_INTERVAL_LARGE_INDEX
|
||||
} else {
|
||||
RESCAN_MIN_INTERVAL
|
||||
};
|
||||
|
||||
let min_ms = min_interval.as_millis() as u64;
|
||||
let now = self.elapsed_ms();
|
||||
|
||||
loop {
|
||||
let last = self.last_admitted.load(Ordering::Acquire);
|
||||
if last != NEVER && now.saturating_sub(last) < min_ms {
|
||||
return false;
|
||||
}
|
||||
// CAS so two concurrent requests cannot both start a walk.
|
||||
if self
|
||||
.last_admitted
|
||||
.compare_exchange(last, now, Ordering::AcqRel, Ordering::Acquire)
|
||||
.is_ok()
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Records an explicit (unthrottled) scan so watcher requests right after
|
||||
/// it are dropped: the index is already fresh.
|
||||
pub(crate) fn note_explicit_scan(&self) {
|
||||
self.last_admitted
|
||||
.store(self.elapsed_ms(), Ordering::Release);
|
||||
}
|
||||
|
||||
fn elapsed_ms(&self) -> u64 {
|
||||
self.epoch.elapsed().as_millis() as u64
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::time::Duration;
|
||||
|
||||
fn throttle_at(ms_ago: u64) -> RescanThrottle {
|
||||
let now = Instant::now();
|
||||
RescanThrottle {
|
||||
epoch: now
|
||||
.checked_sub(Duration::from_millis(ms_ago))
|
||||
.expect("monotonic clock older than the rewind"),
|
||||
last_admitted: AtomicU64::new(0),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn first_request_is_always_admitted() {
|
||||
let throttle = RescanThrottle::default();
|
||||
assert!(throttle.admit(100, true));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn requests_inside_the_cooldown_are_dropped() {
|
||||
let throttle = throttle_at(1_000);
|
||||
assert!(!throttle.admit(100, false));
|
||||
assert!(!throttle.admit(100, false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_large_index_outside_a_git_repo_uses_the_slower_cadence() {
|
||||
// A minute is past the normal cooldown but not the large-index one.
|
||||
let throttle = throttle_at(60_000);
|
||||
assert!(throttle.admit(100, false));
|
||||
|
||||
let throttle = throttle_at(60_000);
|
||||
assert!(!throttle.admit(LARGE_INDEX_FILE_COUNT, false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_git_repo_keeps_the_normal_cadence_at_any_size() {
|
||||
let throttle = throttle_at(60_000);
|
||||
assert!(throttle.admit(LARGE_INDEX_FILE_COUNT, true));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cooldown_expiry_admits_again() {
|
||||
let throttle = throttle_at(RESCAN_MIN_INTERVAL.as_millis() as u64 + 1);
|
||||
assert!(throttle.admit(100, false));
|
||||
// Admission rearms the cooldown.
|
||||
assert!(!throttle.admit(100, false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn explicit_scan_rearms_the_cooldown() {
|
||||
let throttle = RescanThrottle::default();
|
||||
throttle.note_explicit_scan();
|
||||
assert!(!throttle.admit(100, false));
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,8 @@ use crate::file_picker::FilePicker;
|
||||
use crate::frecency::FrecencyTracker;
|
||||
use crate::git::GitStatusCache;
|
||||
use crate::query_tracker::QueryTracker;
|
||||
use crate::rescan_stats::{RescanCounters, RescanReason, RescanStats};
|
||||
use crate::rescan_throttle::RescanThrottle;
|
||||
use crate::scan::ScanJob;
|
||||
use crate::watch::{WatchEvent, WatchId, WatchOptions, WatchRegistry};
|
||||
use git2::Repository;
|
||||
@@ -77,6 +79,8 @@ pub struct SharedPickerInner {
|
||||
/// Watch subscriptions live outside the picker lock so delivery and
|
||||
/// (un)subscribing never contend with searches.
|
||||
watchers: Arc<WatchRegistry>,
|
||||
rescans: RescanCounters,
|
||||
rescan_throttle: RescanThrottle,
|
||||
}
|
||||
|
||||
impl Default for SharedPickerInner {
|
||||
@@ -84,6 +88,8 @@ impl Default for SharedPickerInner {
|
||||
Self {
|
||||
picker: parking_lot::RwLock::new(None),
|
||||
watchers: Arc::new(WatchRegistry::default()),
|
||||
rescans: RescanCounters::default(),
|
||||
rescan_throttle: RescanThrottle::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -199,6 +205,39 @@ impl SharedFilePicker {
|
||||
/// Performs a safe async rescan. Guarantees only single active rescan per picker.
|
||||
/// If many rescans requested the last one guaranteed to be finished.
|
||||
pub fn trigger_full_rescan_async(&self, shared_frecency: &SharedFrecency) -> Result<(), Error> {
|
||||
self.trigger_full_rescan_with_reason(shared_frecency, RescanReason::Explicit)
|
||||
.map(|_| ())
|
||||
}
|
||||
|
||||
/// Returns admitted and throttled rescan requests by reason.
|
||||
/// Counters start at picker creation or the last reset.
|
||||
pub fn rescan_stats(&self) -> RescanStats {
|
||||
self.0.rescans.snapshot()
|
||||
}
|
||||
|
||||
pub fn reset_rescan_stats(&self) {
|
||||
self.0.rescans.reset();
|
||||
}
|
||||
|
||||
/// Returns `Ok(true)` when a rescan was started (or queued behind an
|
||||
/// active scan) and `Ok(false)` when the request was throttled — the
|
||||
/// caller must then fall back to incremental event processing.
|
||||
pub(crate) fn trigger_full_rescan_with_reason(
|
||||
&self,
|
||||
shared_frecency: &SharedFrecency,
|
||||
reason: RescanReason,
|
||||
) -> Result<bool, Error> {
|
||||
// for giant folders we have no other choice other than throttling rescans
|
||||
// if user is running application in millions of files with a ton of rescan events
|
||||
// we drop / throttle some of requests to avoid constant burst of IO
|
||||
if reason == RescanReason::Explicit {
|
||||
self.0.rescan_throttle.note_explicit_scan();
|
||||
} else if !self.check_rescan_throttle(reason) {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
self.0.rescans.record(reason);
|
||||
|
||||
match ScanJob::new_rescan(self, shared_frecency)? {
|
||||
Some(job) => {
|
||||
job.spawn();
|
||||
@@ -219,7 +258,27 @@ impl SharedFilePicker {
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
fn check_rescan_throttle(&self, reason: RescanReason) -> bool {
|
||||
let (live_files, has_git) = self
|
||||
.read()
|
||||
.ok()
|
||||
.and_then(|guard| {
|
||||
guard
|
||||
.as_ref()
|
||||
.map(|picker| (picker.live_file_count(), picker.has_git_repo()))
|
||||
})
|
||||
.unwrap_or((0, false));
|
||||
|
||||
if self.0.rescan_throttle.admit(live_files, has_git) {
|
||||
return true;
|
||||
}
|
||||
|
||||
self.0.rescans.record_throttled(reason);
|
||||
tracing::debug!(%reason, live_files, "Rescan throttled, skipping");
|
||||
false
|
||||
}
|
||||
|
||||
/// Subscribe to filesystem changes matching `pattern`.
|
||||
@@ -432,20 +491,31 @@ impl<T: LmdbStore> SharedDb<T> {
|
||||
|
||||
/// Drop the in-memory tracker and delete the on-disk database directory.
|
||||
///
|
||||
/// Acquires the write lock, ensuring all readers (including any active mmap
|
||||
/// access) are finished before the LMDB environment is closed and the files
|
||||
/// are removed.
|
||||
///
|
||||
/// Returns `Ok(Some(path))` with the deleted path, or `Ok(None)` if no tracker was initialized.
|
||||
pub fn destroy(&self) -> Result<Option<PathBuf>, Error> {
|
||||
let mut guard = self.write()?;
|
||||
let Some(tracker) = guard.take() else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
let closing_event = match tracker.shared_env().destroy() {
|
||||
Ok(closing) => closing,
|
||||
Err(e) => {
|
||||
*guard = Some(tracker);
|
||||
return Err(e);
|
||||
}
|
||||
};
|
||||
|
||||
let db_path = tracker.env().path().to_path_buf();
|
||||
// Drop closes the LMDB env and unmaps the files
|
||||
drop(tracker);
|
||||
drop(guard);
|
||||
|
||||
// Deleting before mdb_env_close finishes would race the unmap.
|
||||
if let Some(event) = closing_event {
|
||||
event.wait_timeout(Duration::from_secs(5));
|
||||
}
|
||||
|
||||
std::fs::remove_dir_all(&db_path).map_err(|source| Error::RemoveDbDir {
|
||||
path: db_path.clone(),
|
||||
source,
|
||||
|
||||
@@ -415,7 +415,10 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_chunked_string_full_path() {
|
||||
let (store, strings, _files) = build_test_store(&["src/components/Button.tsx"]);
|
||||
let (store, strings, _files) = build_test_store(&[
|
||||
"src/components/Button.tsx",
|
||||
"src/components/Button.test.tsx",
|
||||
]);
|
||||
let arena = store.as_arena_ptr();
|
||||
let cs = &strings[0];
|
||||
|
||||
@@ -423,6 +426,15 @@ mod tests {
|
||||
assert_eq!(cs.read_to_buf(arena, &mut buf), "src/components/Button.tsx");
|
||||
assert_eq!(cs.byte_len, 25);
|
||||
assert_eq!(cs.filename_offset, 15);
|
||||
|
||||
let cs = &strings[1];
|
||||
let mut buf = [0u8; 512];
|
||||
assert_eq!(
|
||||
cs.read_to_buf(arena, &mut buf),
|
||||
"src/components/Button.test.tsx"
|
||||
);
|
||||
assert_eq!(cs.byte_len, 30);
|
||||
assert_eq!(cs.filename_offset, 15);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -20,6 +20,8 @@ pub(crate) use ripgrep::walk_collect_files;
|
||||
|
||||
pub(crate) struct WalkOutput {
|
||||
pub(crate) pairs: Vec<(FileItem, String)>,
|
||||
/// Every non-ignored directory the walk visited, relative, ending with /
|
||||
pub(crate) dirs: Vec<String>,
|
||||
pub(crate) ignore_rules: Option<WalkIgnoreRules>,
|
||||
}
|
||||
|
||||
|
||||
@@ -34,9 +34,12 @@ pub(crate) fn walk_collect_files(
|
||||
|
||||
let walker = walk_builder.build_parallel();
|
||||
|
||||
let pairs = parking_lot::Mutex::new(Vec::<(FileItem, String)>::new());
|
||||
// Single lock for both collections: every entry is either a file or a
|
||||
// dir, so this keeps one mutex acquisition per entry.
|
||||
let collected =
|
||||
parking_lot::Mutex::new((Vec::<(FileItem, String)>::new(), Vec::<String>::new()));
|
||||
walker.run(|| {
|
||||
let pairs = &pairs;
|
||||
let collected = &collected;
|
||||
let counter = Arc::clone(synced_files_count);
|
||||
let base_path = base_path.to_path_buf();
|
||||
|
||||
@@ -58,15 +61,27 @@ pub(crate) fn walk_collect_files(
|
||||
let (file_item, rel_path) =
|
||||
FileItem::new_from_walk(path, &base_path, None, metadata.as_ref());
|
||||
|
||||
pairs.lock().push((file_item, rel_path));
|
||||
collected.lock().0.push((file_item, rel_path));
|
||||
counter.fetch_add(1, Ordering::Relaxed);
|
||||
} else if entry.depth() > 0 && entry.file_type().is_some_and(|ft| ft.is_dir()) {
|
||||
let path = entry.path();
|
||||
if !is_git_file(path)
|
||||
&& let Ok(rel) = path.strip_prefix(&base_path)
|
||||
{
|
||||
let mut rel = crate::path_utils::to_canonical_slashes(&rel.to_string_lossy())
|
||||
.into_owned();
|
||||
rel.push('/');
|
||||
collected.lock().1.push(rel);
|
||||
}
|
||||
}
|
||||
ignore::WalkState::Continue
|
||||
})
|
||||
});
|
||||
|
||||
let (pairs, dirs) = collected.into_inner();
|
||||
Ok(WalkOutput {
|
||||
pairs: pairs.into_inner(),
|
||||
pairs,
|
||||
dirs,
|
||||
ignore_rules: None,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1,10 +1,8 @@
|
||||
//! Filesystem traversal backed by zlob's native parallel walker.
|
||||
//! Active when the `zlob` feature is enabled (requires the Zig toolchain).
|
||||
|
||||
use crate::file_picker::is_known_binary_extension_basename;
|
||||
use crate::ignore::IGNORED_DIRS;
|
||||
use crate::types::FileItem;
|
||||
use crate::walk::{WalkIgnoreRules, WalkOutput};
|
||||
use parking_lot::Mutex;
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
@@ -48,12 +46,25 @@ pub(crate) fn walk_collect_files(
|
||||
tracing::warn!(?e, "zlob extra_ignore rejected; walking without it");
|
||||
}
|
||||
|
||||
let pairs = parking_lot::Mutex::new(Vec::<(FileItem, String)>::new());
|
||||
// Single lock for both collections: every entry is either a file or a
|
||||
// dir, so this keeps one mutex acquisition per entry.
|
||||
let collected = Mutex::new((Vec::new(), Vec::new()));
|
||||
|
||||
let outcome = match builder.run(|entry| {
|
||||
if !entry.is_file() {
|
||||
// unlike ripgrep walker zlob doesnt show .git files
|
||||
if entry.is_dir() {
|
||||
let rel_bytes = entry.relative_path_bytes();
|
||||
if !rel_bytes.is_empty() {
|
||||
let mut rel = String::from_utf8_lossy(rel_bytes).into_owned();
|
||||
rel.push('/');
|
||||
collected.lock().1.push(rel);
|
||||
}
|
||||
}
|
||||
|
||||
return WalkState::Continue;
|
||||
}
|
||||
|
||||
let rel_bytes = entry.relative_path_bytes();
|
||||
|
||||
// `basename()` returns `&str` for files only.
|
||||
@@ -73,9 +84,9 @@ pub(crate) fn walk_collect_files(
|
||||
let rel_str = String::from_utf8_lossy(rel_bytes).into_owned();
|
||||
let item = FileItem::new_raw(basename_offset, size, modified, None, is_binary);
|
||||
|
||||
let mut guard = pairs.lock();
|
||||
guard.push((item, rel_str));
|
||||
let n = guard.len();
|
||||
let mut guard = collected.lock();
|
||||
guard.0.push((item, rel_str));
|
||||
let n = guard.0.len();
|
||||
drop(guard);
|
||||
|
||||
if n % PROGRESS_STEP == 0 {
|
||||
@@ -93,7 +104,7 @@ pub(crate) fn walk_collect_files(
|
||||
}
|
||||
};
|
||||
|
||||
let pairs = pairs.into_inner();
|
||||
let (pairs, dirs) = collected.into_inner();
|
||||
// Always report the exact final total regardless of the last step.
|
||||
synced_files_count.store(pairs.len(), Ordering::Relaxed);
|
||||
|
||||
@@ -106,6 +117,7 @@ pub(crate) fn walk_collect_files(
|
||||
|
||||
Ok(WalkOutput {
|
||||
pairs,
|
||||
dirs,
|
||||
ignore_rules,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ use crate::constants::MAX_OVERFLOW_FILES;
|
||||
use crate::error::Error;
|
||||
use crate::file_picker::FFFMode;
|
||||
use crate::git_status_worker::GitStatusWorker;
|
||||
use crate::rescan_stats::RescanReason;
|
||||
use crate::shared::{SharedFilePicker, SharedFrecency};
|
||||
use crate::sort_buffer::sort_with_buffer;
|
||||
use crate::watch::{RawWatchEvent, WatchEventKind};
|
||||
@@ -22,10 +23,19 @@ type Debouncer = notify_debouncer_full::Debouncer<notify::RecommendedWatcher, No
|
||||
/// are fully joined before `stop()` / `Drop` returns.
|
||||
pub struct BackgroundWatcher {
|
||||
debouncer: Arc<Mutex<Option<Debouncer>>>,
|
||||
watch_tx: Option<mpsc::Sender<PathBuf>>,
|
||||
watch_tx: Option<mpsc::Sender<WatchTask>>,
|
||||
owner_thread: Option<std::thread::JoinHandle<()>>,
|
||||
}
|
||||
|
||||
enum WatchTask {
|
||||
/// Only subscribe to a specific path, this is happening when we did rescun and have to update
|
||||
/// the watcher only
|
||||
Subscribe(PathBuf),
|
||||
/// This is requires a separate walk of the new directory copies or created within a scan
|
||||
/// window because it might contain subdirectories we have to walk, prune, and add to index
|
||||
IndexNewDir(PathBuf),
|
||||
}
|
||||
|
||||
const DEBOUNCE_TIMEOUT: Duration = Duration::from_millis(50);
|
||||
/// Minimum seconds between frecency tracks of the same file in AI mode.
|
||||
/// Prevents score inflation from rapid burst edits by AI agents.
|
||||
@@ -77,7 +87,7 @@ impl BackgroundWatcher {
|
||||
// spare watcher (configurable by the user, usually 100k - 1m)
|
||||
let use_recursive = cfg!(any(target_os = "macos", target_os = "windows"));
|
||||
|
||||
let (watch_tx, watch_rx) = mpsc::channel::<PathBuf>();
|
||||
let (watch_tx, watch_rx) = mpsc::channel::<WatchTask>();
|
||||
let watch_tx_for_debouncer = watch_tx.clone();
|
||||
|
||||
let owner_weak_picker = shared_picker.weaken();
|
||||
@@ -108,48 +118,46 @@ impl BackgroundWatcher {
|
||||
.name("fff-watcher-own".into())
|
||||
.spawn(move || {
|
||||
let _g = owner_span.enter();
|
||||
while let Ok(dir) = watch_rx.recv() {
|
||||
while let Ok(task) = watch_rx.recv() {
|
||||
// if the picker is dropped we do need to exit the loop
|
||||
let Some(strong_picker) = owner_weak_picker.upgrade() else {
|
||||
break;
|
||||
};
|
||||
|
||||
// Only inotify (Linux) has no kernel-level recursion, so
|
||||
// it's the only platform that needs a per-subdir watch to
|
||||
// be registered at runtime. macOS FSEvents and Windows
|
||||
// ReadDirectoryChangesW are already watching recursively
|
||||
// from the base path (see `create_debouncer`), and
|
||||
// registering a second overlapping stream there produces
|
||||
// duplicate/out-of-order events.
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
// Register the new directory with the debouncer, then
|
||||
// drop the mutex BEFORE doing picker-side work — see
|
||||
// the comment on `BackgroundWatcher::stop` for the
|
||||
// lock-ordering rationale.
|
||||
let mut guard = owner_debouncer.lock();
|
||||
let Some(debouncer) = guard.as_mut() else {
|
||||
break;
|
||||
};
|
||||
let (dir, is_new_dir) = match task {
|
||||
WatchTask::Subscribe(dir) => (dir, false),
|
||||
WatchTask::IndexNewDir(dir) => (dir, true),
|
||||
};
|
||||
|
||||
if let Err(e) = debouncer.watch(&dir, RecursiveMode::NonRecursive) {
|
||||
warn!(
|
||||
?e,
|
||||
dir = %dir.display(),
|
||||
"Failed to init watcher for new directory"
|
||||
);
|
||||
}
|
||||
// Register the watch BEFORE walking so files created mid-walk still handled
|
||||
#[cfg(target_os = "linux")]
|
||||
if !watch_dirs_nonrecursive(&owner_debouncer, std::iter::once(dir.as_path())) {
|
||||
break;
|
||||
}
|
||||
|
||||
track_files_from_new_directories(
|
||||
&dir,
|
||||
&strong_picker,
|
||||
&owner_git_workdir,
|
||||
&owner_git_worker,
|
||||
);
|
||||
if is_new_dir {
|
||||
// need to call this on every platform to add subdirectories from the
|
||||
// new folders to the picker, but on linux we have to handle the subdirs
|
||||
let subdirs = index_new_directory(
|
||||
&dir,
|
||||
&strong_picker,
|
||||
&owner_git_workdir,
|
||||
&owner_git_worker,
|
||||
);
|
||||
|
||||
// Transient strong ref drops here, back
|
||||
// to weak-only before the next `recv()`.
|
||||
// on linux we manually resubscribe for new inodes
|
||||
#[cfg(target_os = "linux")]
|
||||
if !watch_dirs_nonrecursive(
|
||||
&owner_debouncer,
|
||||
subdirs.iter().map(|p| p.as_path()),
|
||||
) {
|
||||
break;
|
||||
}
|
||||
|
||||
drop(subdirs); // need it cause subdirs is unused on non-linux target'
|
||||
}
|
||||
|
||||
drop(strong_picker);
|
||||
}
|
||||
|
||||
tracing::info!("Background watcher is stopped");
|
||||
@@ -171,7 +179,7 @@ impl BackgroundWatcher {
|
||||
shared_frecency: SharedFrecency,
|
||||
mode: FFFMode,
|
||||
use_recursive: bool,
|
||||
watch_tx: mpsc::Sender<PathBuf>,
|
||||
watch_tx: mpsc::Sender<WatchTask>,
|
||||
git_status_worker: Arc<GitStatusWorker>,
|
||||
) -> Result<Debouncer, Error> {
|
||||
let config = Config::default()
|
||||
@@ -206,7 +214,7 @@ impl BackgroundWatcher {
|
||||
|
||||
// every new directory created has to be reflected in the picker state
|
||||
for dir in new_dirs {
|
||||
if let Err(e) = watch_tx.send(dir) {
|
||||
if let Err(e) = watch_tx.send(WatchTask::IndexNewDir(dir)) {
|
||||
error!(?e, "Failed to send directory update error");
|
||||
}
|
||||
}
|
||||
@@ -304,7 +312,7 @@ impl BackgroundWatcher {
|
||||
|
||||
pub(crate) fn request_watch_dir(&self, dir: PathBuf) -> bool {
|
||||
match self.watch_tx.as_ref() {
|
||||
Some(tx) => tx.send(dir).is_ok(),
|
||||
Some(tx) => tx.send(WatchTask::Subscribe(dir)).is_ok(),
|
||||
None => false,
|
||||
}
|
||||
}
|
||||
@@ -317,7 +325,7 @@ impl Drop for BackgroundWatcher {
|
||||
}
|
||||
|
||||
#[tracing::instrument(name = "fs_events", skip(events, shared_picker, shared_frecency, git_status_worker), level = Level::DEBUG)]
|
||||
fn handle_debounced_events(
|
||||
pub(crate) fn handle_debounced_events(
|
||||
mode: FFFMode,
|
||||
events: Vec<DebouncedEvent>,
|
||||
base_path: &Path,
|
||||
@@ -335,8 +343,8 @@ fn handle_debounced_events(
|
||||
.ok()
|
||||
.and_then(|g| g.as_ref().and_then(|p| p.ignore_rules()));
|
||||
let filter = IgnoreFilter::new(base_path, walker_rules, repo.as_ref());
|
||||
let mut need_full_rescan = false;
|
||||
let mut need_full_git_rescan = false;
|
||||
let mut batch_overflow_attempted = false;
|
||||
let mut paths_to_remove = Vec::new();
|
||||
let mut dirs_to_remove: Vec<PathBuf> = Vec::new();
|
||||
let mut paths_to_add_or_modify = Vec::new();
|
||||
@@ -346,6 +354,21 @@ fn handle_debounced_events(
|
||||
let watch_registry = shared_picker.watch_registry();
|
||||
let need_events_propagation = watch_registry.is_active();
|
||||
|
||||
let try_trigger_full_rescan = |reason: RescanReason| -> bool {
|
||||
match shared_picker.trigger_full_rescan_with_reason(shared_frecency, reason) {
|
||||
Ok(true) => {
|
||||
warn!(%reason, "Triggering full rescan");
|
||||
watch_registry.dispatch_rescan(base_path);
|
||||
true
|
||||
}
|
||||
Ok(false) => false,
|
||||
Err(e) => {
|
||||
error!(%reason, "Failed to trigger full rescan: {:?}", e);
|
||||
false
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
for debounced_event in &events {
|
||||
// It is very important to not react to the access errors because we inevitably
|
||||
// gonna trigger the sync by our own preview or other unnecessary noise
|
||||
@@ -363,22 +386,19 @@ fn handle_debounced_events(
|
||||
// When macOS FSEvents (or other backends) overflow their event buffer, the kernel
|
||||
// drops individual events and emits a rescan flag telling us to re-scan the subtree
|
||||
if debounced_event.event.need_rescan() {
|
||||
if debounced_event.event.paths.len() < 16 // this should be usually one event
|
||||
let small_and_known = debounced_event.event.paths.len() < 16 // this should be usually one event
|
||||
&& debounced_event
|
||||
.paths
|
||||
.iter()
|
||||
// but we are smart enough and not falling into the paths
|
||||
.all(|p| !p.is_dir() && !filter.is_ignored(p))
|
||||
{
|
||||
break;
|
||||
.all(|p| !p.is_dir() && !filter.is_ignored(p));
|
||||
|
||||
if !small_and_known && try_trigger_full_rescan(RescanReason::KernelEventLoss) {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
warn!(
|
||||
"Received rescan event for paths {:?}, triggering full rescan",
|
||||
debounced_event.event.paths
|
||||
);
|
||||
need_full_rescan = true;
|
||||
break;
|
||||
// Small batches and throttled rescans fall through: the listed
|
||||
// paths are still applied incrementally below.
|
||||
}
|
||||
|
||||
tracing::debug!(event = ?debounced_event.event, "Processing FS event");
|
||||
@@ -387,13 +407,24 @@ fn handle_debounced_events(
|
||||
path.file_name().and_then(|f| f.to_str()),
|
||||
Some(".ignore") | Some(".gitignore")
|
||||
) {
|
||||
if path
|
||||
.parent()
|
||||
.is_some_and(|parent| filter.is_ignored(parent))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
info!(
|
||||
"Detected change in ignore definition file: {}",
|
||||
path.display()
|
||||
);
|
||||
|
||||
need_full_rescan = true;
|
||||
break;
|
||||
if try_trigger_full_rescan(RescanReason::IgnoreFileChanged) {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
// Throttled: fall through so the ignore file itself stays
|
||||
// indexed; the stale rules heal on the next admitted rescan.
|
||||
}
|
||||
|
||||
if is_dotgit_change_affecting_status(path, &repo) {
|
||||
@@ -455,29 +486,18 @@ fn handle_debounced_events(
|
||||
}
|
||||
|
||||
affected_paths_count += debounced_event.event.paths.len();
|
||||
if affected_paths_count > MAX_OVERFLOW_FILES {
|
||||
if !batch_overflow_attempted && affected_paths_count > MAX_OVERFLOW_FILES * 4 {
|
||||
batch_overflow_attempted = true;
|
||||
warn!(
|
||||
?affected_paths_count,
|
||||
max = MAX_OVERFLOW_FILES,
|
||||
max = MAX_OVERFLOW_FILES * 4,
|
||||
"Too many affected paths in a single batch, triggering full rescan",
|
||||
);
|
||||
|
||||
need_full_rescan = true;
|
||||
break;
|
||||
if try_trigger_full_rescan(RescanReason::EventBatchOverflow) {
|
||||
return Vec::new();
|
||||
}
|
||||
}
|
||||
|
||||
if need_full_rescan {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if need_full_rescan {
|
||||
info!(?affected_paths_count, "Triggering full rescan");
|
||||
watch_registry.dispatch_rescan(base_path);
|
||||
if let Err(e) = shared_picker.trigger_full_rescan_async(shared_frecency) {
|
||||
error!("Failed to trigger full rescan: {:?}", e);
|
||||
}
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
// It's important to get the allocated sort
|
||||
@@ -504,7 +524,7 @@ fn handle_debounced_events(
|
||||
}
|
||||
|
||||
let mut files_to_update_git_status = Vec::new();
|
||||
let mut need_full_rescan = false;
|
||||
let mut index_update_rejected = false;
|
||||
let mut overflow_count = 0;
|
||||
let mut removed_from_dirs = Vec::new();
|
||||
let mut watch_events = ahash::AHashMap::new();
|
||||
@@ -565,6 +585,13 @@ fn handle_debounced_events(
|
||||
|
||||
files_to_update_git_status.reserve(paths_to_add_or_modify.len());
|
||||
for path in &paths_to_add_or_modify {
|
||||
if picker.get_overflow_files().len() >= MAX_OVERFLOW_FILES
|
||||
&& picker.get_file_by_path(path).is_none()
|
||||
{
|
||||
index_update_rejected = true;
|
||||
break;
|
||||
}
|
||||
|
||||
let existed = need_events_propagation && picker.get_file_by_path(path).is_some();
|
||||
|
||||
if picker.handle_create_or_modify(path).is_some() {
|
||||
@@ -579,7 +606,7 @@ fn handle_debounced_events(
|
||||
watch_events.insert(path.to_path_buf(), kind);
|
||||
}
|
||||
} else {
|
||||
need_full_rescan = true;
|
||||
index_update_rejected = true;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -591,13 +618,22 @@ fn handle_debounced_events(
|
||||
overflow_count, "File index changes applied",
|
||||
);
|
||||
|
||||
if need_full_rescan || overflow_count > MAX_OVERFLOW_FILES {
|
||||
info!("Watcher faced limit of index overflow. Triggering rescan");
|
||||
watch_registry.dispatch_rescan(base_path);
|
||||
if let Err(e) = shared_picker.trigger_full_rescan_async(shared_frecency) {
|
||||
error!("Failed to trigger full rescan: {:?}", e);
|
||||
}
|
||||
} else if need_events_propagation {
|
||||
let rescan_started = if index_update_rejected || overflow_count > MAX_OVERFLOW_FILES {
|
||||
let reason = if index_update_rejected {
|
||||
RescanReason::IndexUpdateRejected
|
||||
} else {
|
||||
RescanReason::OverflowCapacity
|
||||
};
|
||||
|
||||
info!(%reason, "Watcher faced limit of index overflow. Triggering rescan");
|
||||
try_trigger_full_rescan(reason)
|
||||
} else {
|
||||
false
|
||||
};
|
||||
|
||||
// When the rescan is throttled the incrementally applied changes are
|
||||
// still the freshest state we have — propagate them to subscribers.
|
||||
if !rescan_started && need_events_propagation {
|
||||
watch_registry.dispatch(
|
||||
base_path,
|
||||
watch_events
|
||||
@@ -657,7 +693,7 @@ fn handle_debounced_events(
|
||||
|
||||
// do not try to update the paths if we anyway going to rescan everything from scratch
|
||||
// no repo => no consumer thread, so don't accumulate paths nobody will drain
|
||||
if !need_full_rescan && repo.is_some() {
|
||||
if !index_update_rejected && repo.is_some() {
|
||||
if need_full_git_rescan {
|
||||
// A full git rescan re-reads every tracked path (including ones that just
|
||||
// went clean after a commit), so it already subsumes the per-path update.
|
||||
@@ -670,58 +706,78 @@ fn handle_debounced_events(
|
||||
new_dirs_to_watch
|
||||
}
|
||||
|
||||
/// After registering a watch on a newly created directory, list its
|
||||
/// immediate children and add any files to the picker.
|
||||
fn track_files_from_new_directories(
|
||||
fn index_new_directory(
|
||||
dir: &Path,
|
||||
shared_picker: &SharedFilePicker,
|
||||
git_workdir: &Option<PathBuf>,
|
||||
git_status_worker: &Arc<GitStatusWorker>,
|
||||
) {
|
||||
let Ok(entries) = std::fs::read_dir(dir) else {
|
||||
return;
|
||||
};
|
||||
|
||||
) -> Vec<PathBuf> {
|
||||
let repo = git_workdir.as_ref().and_then(|p| Repository::open(p).ok());
|
||||
// Prefer the walker's ignore rules; read base_path + rules from the picker.
|
||||
let (base_path, walker_rules) = match shared_picker.read().ok().and_then(|g| {
|
||||
g.as_ref()
|
||||
.map(|p| (p.base_path().to_path_buf(), p.ignore_rules()))
|
||||
let (base_path, walker_rules, follow_symlinks) = match shared_picker.read().ok().and_then(|g| {
|
||||
g.as_ref().map(|p| {
|
||||
(
|
||||
p.base_path().to_path_buf(),
|
||||
p.ignore_rules(),
|
||||
p.follows_symlinks(),
|
||||
)
|
||||
})
|
||||
}) {
|
||||
Some(pair) => pair,
|
||||
None => return,
|
||||
Some(triple) => triple,
|
||||
None => return Vec::new(),
|
||||
};
|
||||
|
||||
let filter = IgnoreFilter::new(&base_path, walker_rules, repo.as_ref());
|
||||
let mut files_to_add = Vec::new();
|
||||
|
||||
for entry in entries.flatten() {
|
||||
if entry.file_type().is_ok_and(|ft| ft.is_file()) {
|
||||
let path = entry.path();
|
||||
// file_type() already ruled out directories — only ignore rules left
|
||||
if !filter.is_ignored(&path) {
|
||||
files_to_add.push(path);
|
||||
}
|
||||
let walk = match crate::walk::walk_collect_files(
|
||||
dir,
|
||||
repo.is_some(),
|
||||
follow_symlinks,
|
||||
1,
|
||||
&Arc::new(std::sync::atomic::AtomicUsize::new(0)),
|
||||
) {
|
||||
Ok(walk) => walk,
|
||||
Err(e) => {
|
||||
warn!(?e, dir = %dir.display(), "Failed to walk new directory");
|
||||
return Vec::new();
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// TODO: figure out a better optimized way for zlob to rerun the directory walk using existing
|
||||
// ignore rules, but currently we have to filter out ignored files on our own
|
||||
let filter = IgnoreFilter::new(&base_path, walker_rules, repo.as_ref());
|
||||
let join_unless_ignored = |relative_path: &str| -> Option<PathBuf> {
|
||||
let path = dir.join(relative_path);
|
||||
(!filter.is_ignored(&path)).then_some(path)
|
||||
};
|
||||
|
||||
let files_to_add: Vec<PathBuf> = walk
|
||||
.pairs
|
||||
.iter()
|
||||
.filter_map(|(_, path)| join_unless_ignored(path))
|
||||
.collect();
|
||||
|
||||
let subdirs: Vec<PathBuf> = walk
|
||||
.dirs
|
||||
.iter()
|
||||
.filter_map(|path| join_unless_ignored(path.trim_end_matches('/')))
|
||||
.collect();
|
||||
|
||||
if files_to_add.is_empty() {
|
||||
return;
|
||||
return subdirs;
|
||||
}
|
||||
|
||||
let mut indexed_files = Vec::with_capacity(files_to_add.len());
|
||||
{
|
||||
let Ok(mut guard) = shared_picker.write() else {
|
||||
return;
|
||||
return subdirs;
|
||||
};
|
||||
|
||||
let Some(ref mut picker) = *guard else {
|
||||
return;
|
||||
return subdirs;
|
||||
};
|
||||
|
||||
for path in &files_to_add {
|
||||
if picker.handle_create_or_modify(path).is_some() {
|
||||
indexed_files.push(path.clone());
|
||||
for path in files_to_add {
|
||||
if picker.handle_create_or_modify(&path).is_some() {
|
||||
indexed_files.push(path);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -746,10 +802,35 @@ fn track_files_from_new_directories(
|
||||
}
|
||||
|
||||
debug!(
|
||||
"Injected {} existing files from new directory {}",
|
||||
"Indexed new {} files from new directory {}",
|
||||
added,
|
||||
dir.display(),
|
||||
);
|
||||
|
||||
subdirs
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn watch_dirs_nonrecursive<'a>(
|
||||
debouncer: &Mutex<Option<Debouncer>>,
|
||||
dirs: impl Iterator<Item = &'a Path>,
|
||||
) -> bool {
|
||||
let mut guard = debouncer.lock();
|
||||
let Some(debouncer) = guard.as_mut() else {
|
||||
return false;
|
||||
};
|
||||
|
||||
for dir in dirs {
|
||||
if let Err(e) = debouncer.watch(dir, RecursiveMode::NonRecursive) {
|
||||
warn!(
|
||||
?e,
|
||||
dir = %dir.display(),
|
||||
"Failed to init watcher for new directory"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
true
|
||||
}
|
||||
|
||||
struct IgnoreFilter<'a> {
|
||||
@@ -777,13 +858,13 @@ impl<'a> IgnoreFilter<'a> {
|
||||
/// Whether `path` (absolute) is ignored.
|
||||
fn is_ignored(&self, path: &Path) -> bool {
|
||||
if let Some(rules) = self.rules.as_ref() {
|
||||
let Ok(rel) = path.strip_prefix(self.base_path) else {
|
||||
let Ok(relative) = path.strip_prefix(self.base_path) else {
|
||||
return false;
|
||||
};
|
||||
// `IgnoreRules::is_ignored` enumerates every ancestor .gitignore
|
||||
// layer internally, so a leaf under an ignored directory (rule
|
||||
// `build/`, path `build/out.rs`) is caught in one call.
|
||||
return rules.is_ignored(rel);
|
||||
return rules.is_ignored(relative);
|
||||
}
|
||||
match self.repo {
|
||||
Some(repo) => repo.is_path_ignored(path) == Ok(true),
|
||||
|
||||
@@ -3,3 +3,7 @@ pub use background_watcher::*;
|
||||
|
||||
mod watch;
|
||||
pub use watch::*;
|
||||
|
||||
// The harness reads rescan counters, which release builds compile out.
|
||||
#[cfg(all(test, rescan_stats))]
|
||||
mod rescan_tests;
|
||||
|
||||
@@ -0,0 +1,621 @@
|
||||
use std::path::PathBuf;
|
||||
use std::process::Command;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use notify::Event;
|
||||
use notify::EventKind;
|
||||
use notify::event::{
|
||||
AccessKind, AccessMode, CreateKind, DataChange, Flag, ModifyKind, RemoveKind, RenameMode,
|
||||
};
|
||||
use notify_debouncer_full::DebouncedEvent;
|
||||
use tempfile::TempDir;
|
||||
|
||||
use super::handle_debounced_events;
|
||||
use crate::constants::MAX_OVERFLOW_FILES;
|
||||
use crate::file_picker::{FFFMode, FilePicker, FilePickerOptions};
|
||||
use crate::git_status_worker::GitStatusWorker;
|
||||
use crate::rescan_stats::{RescanReason, RescanStats};
|
||||
use crate::shared::{SharedFilePicker, SharedFrecency};
|
||||
|
||||
#[test]
|
||||
fn saving_an_indexed_file_stays_incremental() {
|
||||
let f = Fixture::new();
|
||||
f.write("src/main.rs", "fn main() {}");
|
||||
f.index();
|
||||
|
||||
f.write("src/main.rs", "fn main() { println!(); }");
|
||||
let delta = f.feed([modify(f.path("src/main.rs"))]);
|
||||
|
||||
f.assert_no_rescan(&delta, "saving a tracked file");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn editor_atomic_save_stays_incremental() {
|
||||
let f = Fixture::new();
|
||||
f.write("src/main.rs", "fn main() {}");
|
||||
f.index();
|
||||
|
||||
// write-to-temp + rename-over-target, the way vim/VSCode/IntelliJ save.
|
||||
f.write("src/main.rs", "fn main() { println!(); }");
|
||||
let target = f.path("src/main.rs");
|
||||
let temp = f.path("src/.main.rs.swp");
|
||||
let delta = f.feed([
|
||||
DebouncedEvent::new(
|
||||
Event::new(EventKind::Create(CreateKind::File)).add_path(temp.clone()),
|
||||
Instant::now(),
|
||||
),
|
||||
DebouncedEvent::new(
|
||||
Event::new(EventKind::Modify(ModifyKind::Name(RenameMode::From)))
|
||||
.add_path(temp.clone()),
|
||||
Instant::now(),
|
||||
),
|
||||
DebouncedEvent::new(
|
||||
Event::new(EventKind::Modify(ModifyKind::Name(RenameMode::To)))
|
||||
.add_path(target.clone()),
|
||||
Instant::now(),
|
||||
),
|
||||
DebouncedEvent::new(
|
||||
Event::new(EventKind::Remove(RemoveKind::File)).add_path(temp),
|
||||
Instant::now(),
|
||||
),
|
||||
]);
|
||||
|
||||
f.assert_no_rescan(&delta, "an atomic editor save");
|
||||
assert!(f.is_indexed("src/main.rs"), "target must stay indexed");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn creating_and_deleting_files_stays_incremental() {
|
||||
let f = Fixture::new();
|
||||
f.write("src/main.rs", "fn main() {}");
|
||||
f.index();
|
||||
|
||||
f.write("src/added.rs", "pub fn added() {}");
|
||||
let created = f.feed([create(f.path("src/added.rs"))]);
|
||||
f.assert_no_rescan(&created, "creating a file");
|
||||
assert!(f.is_indexed("src/added.rs"));
|
||||
|
||||
f.remove("src/added.rs");
|
||||
let removed = f.feed([remove_file(f.path("src/added.rs"))]);
|
||||
f.assert_no_rescan(&removed, "deleting a file");
|
||||
assert!(!f.is_indexed("src/added.rs"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deleting_a_directory_stays_incremental() {
|
||||
let f = Fixture::new();
|
||||
f.write("src/main.rs", "fn main() {}");
|
||||
f.write("src/nested/a.rs", "");
|
||||
f.write("src/nested/b.rs", "");
|
||||
f.index();
|
||||
|
||||
std::fs::remove_dir_all(f.path("src/nested")).unwrap();
|
||||
let delta = f.feed([DebouncedEvent::new(
|
||||
Event::new(EventKind::Remove(RemoveKind::Folder)).add_path(f.path("src/nested")),
|
||||
Instant::now(),
|
||||
)]);
|
||||
|
||||
f.assert_no_rescan(&delta, "deleting a directory");
|
||||
assert!(!f.is_indexed("src/nested/a.rs"));
|
||||
assert!(f.is_indexed("src/main.rs"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn read_only_access_events_are_ignored() {
|
||||
let f = Fixture::new();
|
||||
f.write("src/main.rs", "fn main() {}");
|
||||
f.index();
|
||||
|
||||
// fff's own preview + grep reads generate these; reacting to them would
|
||||
// make the picker rescan whenever the user scrolls the result list.
|
||||
let path = f.path("src/main.rs");
|
||||
let delta = f.feed([
|
||||
DebouncedEvent::new(
|
||||
Event::new(EventKind::Access(AccessKind::Read)).add_path(path.clone()),
|
||||
Instant::now(),
|
||||
),
|
||||
DebouncedEvent::new(
|
||||
Event::new(EventKind::Access(AccessKind::Open(AccessMode::Read)))
|
||||
.add_path(path.clone()),
|
||||
Instant::now(),
|
||||
),
|
||||
DebouncedEvent::new(
|
||||
Event::new(EventKind::Access(AccessKind::Close(AccessMode::Read))).add_path(path),
|
||||
Instant::now(),
|
||||
),
|
||||
]);
|
||||
|
||||
f.assert_no_rescan(&delta, "read-only access events");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recreating_the_same_paths_does_not_consume_overflow_capacity() {
|
||||
let f = Fixture::new();
|
||||
f.write("src/main.rs", "fn main() {}");
|
||||
f.index();
|
||||
|
||||
// Recreated paths must reuse their overflow slots.
|
||||
for _ in 0..8 {
|
||||
for i in 0..200 {
|
||||
let rel = format!("gen/out{i}.rs");
|
||||
f.write(&rel, "generated");
|
||||
f.feed([create(f.path(&rel))]);
|
||||
}
|
||||
for i in 0..200 {
|
||||
let rel = format!("gen/out{i}.rs");
|
||||
f.remove(&rel);
|
||||
f.feed([remove_file(f.path(&rel))]);
|
||||
}
|
||||
}
|
||||
|
||||
let delta = f.all_rescans();
|
||||
f.assert_no_rescan(&delta, "1600 create/delete cycles over 200 stable paths");
|
||||
assert!(
|
||||
f.overflow_len() <= 200,
|
||||
"each path must claim one overflow slot at most, got {}",
|
||||
f.overflow_len()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn writes_inside_a_gitignored_directory_stay_incremental() {
|
||||
let f = Fixture::with_git();
|
||||
f.write(".gitignore", "target/\nnode_modules/\n");
|
||||
f.write("src/main.rs", "fn main() {}");
|
||||
f.index();
|
||||
|
||||
let mut events = Vec::new();
|
||||
for i in 0..64 {
|
||||
let rel = format!("target/debug/artifact{i}.o");
|
||||
f.write(&rel, "binary");
|
||||
events.push(create(f.path(&rel)));
|
||||
}
|
||||
|
||||
let delta = f.feed(events);
|
||||
f.assert_no_rescan(&delta, "build output written into an ignored directory");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ignored_event_batch_above_index_capacity_stays_incremental() {
|
||||
let f = Fixture::with_git();
|
||||
f.write(".gitignore", "node_modules/\n");
|
||||
f.write("src/main.rs", "fn main() {}");
|
||||
f.index();
|
||||
|
||||
let events = (0..MAX_OVERFLOW_FILES + 1)
|
||||
.map(|i| {
|
||||
let rel = format!("node_modules/pkg/file{i}.js");
|
||||
f.write(&rel, "");
|
||||
create(f.path(&rel))
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let delta = f.feed(events);
|
||||
f.assert_no_rescan(&delta, "ignored events above the index capacity");
|
||||
assert_eq!(f.overflow_len(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn repeated_edits_above_index_capacity_stay_incremental() {
|
||||
let f = Fixture::new();
|
||||
f.write("src/main.rs", "fn main() {}");
|
||||
f.index();
|
||||
|
||||
let path = f.path("src/main.rs");
|
||||
let events = (0..MAX_OVERFLOW_FILES + 1)
|
||||
.map(|_| modify(path.clone()))
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let delta = f.feed(events);
|
||||
f.assert_no_rescan(&delta, "repeated edits above the index capacity");
|
||||
assert_eq!(f.overflow_len(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ignore_file_inside_an_ignored_directory_stays_incremental() {
|
||||
let f = Fixture::with_git();
|
||||
f.write(".gitignore", "node_modules/\n");
|
||||
f.write("src/main.rs", "fn main() {}");
|
||||
f.index();
|
||||
|
||||
let ignore_files =
|
||||
["left-pad", "lodash", "typescript"].map(|pkg| format!("node_modules/{pkg}/.gitignore"));
|
||||
|
||||
for rel in &ignore_files {
|
||||
f.write(rel, "dist\n");
|
||||
}
|
||||
let delta = f.feed(ignore_files.iter().map(|rel| create(f.path(rel))));
|
||||
f.assert_no_rescan(&delta, "creating ignored .gitignore files");
|
||||
|
||||
for rel in &ignore_files {
|
||||
f.write(rel, "build\n");
|
||||
}
|
||||
let delta = f.feed(ignore_files.iter().map(|rel| modify(f.path(rel))));
|
||||
f.assert_no_rescan(&delta, "modifying ignored .gitignore files");
|
||||
|
||||
for rel in &ignore_files {
|
||||
f.remove(rel);
|
||||
}
|
||||
let delta = f.feed(ignore_files.iter().map(|rel| remove_file(f.path(rel))));
|
||||
f.assert_no_rescan(&delta, "removing ignored .gitignore files");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ignore_file_inside_an_indexed_directory_triggers_a_rescan() {
|
||||
let f = Fixture::with_git();
|
||||
f.write("src/.gitignore", ".gitignore\ngenerated/\n");
|
||||
f.write("src/main.rs", "fn main() {}");
|
||||
f.index();
|
||||
|
||||
f.write("src/.gitignore", ".gitignore\ngenerated/\nbuild/\n");
|
||||
let delta = f.feed([modify(f.path("src/.gitignore"))]);
|
||||
|
||||
assert_eq!(delta.count(RescanReason::IgnoreFileChanged), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn git_internal_churn_stays_incremental() {
|
||||
let f = Fixture::with_git();
|
||||
f.write("src/main.rs", "fn main() {}");
|
||||
f.index();
|
||||
|
||||
let git_dir = f.path(".git");
|
||||
let delta = f.feed([
|
||||
create(git_dir.join("index.lock")),
|
||||
modify(git_dir.join("index")),
|
||||
remove_file(git_dir.join("index.lock")),
|
||||
modify(git_dir.join("HEAD")),
|
||||
modify(git_dir.join("logs/HEAD")),
|
||||
modify(git_dir.join("COMMIT_EDITMSG")),
|
||||
modify(git_dir.join("refs/heads/main")),
|
||||
]);
|
||||
|
||||
f.assert_no_rescan(&delta, "git writing its own metadata");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn changing_the_root_ignore_file_triggers_a_rescan() {
|
||||
let f = Fixture::with_git();
|
||||
f.write(".gitignore", "target/\n");
|
||||
f.write("src/main.rs", "fn main() {}");
|
||||
f.index();
|
||||
|
||||
f.write(".gitignore", "target/\nsrc/\n");
|
||||
let delta = f.feed([modify(f.path(".gitignore"))]);
|
||||
|
||||
assert_eq!(
|
||||
delta.count(RescanReason::IgnoreFileChanged),
|
||||
1,
|
||||
"the indexed set depends on the root ignore rules, got {delta}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn kernel_event_loss_on_a_directory_triggers_a_rescan() {
|
||||
let f = Fixture::new();
|
||||
f.write("src/main.rs", "fn main() {}");
|
||||
f.index();
|
||||
|
||||
let delta = f.feed([DebouncedEvent::new(
|
||||
Event::new(EventKind::Modify(ModifyKind::Any))
|
||||
.add_path(f.path("src"))
|
||||
.set_flag(Flag::Rescan),
|
||||
Instant::now(),
|
||||
)]);
|
||||
|
||||
assert_eq!(
|
||||
delta.count(RescanReason::KernelEventLoss),
|
||||
1,
|
||||
"a dropped-events flag over a directory means unknown subtree state, got {delta}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn new_files_above_index_capacity_trigger_a_rescan() {
|
||||
let f = Fixture::new();
|
||||
f.write("src/main.rs", "fn main() {}");
|
||||
f.index();
|
||||
|
||||
let events = (0..MAX_OVERFLOW_FILES + 1)
|
||||
.map(|i| {
|
||||
let rel = format!("src/bulk{i}.rs");
|
||||
f.write(&rel, "");
|
||||
create(f.path(&rel))
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let delta = f.feed(events);
|
||||
assert_eq!(
|
||||
delta.count(RescanReason::IndexUpdateRejected),
|
||||
1,
|
||||
"new files above the overflow region cannot be applied incrementally, got {delta}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn batch_at_the_overflow_boundary_stays_incremental() {
|
||||
let f = Fixture::new();
|
||||
f.write("src/main.rs", "fn main() {}");
|
||||
f.index();
|
||||
|
||||
let events = (0..MAX_OVERFLOW_FILES)
|
||||
.map(|i| {
|
||||
let rel = format!("src/bulk{i}.rs");
|
||||
f.write(&rel, "");
|
||||
create(f.path(&rel))
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let delta = f.feed(events);
|
||||
f.assert_no_rescan(&delta, "a batch exactly at the overflow limit");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn event_batch_at_four_times_index_capacity_stays_incremental() {
|
||||
let f = Fixture::new();
|
||||
f.write("src/main.rs", "fn main() {}");
|
||||
f.index();
|
||||
|
||||
let path = f.path("src/main.rs");
|
||||
let events = (0..MAX_OVERFLOW_FILES * 4)
|
||||
.map(|_| modify(path.clone()))
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let delta = f.feed(events);
|
||||
f.assert_no_rescan(&delta, "an event batch exactly at the event limit");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn event_batch_above_four_times_index_capacity_triggers_a_rescan() {
|
||||
let f = Fixture::new();
|
||||
f.write("src/main.rs", "fn main() {}");
|
||||
f.index();
|
||||
|
||||
let path = f.path("src/main.rs");
|
||||
let events = (0..MAX_OVERFLOW_FILES * 4 + 1)
|
||||
.map(|_| modify(path.clone()))
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let delta = f.feed(events);
|
||||
assert_eq!(
|
||||
delta.count(RescanReason::EventBatchOverflow),
|
||||
1,
|
||||
"an event batch above four times the index capacity must rescan, got {delta}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn repeated_triggers_inside_the_cooldown_collapse_to_one_rescan() {
|
||||
let f = Fixture::with_git();
|
||||
f.write(".gitignore", "target/\n");
|
||||
f.write("src/main.rs", "fn main() {}");
|
||||
f.index();
|
||||
|
||||
// Repeated batches during the cooldown must share one walk.
|
||||
for round in 0..50 {
|
||||
f.write(".gitignore", &format!("target/\n# round {round}\n"));
|
||||
f.feed([modify(f.path(".gitignore"))]);
|
||||
}
|
||||
|
||||
let stats = f.all_rescans();
|
||||
assert_eq!(
|
||||
stats.total, 1,
|
||||
"50 triggers inside the cooldown must collapse to a single walk, got {stats}"
|
||||
);
|
||||
assert_eq!(
|
||||
stats.throttled, 49,
|
||||
"every suppressed request must be accounted for, got {stats}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_explicit_request_is_never_throttled() {
|
||||
let f = Fixture::with_git();
|
||||
f.write(".gitignore", "target/\n");
|
||||
f.write("src/main.rs", "fn main() {}");
|
||||
f.index();
|
||||
|
||||
// Burn the cooldown with a watcher trigger, then confirm a user-initiated
|
||||
// refresh still goes through.
|
||||
f.write(".gitignore", "target/\nsrc/\n");
|
||||
f.feed([modify(f.path(".gitignore"))]);
|
||||
|
||||
for _ in 0..3 {
|
||||
f.picker.trigger_full_rescan_async(&f.frecency).unwrap();
|
||||
}
|
||||
|
||||
let stats = f.all_rescans();
|
||||
assert_eq!(
|
||||
stats.count(RescanReason::Explicit),
|
||||
3,
|
||||
"explicit refreshes must bypass the throttle, got {stats}"
|
||||
);
|
||||
assert_eq!(stats.count_throttled(RescanReason::Explicit), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn events_after_a_suppressed_kernel_rescan_are_still_applied() {
|
||||
let f = Fixture::new();
|
||||
f.write("src/main.rs", "fn main() {}");
|
||||
f.index();
|
||||
|
||||
f.write("src/added.rs", "pub fn added() {}");
|
||||
let delta = f.feed([
|
||||
DebouncedEvent::new(
|
||||
Event::new(EventKind::Modify(ModifyKind::Data(DataChange::Content)))
|
||||
.add_path(f.path("src/main.rs"))
|
||||
.set_flag(Flag::Rescan),
|
||||
Instant::now(),
|
||||
),
|
||||
create(f.path("src/added.rs")),
|
||||
]);
|
||||
|
||||
f.assert_no_rescan(&delta, "a dropped-events flag over a single tracked file");
|
||||
assert!(
|
||||
f.is_indexed("src/added.rs"),
|
||||
"suppressing the rescan must not drop the rest of the batch"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_throttled_ignore_file_event_is_still_applied_incrementally() {
|
||||
let f = Fixture::with_git();
|
||||
f.write(".gitignore", "target/\n");
|
||||
f.write("src/main.rs", "fn main() {}");
|
||||
f.index();
|
||||
|
||||
// Burn the cooldown: deleting .gitignore admits a full rescan.
|
||||
f.remove(".gitignore");
|
||||
let delta = f.feed([remove_file(f.path(".gitignore"))]);
|
||||
assert_eq!(delta.count(RescanReason::IgnoreFileChanged), 1);
|
||||
f.picker.wait_for_indexing_complete(Duration::from_secs(10));
|
||||
|
||||
// Recreating it inside the cooldown throttles the rescan, but the file
|
||||
// itself must re-enter the index via the incremental fallback.
|
||||
f.write(".gitignore", "target/\n__ignored_x/\n");
|
||||
let delta = f.feed([create(f.path(".gitignore"))]);
|
||||
assert_eq!(delta.total, 0, "the rescan must be throttled, got {delta}");
|
||||
assert_eq!(delta.count_throttled(RescanReason::IgnoreFileChanged), 1);
|
||||
assert!(
|
||||
f.is_indexed(".gitignore"),
|
||||
"a throttled ignore-file event must still index the file itself"
|
||||
);
|
||||
}
|
||||
|
||||
struct Fixture {
|
||||
base: PathBuf,
|
||||
picker: SharedFilePicker,
|
||||
frecency: SharedFrecency,
|
||||
git_workdir: Option<PathBuf>,
|
||||
git_worker: Arc<GitStatusWorker>,
|
||||
// Dropped last so background work started by a triggered rescan still
|
||||
// sees the tree it was asked to walk.
|
||||
_tmp: TempDir,
|
||||
}
|
||||
|
||||
impl Fixture {
|
||||
fn new() -> Self {
|
||||
Self::build(false)
|
||||
}
|
||||
|
||||
fn with_git() -> Self {
|
||||
Self::build(true)
|
||||
}
|
||||
|
||||
fn build(git: bool) -> Self {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let base = crate::path_utils::canonicalize(tmp.path()).unwrap();
|
||||
let git_workdir = git.then(|| {
|
||||
let status = Command::new("git")
|
||||
.args(["init", "-b", "main"])
|
||||
.current_dir(&base)
|
||||
.output()
|
||||
.expect("git init");
|
||||
assert!(status.status.success(), "git init failed");
|
||||
base.clone()
|
||||
});
|
||||
|
||||
Self {
|
||||
base,
|
||||
picker: SharedFilePicker::default(),
|
||||
frecency: SharedFrecency::noop(),
|
||||
git_workdir,
|
||||
git_worker: GitStatusWorker::new(),
|
||||
_tmp: tmp,
|
||||
}
|
||||
}
|
||||
|
||||
fn index(&self) {
|
||||
let mut picker = FilePicker::new(FilePickerOptions {
|
||||
base_path: self.base.to_string_lossy().into_owned(),
|
||||
watch: false,
|
||||
..Default::default()
|
||||
})
|
||||
.unwrap();
|
||||
picker.collect_files().unwrap();
|
||||
self.picker.rebase_watches(&self.base);
|
||||
*self.picker.write().unwrap() = Some(picker);
|
||||
}
|
||||
|
||||
fn feed(&self, events: impl IntoIterator<Item = DebouncedEvent>) -> RescanStats {
|
||||
let before = self.picker.rescan_stats();
|
||||
handle_debounced_events(
|
||||
FFFMode::Neovim,
|
||||
events.into_iter().collect(),
|
||||
&self.base,
|
||||
&self.git_workdir,
|
||||
&self.picker,
|
||||
&self.frecency,
|
||||
&self.git_worker,
|
||||
);
|
||||
|
||||
self.picker.rescan_stats().since(&before)
|
||||
}
|
||||
|
||||
fn assert_no_rescan(&self, delta: &RescanStats, what: &str) {
|
||||
assert_eq!(delta.total, 0, "{what} must not trigger a rescan: {delta}");
|
||||
}
|
||||
|
||||
fn path(&self, rel: &str) -> PathBuf {
|
||||
self.base.join(rel)
|
||||
}
|
||||
|
||||
fn write(&self, rel: &str, contents: &str) {
|
||||
let path = self.path(rel);
|
||||
std::fs::create_dir_all(path.parent().unwrap()).unwrap();
|
||||
std::fs::write(path, contents).unwrap();
|
||||
}
|
||||
|
||||
fn remove(&self, rel: &str) {
|
||||
std::fs::remove_file(self.path(rel)).unwrap();
|
||||
}
|
||||
|
||||
fn is_indexed(&self, rel: &str) -> bool {
|
||||
let guard = self.picker.read().unwrap();
|
||||
guard
|
||||
.as_ref()
|
||||
.and_then(|p| p.get_file_by_path(self.path(rel)))
|
||||
.is_some_and(|file| !file.is_deleted())
|
||||
}
|
||||
|
||||
fn all_rescans(&self) -> RescanStats {
|
||||
self.picker.rescan_stats()
|
||||
}
|
||||
|
||||
fn overflow_len(&self) -> usize {
|
||||
let guard = self.picker.read().unwrap();
|
||||
guard
|
||||
.as_ref()
|
||||
.map(|p| p.get_overflow_files().len())
|
||||
.unwrap_or(0)
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Fixture {
|
||||
fn drop(&mut self) {
|
||||
// A test that intentionally triggers a rescan leaves a walk running on
|
||||
// the background pool; let it finish before the tree disappears.
|
||||
self.picker
|
||||
.wait_for_indexing_complete(Duration::from_secs(10));
|
||||
}
|
||||
}
|
||||
|
||||
fn event(kind: EventKind, path: PathBuf) -> DebouncedEvent {
|
||||
DebouncedEvent::new(Event::new(kind).add_path(path), Instant::now())
|
||||
}
|
||||
|
||||
fn create(path: PathBuf) -> DebouncedEvent {
|
||||
event(EventKind::Create(CreateKind::File), path)
|
||||
}
|
||||
|
||||
fn modify(path: PathBuf) -> DebouncedEvent {
|
||||
event(
|
||||
EventKind::Modify(ModifyKind::Data(DataChange::Content)),
|
||||
path,
|
||||
)
|
||||
}
|
||||
|
||||
fn remove_file(path: PathBuf) -> DebouncedEvent {
|
||||
event(EventKind::Remove(RemoveKind::File), path)
|
||||
}
|
||||
@@ -254,3 +254,48 @@ fn recreated_directory_reappears_in_dir_search() {
|
||||
search_dirs(&picker, "phoenix")
|
||||
);
|
||||
}
|
||||
|
||||
/// Regression for #725: a dir that is EMPTY at scan time must be indexed —
|
||||
/// searchable in dir search and watched so later file creations are seen.
|
||||
#[test]
|
||||
fn empty_directory_at_scan_is_searchable_and_watched() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let base = fff_search::path_utils::canonicalize(tmp.path()).unwrap();
|
||||
fs::create_dir_all(base.join("commands")).unwrap();
|
||||
fs::write(base.join("keep.rs"), "x").unwrap();
|
||||
|
||||
let (picker, _frecency) = make_watched_picker(&base);
|
||||
|
||||
assert!(
|
||||
search_dirs(&picker, "commands")
|
||||
.iter()
|
||||
.any(|d| d.starts_with("commands")),
|
||||
"empty dir must be searchable right after the scan, got: {:?}",
|
||||
search_dirs(&picker, "commands")
|
||||
);
|
||||
|
||||
// The empty dir must reuse its scan-built DirItem when a file lands in it
|
||||
// and the watcher must have registered a watch on it (the #725 repro).
|
||||
fs::write(base.join("commands/review.md"), "# review").unwrap();
|
||||
assert!(
|
||||
wait_until(
|
||||
|| {
|
||||
let guard = picker.read().unwrap();
|
||||
let p = guard.as_ref().unwrap();
|
||||
p.get_file_by_path(base.join("commands/review.md"))
|
||||
.is_some()
|
||||
},
|
||||
Duration::from_secs(10)
|
||||
),
|
||||
"file created in a scan-time-empty dir must be indexed"
|
||||
);
|
||||
|
||||
let guard = picker.read().unwrap();
|
||||
let p = guard.as_ref().unwrap();
|
||||
let commands_dirs = p
|
||||
.get_dirs()
|
||||
.iter()
|
||||
.filter(|d| d.relative_path(p).starts_with("commands"))
|
||||
.count();
|
||||
assert_eq!(commands_dirs, 1, "no duplicate DirItem for the empty dir");
|
||||
}
|
||||
|
||||
@@ -773,7 +773,7 @@ fn read_truth_status(base: &Path) -> BTreeMap<String, Status> {
|
||||
|
||||
let mut out = BTreeMap::new();
|
||||
for entry in statuses.iter() {
|
||||
if let Some(p) = entry.path() {
|
||||
if let Ok(p) = entry.path() {
|
||||
// git2 returns forward-slash paths; accept as-is.
|
||||
out.insert(p.to_string(), entry.status());
|
||||
}
|
||||
@@ -1258,7 +1258,7 @@ fn get_baseline_status_from_git(base: &Path) -> Vec<Live> {
|
||||
Err(_) => return out,
|
||||
};
|
||||
for entry in statuses.iter() {
|
||||
if let Some(p) = entry.path() {
|
||||
if let Ok(p) = entry.path() {
|
||||
let abs = base.join(p);
|
||||
// Must be a real file *right now* — ignore stale WT_DELETED rows.
|
||||
if abs.is_file() {
|
||||
|
||||
@@ -0,0 +1,123 @@
|
||||
//! One process must be able to hold many trackers over the same LMDB path
|
||||
//! (issues #700/#760): they share a single pooled env instead of failing
|
||||
//! with `EnvAlreadyOpened`.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use fff_search::frecency::FrecencyTracker;
|
||||
use fff_search::query_tracker::QueryTracker;
|
||||
use fff_search::shared::SharedFrecency;
|
||||
|
||||
fn unique_dir(name: &str) -> PathBuf {
|
||||
let dir = std::env::temp_dir().join(format!("fff-env-pool-{name}-{}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
dir
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn same_path_trackers_share_one_env() {
|
||||
let dir = unique_dir("share");
|
||||
let file = Path::new("/virtual/env-pool/shared.rs");
|
||||
|
||||
let a = FrecencyTracker::open(&dir).expect("first open");
|
||||
let b = FrecencyTracker::open(&dir).expect("second open in the same process (#700/#760)");
|
||||
|
||||
a.track_access(file).expect("write via a");
|
||||
assert_eq!(b.access_count(file).expect("read via b"), 1);
|
||||
|
||||
drop(a);
|
||||
b.track_access(file)
|
||||
.expect("b must stay usable after a drops");
|
||||
assert_eq!(b.access_count(file).unwrap(), 2);
|
||||
drop(b);
|
||||
|
||||
let c = FrecencyTracker::open(&dir).expect("reopen after all handles dropped");
|
||||
assert_eq!(
|
||||
c.access_count(file).unwrap(),
|
||||
2,
|
||||
"data persisted across reopen"
|
||||
);
|
||||
|
||||
drop(c);
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn concurrent_open_and_drop_never_collide() {
|
||||
let dir = unique_dir("hammer");
|
||||
let file = Path::new("/virtual/env-pool/hammer.rs");
|
||||
|
||||
let mut handles = Vec::new();
|
||||
for t in 0..8 {
|
||||
let dir = dir.clone();
|
||||
handles.push(std::thread::spawn(move || {
|
||||
for i in 0..100 {
|
||||
let tracker = FrecencyTracker::open(&dir)
|
||||
.unwrap_or_else(|e| panic!("thread {t} iteration {i}: {e}"));
|
||||
if i % 20 == 0 {
|
||||
tracker.track_access(file).expect("track access");
|
||||
}
|
||||
}
|
||||
}));
|
||||
}
|
||||
for handle in handles {
|
||||
handle.join().expect("no thread may panic");
|
||||
}
|
||||
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn different_store_on_same_path_is_rejected_with_clear_error() {
|
||||
let dir = unique_dir("mismatch");
|
||||
|
||||
let _frecency = FrecencyTracker::open(&dir).expect("frecency open");
|
||||
let err = QueryTracker::open(&dir).expect_err("env options differ, must be rejected");
|
||||
|
||||
let msg = err.to_string();
|
||||
assert!(
|
||||
msg.contains("frecency") && msg.contains("query"),
|
||||
"error must name both stores so the user can fix their config, got: {msg}"
|
||||
);
|
||||
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn destroy_refuses_while_shared_then_succeeds_when_sole() {
|
||||
let dir = unique_dir("destroy");
|
||||
let file = Path::new("/virtual/env-pool/destroy.rs");
|
||||
|
||||
let shared = SharedFrecency::default();
|
||||
shared
|
||||
.init(FrecencyTracker::open(&dir).expect("init open"))
|
||||
.expect("init");
|
||||
let other = FrecencyTracker::open(&dir).expect("second handle over the same db");
|
||||
|
||||
shared
|
||||
.destroy()
|
||||
.expect_err("destroy must refuse while another tracker uses the env");
|
||||
|
||||
// Refusal must keep both the files and the shared handle intact.
|
||||
assert!(
|
||||
dir.join("data.mdb").exists(),
|
||||
"db files survive a refused destroy"
|
||||
);
|
||||
shared
|
||||
.read()
|
||||
.expect("read lock")
|
||||
.as_ref()
|
||||
.expect("tracker restored after refused destroy")
|
||||
.track_access(file)
|
||||
.expect("shared handle still works");
|
||||
|
||||
drop(other);
|
||||
let removed = shared
|
||||
.destroy()
|
||||
.expect("sole-owner destroy succeeds")
|
||||
.expect("a path was removed");
|
||||
assert!(
|
||||
!removed.exists(),
|
||||
"db dir deleted once nobody shares the env"
|
||||
);
|
||||
}
|
||||
@@ -8,8 +8,9 @@
|
||||
//! 3. The watcher's event handler detects the directory Create event,
|
||||
//! collects it, and sends it to the owner thread via `watch_tx`.
|
||||
//! 4. The owner thread adds a NonRecursive watch on the new directory and
|
||||
//! does a flat (non-recursive) read_dir to inject files that already
|
||||
//! exist (race-window coverage).
|
||||
//! walks its subtree (`index_new_directory`) to inject files that
|
||||
//! already exist (race-window + burst/mv-in coverage) and to watch
|
||||
//! nested subdirectories.
|
||||
//! 5. Files created *after* the watch is established are picked up via
|
||||
//! normal event delivery.
|
||||
//!
|
||||
@@ -469,6 +470,160 @@ fn burst_file_creation_in_new_directory() {
|
||||
}
|
||||
}
|
||||
|
||||
/// bug pinning #725: a directory that already exists but is EMPTY at
|
||||
/// initial scan time is absent from `sync_data.dirs` and missing watch events
|
||||
#[test]
|
||||
fn file_created_in_preexisting_empty_directory() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let base = tmp.path().canonicalize().unwrap();
|
||||
|
||||
// `commands/` is empty during the initial scan — only `init.lua` is indexed.
|
||||
fs::create_dir_all(base.join("commands")).unwrap();
|
||||
fs::write(base.join("init.lua"), "-- init\n").unwrap();
|
||||
|
||||
let (shared_picker, _frecency) = make_watched_picker(&base);
|
||||
wait_ready(&shared_picker);
|
||||
|
||||
// Now write a file into the directory that was empty at scan time.
|
||||
fs::write(
|
||||
base.join("commands/review.md"),
|
||||
"# Review\nEMPTY_DIR_REVIEW_TOKEN\n",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let elapsed = poll_until(
|
||||
&shared_picker,
|
||||
WATCHER_TIMEOUT,
|
||||
"file commands/review.md created in a pre-existing empty directory",
|
||||
|picker| {
|
||||
picker
|
||||
.get_files()
|
||||
.iter()
|
||||
.any(|f| f.relative_path(picker).contains("review.md"))
|
||||
},
|
||||
);
|
||||
eprintln!(
|
||||
" File in pre-existing empty directory detected in {:.0}ms",
|
||||
elapsed.as_secs_f64() * 1000.0
|
||||
);
|
||||
}
|
||||
|
||||
/// Same as above but with a nested chain of empty directories under an
|
||||
/// indexed one: every level of the empty subtree must be watched.
|
||||
#[test]
|
||||
fn file_created_in_nested_preexisting_empty_directories() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let base = tmp.path().canonicalize().unwrap();
|
||||
|
||||
// `src/` is indexed (has a file); `src/plugins/extra/` is an empty chain.
|
||||
fs::create_dir_all(base.join("src/plugins/extra")).unwrap();
|
||||
fs::write(base.join("src/main.rs"), "fn main() {}\n").unwrap();
|
||||
|
||||
git_init_and_commit(&base);
|
||||
|
||||
let (shared_picker, _frecency) = make_watched_picker(&base);
|
||||
wait_ready(&shared_picker);
|
||||
|
||||
fs::write(
|
||||
base.join("src/plugins/extra/loader.rs"),
|
||||
"pub fn load() {}\nconst TOKEN: &str = \"NESTED_EMPTY_DIR_TOKEN\";\n",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let elapsed = poll_until(
|
||||
&shared_picker,
|
||||
WATCHER_TIMEOUT,
|
||||
"file src/plugins/extra/loader.rs created in nested empty directories",
|
||||
|picker| {
|
||||
picker
|
||||
.get_files()
|
||||
.iter()
|
||||
.any(|f| f.relative_path(picker).contains("loader.rs"))
|
||||
},
|
||||
);
|
||||
eprintln!(
|
||||
" File in nested empty directories detected in {:.0}ms",
|
||||
elapsed.as_secs_f64() * 1000.0
|
||||
);
|
||||
|
||||
poll_until(
|
||||
&shared_picker,
|
||||
WATCHER_TIMEOUT,
|
||||
"grep finds NESTED_EMPTY_DIR_TOKEN",
|
||||
|picker| grep_plain_count(picker, "NESTED_EMPTY_DIR_TOKEN") >= 1,
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nested_tree_created_in_one_burst_detected() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let base = tmp.path().canonicalize().unwrap();
|
||||
|
||||
fs::write(base.join("root.txt"), "root file\n").unwrap();
|
||||
git_init_and_commit(&base);
|
||||
|
||||
let (shared_picker, _frecency) = make_watched_picker(&base);
|
||||
wait_ready(&shared_picker);
|
||||
|
||||
// No sleeps between levels: the watcher sees one Create for `pkg` and
|
||||
// must index the whole subtree from it.
|
||||
fs::create_dir_all(base.join("pkg/src/nested")).unwrap();
|
||||
fs::write(base.join("pkg/Cargo.toml"), "[package]\n").unwrap();
|
||||
fs::write(
|
||||
base.join("pkg/src/lib.rs"),
|
||||
"const TOKEN: &str = \"BURST_TREE_LIB_TOKEN\";\n",
|
||||
)
|
||||
.unwrap();
|
||||
fs::write(
|
||||
base.join("pkg/src/nested/deep.rs"),
|
||||
"const TOKEN: &str = \"BURST_TREE_DEEP_TOKEN\";\n",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
for rel in ["pkg/Cargo.toml", "pkg/src/lib.rs", "pkg/src/nested/deep.rs"] {
|
||||
let elapsed = poll_until(
|
||||
&shared_picker,
|
||||
WATCHER_TIMEOUT,
|
||||
&format!("burst-created file {rel}"),
|
||||
|picker| {
|
||||
picker
|
||||
.get_files()
|
||||
.iter()
|
||||
.any(|f| f.relative_path(picker) == rel)
|
||||
},
|
||||
);
|
||||
eprintln!(
|
||||
" Burst file {rel} detected in {:.0}ms",
|
||||
elapsed.as_secs_f64() * 1000.0
|
||||
);
|
||||
}
|
||||
|
||||
// Files created later at the deepest level need the nested watches too.
|
||||
fs::write(
|
||||
base.join("pkg/src/nested/late.rs"),
|
||||
"const TOKEN: &str = \"BURST_TREE_LATE_TOKEN\";\n",
|
||||
)
|
||||
.unwrap();
|
||||
poll_until(
|
||||
&shared_picker,
|
||||
WATCHER_TIMEOUT,
|
||||
"late file in burst-created nested dir",
|
||||
|picker| {
|
||||
picker
|
||||
.get_files()
|
||||
.iter()
|
||||
.any(|f| f.relative_path(picker).ends_with("late.rs"))
|
||||
},
|
||||
);
|
||||
|
||||
poll_until(
|
||||
&shared_picker,
|
||||
WATCHER_TIMEOUT,
|
||||
"grep finds BURST_TREE_DEEP_TOKEN",
|
||||
|picker| grep_plain_count(picker, "BURST_TREE_DEEP_TOKEN") >= 1,
|
||||
);
|
||||
}
|
||||
|
||||
/// Verify that gitignored directories created at runtime are NOT watched
|
||||
/// and their files do NOT appear in the index.
|
||||
#[test]
|
||||
|
||||
@@ -222,6 +222,17 @@ fn multi_grep_with_file_path_suffix_constraint() {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multi_grep_with_missing_file_path_constraint_returns_no_matches() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let picker = create_picker(tmp.path(), &[("other.lua", "handleRequest\n")]);
|
||||
|
||||
let constraints = [Constraint::FilePath("missing.lua")];
|
||||
let result = picker.multi_grep(&["handleRequest"], &constraints, &plain_opts());
|
||||
|
||||
assert!(result.matches.is_empty());
|
||||
}
|
||||
|
||||
/// Glob constraints must match native Windows paths — the picker normalises
|
||||
/// separators when handing paths to the glob matcher.
|
||||
#[test]
|
||||
|
||||
@@ -0,0 +1,343 @@
|
||||
#![cfg(rescan_stats)]
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Command;
|
||||
use std::time::Duration;
|
||||
|
||||
use fff_search::file_picker::{FFFMode, FilePicker};
|
||||
use fff_search::{FilePickerOptions, RescanStats, SharedFilePicker, SharedFrecency};
|
||||
use tempfile::TempDir;
|
||||
|
||||
const SETTLE: Duration = Duration::from_millis(600);
|
||||
|
||||
#[test]
|
||||
fn saving_source_files_does_not_rescan() {
|
||||
let repo = WatchedRepo::new(|base| {
|
||||
write(base, ".gitignore", "target/\n");
|
||||
for i in 0..20 {
|
||||
write(base, &format!("src/mod{i}.rs"), "pub fn f() {}");
|
||||
}
|
||||
});
|
||||
|
||||
for round in 0..10 {
|
||||
for i in 0..20 {
|
||||
repo.write(
|
||||
&format!("src/mod{i}.rs"),
|
||||
&format!("pub fn f() {{ let _ = {round}; }}"),
|
||||
);
|
||||
}
|
||||
repo.settle();
|
||||
}
|
||||
|
||||
repo.assert_quiet("200 file saves");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_output_in_ignored_directories_does_not_rescan() {
|
||||
let repo = WatchedRepo::new(|base| {
|
||||
write(base, ".gitignore", "target/\nnode_modules/\ndist/\n");
|
||||
write(base, "src/main.rs", "fn main() {}");
|
||||
});
|
||||
|
||||
for round in 0..4 {
|
||||
for i in 0..150 {
|
||||
repo.write(&format!("target/debug/deps/unit-{round}-{i}.o"), "binary");
|
||||
repo.write(&format!("dist/chunk-{round}-{i}.js"), "bundled");
|
||||
}
|
||||
repo.settle();
|
||||
}
|
||||
|
||||
repo.assert_quiet("1200 build artifacts written into ignored directories");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn adding_source_files_and_directories_does_not_rescan() {
|
||||
let repo = WatchedRepo::new(|base| {
|
||||
write(base, ".gitignore", "target/\n");
|
||||
write(base, "src/main.rs", "fn main() {}");
|
||||
});
|
||||
|
||||
for i in 0..40 {
|
||||
repo.write(&format!("src/feature{i}/mod.rs"), "pub mod inner;");
|
||||
repo.write(&format!("src/feature{i}/inner.rs"), "pub fn go() {}");
|
||||
}
|
||||
repo.settle();
|
||||
|
||||
assert!(
|
||||
repo.wait_indexed("src/feature39/inner.rs"),
|
||||
"watcher must index files in newly created directories"
|
||||
);
|
||||
repo.assert_quiet("40 new directories with 80 files");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recreating_generated_files_does_not_rescan() {
|
||||
let repo = WatchedRepo::new(|base| {
|
||||
write(base, ".gitignore", "target/\n");
|
||||
write(base, "src/main.rs", "fn main() {}");
|
||||
});
|
||||
|
||||
// Recreated paths must reuse their overflow slots.
|
||||
for round in 0..12 {
|
||||
for i in 0..40 {
|
||||
repo.write(&format!("src/generated/api{i}.rs"), "pub struct A;");
|
||||
}
|
||||
repo.settle();
|
||||
for i in 0..40 {
|
||||
repo.remove(&format!("src/generated/api{i}.rs"));
|
||||
}
|
||||
repo.settle();
|
||||
assert!(
|
||||
repo.overflow_len() <= 64,
|
||||
"round {round}: regenerating the same paths grew the overflow region to {}",
|
||||
repo.overflow_len()
|
||||
);
|
||||
}
|
||||
|
||||
repo.assert_quiet("12 codegen cycles over 40 stable paths");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn git_workflow_does_not_rescan() {
|
||||
let repo = WatchedRepo::new(|base| {
|
||||
write(base, ".gitignore", "target/\n");
|
||||
write(base, "src/main.rs", "fn main() {}");
|
||||
write(base, "src/lib.rs", "pub mod thing;");
|
||||
git(base, &["init", "-b", "main"]);
|
||||
git(base, &["add", "-A"]);
|
||||
git(base, &["commit", "-m", "initial"]);
|
||||
});
|
||||
|
||||
repo.write("src/main.rs", "fn main() { println!(\"hi\"); }");
|
||||
repo.settle();
|
||||
repo.git(&["add", "-A"]);
|
||||
repo.settle();
|
||||
repo.git(&["commit", "-m", "second"]);
|
||||
repo.settle();
|
||||
repo.git(&["checkout", "-b", "feature"]);
|
||||
repo.settle();
|
||||
repo.write("src/feature.rs", "pub fn feature() {}");
|
||||
repo.git(&["add", "-A"]);
|
||||
repo.git(&["commit", "-m", "feature"]);
|
||||
repo.settle();
|
||||
repo.git(&["checkout", "main"]);
|
||||
repo.settle();
|
||||
repo.git(&["merge", "feature"]);
|
||||
repo.settle();
|
||||
|
||||
repo.assert_quiet("a commit / branch / merge cycle");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reading_files_does_not_rescan() {
|
||||
let repo = WatchedRepo::new(|base| {
|
||||
write(base, ".gitignore", "target/\n");
|
||||
for i in 0..50 {
|
||||
write(base, &format!("src/mod{i}.rs"), "pub fn f() {}");
|
||||
}
|
||||
});
|
||||
|
||||
// Preview rendering and grep open every file in the result list. Reacting
|
||||
// to those reads would make the picker rescan while the user scrolls.
|
||||
for _ in 0..5 {
|
||||
for i in 0..50 {
|
||||
let _ = std::fs::read(repo.path(&format!("src/mod{i}.rs"))).unwrap();
|
||||
}
|
||||
}
|
||||
repo.settle();
|
||||
|
||||
repo.assert_quiet("reading every indexed file");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn npm_install_style_churn_does_not_rescan() {
|
||||
let repo = WatchedRepo::new(|base| {
|
||||
write(base, ".gitignore", "node_modules/\n");
|
||||
write(base, "src/index.ts", "export const a = 1;");
|
||||
});
|
||||
|
||||
for pkg in 0..100 {
|
||||
repo.write(&format!("node_modules/pkg{pkg}/package.json"), "{}");
|
||||
repo.write(
|
||||
&format!("node_modules/pkg{pkg}/index.js"),
|
||||
"module.exports={}",
|
||||
);
|
||||
repo.write(&format!("node_modules/pkg{pkg}/.gitignore"), "dist\n");
|
||||
}
|
||||
repo.settle();
|
||||
repo.settle();
|
||||
|
||||
repo.assert_quiet("an npm install into an ignored node_modules");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_churning_root_is_capped_at_one_rescan_per_cooldown() {
|
||||
let repo = WatchedRepo::new(|base| {
|
||||
write(base, "src/main.rs", "fn main() {}");
|
||||
});
|
||||
|
||||
// Root ignore changes force watcher rescan requests.
|
||||
for round in 0..25 {
|
||||
repo.write(".gitignore", &format!("target/\n# round {round}\n"));
|
||||
std::thread::sleep(Duration::from_millis(120));
|
||||
}
|
||||
repo.settle();
|
||||
|
||||
let stats = repo.rescans();
|
||||
assert!(
|
||||
stats.total <= 1,
|
||||
"a churning root must not exceed one walk per cooldown, got {stats}"
|
||||
);
|
||||
assert!(
|
||||
stats.throttled > 0,
|
||||
"the suppressed triggers must be recorded, got {stats}"
|
||||
);
|
||||
}
|
||||
|
||||
struct WatchedRepo {
|
||||
base: PathBuf,
|
||||
picker: SharedFilePicker,
|
||||
_frecency: SharedFrecency,
|
||||
_tmp: TempDir,
|
||||
}
|
||||
|
||||
impl WatchedRepo {
|
||||
fn new(setup: impl FnOnce(&Path)) -> Self {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let base = fff_search::path_utils::canonicalize(tmp.path()).unwrap();
|
||||
setup(&base);
|
||||
|
||||
let picker = SharedFilePicker::default();
|
||||
let frecency = SharedFrecency::noop();
|
||||
FilePicker::new_with_shared_state(
|
||||
picker.clone(),
|
||||
frecency.clone(),
|
||||
FilePickerOptions {
|
||||
base_path: base.to_string_lossy().into_owned(),
|
||||
enable_mmap_cache: false,
|
||||
mode: FFFMode::Neovim,
|
||||
watch: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.expect("failed to create file picker");
|
||||
|
||||
assert!(
|
||||
picker.wait_for_scan(Duration::from_secs(60)),
|
||||
"timed out waiting for the initial scan"
|
||||
);
|
||||
assert!(
|
||||
picker.wait_for_watcher(Duration::from_secs(60)),
|
||||
"timed out waiting for the watcher"
|
||||
);
|
||||
|
||||
let repo = Self {
|
||||
base,
|
||||
picker,
|
||||
_frecency: frecency,
|
||||
_tmp: tmp,
|
||||
};
|
||||
repo.settle();
|
||||
repo.picker.reset_rescan_stats();
|
||||
repo
|
||||
}
|
||||
|
||||
fn settle(&self) {
|
||||
std::thread::sleep(SETTLE);
|
||||
assert!(
|
||||
self.picker
|
||||
.wait_for_indexing_complete(Duration::from_secs(60)),
|
||||
"timed out waiting for background indexing to finish"
|
||||
);
|
||||
}
|
||||
|
||||
fn assert_quiet(&self, workload: &str) {
|
||||
let stats = self.rescans();
|
||||
assert_eq!(
|
||||
stats.watcher_triggered(),
|
||||
0,
|
||||
"{workload} must be absorbed incrementally, but the watcher fell back to {stats}"
|
||||
);
|
||||
}
|
||||
|
||||
fn rescans(&self) -> RescanStats {
|
||||
self.picker.rescan_stats()
|
||||
}
|
||||
|
||||
fn path(&self, rel: &str) -> PathBuf {
|
||||
self.base.join(rel)
|
||||
}
|
||||
|
||||
fn write(&self, rel: &str, contents: &str) {
|
||||
write(&self.base, rel, contents);
|
||||
}
|
||||
|
||||
fn remove(&self, rel: &str) {
|
||||
std::fs::remove_file(self.path(rel)).unwrap();
|
||||
}
|
||||
|
||||
fn git(&self, args: &[&str]) {
|
||||
git(&self.base, args);
|
||||
}
|
||||
|
||||
fn wait_indexed(&self, rel: &str) -> bool {
|
||||
let deadline = std::time::Instant::now() + Duration::from_secs(30);
|
||||
while std::time::Instant::now() < deadline {
|
||||
if self.is_indexed(rel) {
|
||||
return true;
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(50));
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
fn is_indexed(&self, rel: &str) -> bool {
|
||||
let guard = self.picker.read().unwrap();
|
||||
guard
|
||||
.as_ref()
|
||||
.and_then(|p| p.get_file_by_path(self.path(rel)))
|
||||
.is_some_and(|file| !file.is_deleted())
|
||||
}
|
||||
|
||||
fn overflow_len(&self) -> usize {
|
||||
let guard = self.picker.read().unwrap();
|
||||
guard
|
||||
.as_ref()
|
||||
.map(|p| p.get_overflow_files().len())
|
||||
.unwrap_or(0)
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for WatchedRepo {
|
||||
fn drop(&mut self) {
|
||||
// Stop the watcher before the tree disappears, otherwise a late batch
|
||||
// races the tempdir removal.
|
||||
if let Ok(mut guard) = self.picker.write() {
|
||||
guard.take();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn write(base: &Path, rel: &str, contents: &str) {
|
||||
let path = base.join(rel);
|
||||
std::fs::create_dir_all(path.parent().unwrap()).unwrap();
|
||||
std::fs::write(path, contents).unwrap();
|
||||
}
|
||||
|
||||
fn git(dir: &Path, args: &[&str]) {
|
||||
let out = Command::new("git")
|
||||
.args(args)
|
||||
.current_dir(dir)
|
||||
.env("GIT_AUTHOR_NAME", "test")
|
||||
.env("GIT_AUTHOR_EMAIL", "test@test.com")
|
||||
.env("GIT_COMMITTER_NAME", "test")
|
||||
.env("GIT_COMMITTER_EMAIL", "test@test.com")
|
||||
.output()
|
||||
.unwrap_or_else(|e| panic!("git {args:?} failed to spawn: {e}"));
|
||||
|
||||
assert!(
|
||||
out.status.success(),
|
||||
"git {args:?} failed: {}",
|
||||
String::from_utf8_lossy(&out.stderr)
|
||||
);
|
||||
}
|
||||
@@ -3,7 +3,7 @@ name = "fff-grep"
|
||||
description = "File grepping logic for fff"
|
||||
license = "MIT"
|
||||
authors = ["Dmitriy Kovalenko <dmtr.kovalenko@outlok.com>"]
|
||||
version = "0.10.1"
|
||||
version = "0.10.4"
|
||||
edition = "2024"
|
||||
|
||||
[lints]
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "fff-mcp"
|
||||
version = "0.10.1"
|
||||
version = "0.10.4"
|
||||
edition = "2024"
|
||||
description = "MCP server for FFF file finder - drop-in replacement for AI code assistant search tools"
|
||||
license = "MIT"
|
||||
@@ -19,8 +19,8 @@ ripgrep = ["fff/ripgrep", "fff-query-parser/ripgrep"]
|
||||
zlob = ["fff/zlob", "fff-query-parser/zlob"]
|
||||
|
||||
[dependencies]
|
||||
fff = { package = "fff-search", path = "../fff-core", default-features = false , version = "0.10.1", features = ["definitions"] }
|
||||
fff-query-parser = { path = "../fff-query-parser", default-features = false , version = "0.10.1" }
|
||||
fff = { package = "fff-search", path = "../fff-core", default-features = false , version = "0.10.4", features = ["definitions"] }
|
||||
fff-query-parser = { path = "../fff-query-parser", default-features = false , version = "0.10.4" }
|
||||
mimalloc = { workspace = true }
|
||||
rmcp = { version = "1.7.0", features = ["server", "transport-io"] }
|
||||
schemars = "1.0"
|
||||
@@ -30,3 +30,13 @@ tokio = { version = "1", features = ["full"] }
|
||||
tracing = { workspace = true }
|
||||
git2 = { workspace = true }
|
||||
clap = { version = "4", features = ["derive", "env"] }
|
||||
|
||||
[target.'cfg(windows)'.dependencies]
|
||||
windows-sys = { version = "0.60", features = [
|
||||
"Win32_Foundation",
|
||||
"Win32_System_Threading",
|
||||
"Win32_System_Diagnostics_ToolHelp",
|
||||
] }
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3.8"
|
||||
|
||||
+85
-13
@@ -1,9 +1,12 @@
|
||||
mod cursor;
|
||||
mod healthcheck;
|
||||
mod output;
|
||||
mod parent;
|
||||
mod server;
|
||||
mod update_check;
|
||||
|
||||
use std::time::{Duration, SystemTime};
|
||||
|
||||
use clap::Parser;
|
||||
use fff::file_picker::FilePicker;
|
||||
use fff::frecency::FrecencyTracker;
|
||||
@@ -92,7 +95,7 @@ pub const MCP_INSTRUCTIONS: &str = concat!(
|
||||
" !generated/ - exclude generated code",
|
||||
);
|
||||
|
||||
/// FFF MCP Server -- a high performance & accuracy file finder for AI code assistants.
|
||||
/// FFF MCP Server - a high performance & accuracy file finder for AI code assistants.
|
||||
#[derive(Parser)]
|
||||
#[command(name = "fff-mcp", version = concat!(env!("CARGO_PKG_VERSION"), " (", env!("FFF_GIT_HASH"), ")"))]
|
||||
pub(crate) struct Args {
|
||||
@@ -131,6 +134,7 @@ pub(crate) struct Args {
|
||||
|
||||
/// Disable the content index built after the initial scan.
|
||||
/// This makes grep calls slower but consumes less RAM (recommended to not turn off)
|
||||
#[arg(long = "no-content-indexing")]
|
||||
no_content_indexing: bool,
|
||||
|
||||
/// Explicitly enable content indexing even when `--no-warmup` is set.
|
||||
@@ -154,15 +158,40 @@ pub(crate) struct Args {
|
||||
#[arg(long = "follow-symlinks")]
|
||||
follow_symlinks: bool,
|
||||
|
||||
/// Allow indexing the user's home directory. FFF refuses to init in `~`
|
||||
/// unless this is set. Also settable via FFF_ENABLE_HOME_SCAN=1.
|
||||
#[arg(
|
||||
long = "enable-home-scan",
|
||||
env = "FFF_ENABLE_HOME_SCAN",
|
||||
num_args = 0..=1,
|
||||
default_missing_value = "true",
|
||||
default_value_t = false,
|
||||
value_parser = clap::builder::BoolishValueParser::new()
|
||||
)]
|
||||
enable_home_scan: bool,
|
||||
|
||||
/// Allow indexing the filesystem root, off by default for the same reason.
|
||||
/// Also settable via FFF_ENABLE_ROOT_SCAN=1.
|
||||
#[arg(
|
||||
long = "enable-root-scan",
|
||||
env = "FFF_ENABLE_ROOT_SCAN",
|
||||
num_args = 0..=1,
|
||||
default_missing_value = "true",
|
||||
default_value_t = false,
|
||||
value_parser = clap::builder::BoolishValueParser::new()
|
||||
)]
|
||||
enable_root_scan: bool,
|
||||
|
||||
/// Run a health check and print diagnostic information, then exit.
|
||||
#[arg(long = "healthcheck")]
|
||||
pub(crate) healthcheck: bool,
|
||||
|
||||
/// Exit after this many seconds of inactivity. 0 = never exit.
|
||||
/// Timeout of inactivity after which fff mcp will be exited. Even if the parent process
|
||||
/// is alive we don't want to occupy resources on index and file watches if fff is unused
|
||||
#[arg(
|
||||
long = "idle-timeout-secs",
|
||||
env = "FFF_MCP_IDLE_TIMEOUT_SECS",
|
||||
default_value_t = 900
|
||||
default_value_t = 60 * 60
|
||||
)]
|
||||
idle_timeout_secs: u64,
|
||||
}
|
||||
@@ -276,7 +305,8 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
.max_cached_files
|
||||
.map(fff::ContentCacheBudget::new_for_repo),
|
||||
follow_symlinks: args.follow_symlinks,
|
||||
..Default::default()
|
||||
enable_home_dir_scanning: args.enable_home_scan,
|
||||
enable_fs_root_scanning: args.enable_root_scan,
|
||||
},
|
||||
)
|
||||
.map_err(|e| format!("Failed to init file picker: {}", e))?;
|
||||
@@ -317,9 +347,20 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
}
|
||||
};
|
||||
|
||||
if idle_timeout_secs > 0 {
|
||||
let parent_watcher = parent::ParentWatcher::new();
|
||||
match &parent_watcher {
|
||||
Some(watcher) => tracing::info!(
|
||||
"Watching parent process (pid {}); will exit when it dies",
|
||||
watcher.parent_pid()
|
||||
),
|
||||
None => tracing::warn!(
|
||||
"Parent process liveness detection unavailable; idle timeout will exit unconditionally"
|
||||
),
|
||||
}
|
||||
|
||||
if idle_timeout_secs > 0 || parent_watcher.is_some() {
|
||||
last_activity.store(
|
||||
std::time::SystemTime::now()
|
||||
SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| d.as_secs())
|
||||
.unwrap_or(0),
|
||||
@@ -328,9 +369,27 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
|
||||
let last_activity_for_watchdog = last_activity.clone();
|
||||
tokio::spawn(async move {
|
||||
let tick = std::time::Duration::from_secs(60);
|
||||
let tick = watchdog_interval();
|
||||
loop {
|
||||
tokio::time::sleep(tick).await;
|
||||
|
||||
if let Some(ref watcher) = parent_watcher {
|
||||
if !watcher.parent_alive() {
|
||||
tracing::info!(
|
||||
"Parent process (pid {}) exited, shutting down",
|
||||
watcher.parent_pid()
|
||||
);
|
||||
flush_logs_and_exit().await;
|
||||
}
|
||||
// Parent is alive: it owns our lifecycle, never exit on idle
|
||||
// Clients like Codex do not restart MCP servers @see #703
|
||||
continue;
|
||||
}
|
||||
|
||||
if idle_timeout_secs == 0 {
|
||||
continue;
|
||||
}
|
||||
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| d.as_secs())
|
||||
@@ -338,12 +397,8 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
|
||||
let last = last_activity_for_watchdog.load(std::sync::atomic::Ordering::Relaxed);
|
||||
if now.saturating_sub(last) >= idle_timeout_secs {
|
||||
tracing::info!(
|
||||
"Exiting after {}s of inactivity (idle_timeout_secs={})",
|
||||
now.saturating_sub(last),
|
||||
idle_timeout_secs
|
||||
);
|
||||
std::process::exit(0);
|
||||
tracing::info!(?idle_timeout_secs, "Exiting due to inactivity",);
|
||||
flush_logs_and_exit().await;
|
||||
}
|
||||
}
|
||||
});
|
||||
@@ -370,3 +425,20 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// Tracing appender is non blocking, to get full log give it some time before hard exit
|
||||
async fn flush_logs_and_exit() -> ! {
|
||||
tokio::time::sleep(std::time::Duration::from_millis(250)).await;
|
||||
std::process::exit(0);
|
||||
}
|
||||
|
||||
fn watchdog_interval() -> Duration {
|
||||
if cfg!(debug_assertions)
|
||||
&& let Some(milliseconds) = std::env::var("FFF_MCP_TEST_WATCHDOG_INTERVAL_MS")
|
||||
.ok()
|
||||
.and_then(|value| value.parse().ok())
|
||||
{
|
||||
return Duration::from_millis(milliseconds);
|
||||
}
|
||||
Duration::from_secs(60)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
#[cfg(unix)]
|
||||
mod imp {
|
||||
pub struct ParentWatcher {
|
||||
ppid: u32,
|
||||
}
|
||||
|
||||
impl ParentWatcher {
|
||||
pub fn new() -> Option<Self> {
|
||||
let ppid = std::os::unix::process::parent_id();
|
||||
// ppid <= 1 means we were spawned by init and can't detect death
|
||||
(ppid > 1).then_some(Self { ppid })
|
||||
}
|
||||
|
||||
pub fn parent_pid(&self) -> u32 {
|
||||
self.ppid
|
||||
}
|
||||
|
||||
// When the parent dies the kernel reparents us, so getppid() changes.
|
||||
// Race-free and immune to PID reuse, unlike kill(ppid, 0).
|
||||
pub fn parent_alive(&self) -> bool {
|
||||
std::os::unix::process::parent_id() == self.ppid
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
mod imp {
|
||||
use windows_sys::Win32::Foundation::{CloseHandle, HANDLE, INVALID_HANDLE_VALUE, WAIT_TIMEOUT};
|
||||
use windows_sys::Win32::System::Diagnostics::ToolHelp::{
|
||||
CreateToolhelp32Snapshot, PROCESSENTRY32, Process32First, Process32Next, TH32CS_SNAPPROCESS,
|
||||
};
|
||||
use windows_sys::Win32::System::Threading::{
|
||||
GetCurrentProcessId, OpenProcess, PROCESS_SYNCHRONIZE, WaitForSingleObject,
|
||||
};
|
||||
|
||||
pub struct ParentWatcher {
|
||||
handle: HANDLE,
|
||||
ppid: u32,
|
||||
}
|
||||
|
||||
// HANDLE is a raw pointer; it is only ever used via WaitForSingleObject
|
||||
// which is thread-safe, so moving/sharing the watcher across threads is fine.
|
||||
unsafe impl Send for ParentWatcher {}
|
||||
unsafe impl Sync for ParentWatcher {}
|
||||
|
||||
impl ParentWatcher {
|
||||
pub fn new() -> Option<Self> {
|
||||
let ppid = parent_pid_of_current()?;
|
||||
let handle = unsafe { OpenProcess(PROCESS_SYNCHRONIZE, 0, ppid) };
|
||||
if handle.is_null() {
|
||||
return None;
|
||||
}
|
||||
// Holding the handle pins the PID, preventing reuse for the process lifetime
|
||||
Some(Self { handle, ppid })
|
||||
}
|
||||
|
||||
pub fn parent_pid(&self) -> u32 {
|
||||
self.ppid
|
||||
}
|
||||
|
||||
pub fn parent_alive(&self) -> bool {
|
||||
unsafe { WaitForSingleObject(self.handle, 0) == WAIT_TIMEOUT }
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ParentWatcher {
|
||||
fn drop(&mut self) {
|
||||
unsafe { CloseHandle(self.handle) };
|
||||
}
|
||||
}
|
||||
|
||||
fn parent_pid_of_current() -> Option<u32> {
|
||||
unsafe {
|
||||
let snapshot = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0);
|
||||
if snapshot == INVALID_HANDLE_VALUE {
|
||||
return None;
|
||||
}
|
||||
let mut entry: PROCESSENTRY32 = std::mem::zeroed();
|
||||
entry.dwSize = std::mem::size_of::<PROCESSENTRY32>() as u32;
|
||||
let current = GetCurrentProcessId();
|
||||
let mut found = None;
|
||||
if Process32First(snapshot, &mut entry) != 0 {
|
||||
loop {
|
||||
if entry.th32ProcessID == current {
|
||||
found = Some(entry.th32ParentProcessID);
|
||||
break;
|
||||
}
|
||||
if Process32Next(snapshot, &mut entry) == 0 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
CloseHandle(snapshot);
|
||||
found
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub use imp::ParentWatcher;
|
||||
@@ -7,7 +7,6 @@ use fff_query_parser::AiGrepConfig;
|
||||
use rmcp::handler::server::wrapper::Parameters;
|
||||
use rmcp::model::*;
|
||||
use rmcp::{ServerHandler, schemars, tool, tool_handler, tool_router};
|
||||
use std::borrow::Cow;
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
@@ -421,7 +420,12 @@ impl FffServer {
|
||||
/// IMPORTANT: Keep queries SHORT — prefer 1-2 terms max.
|
||||
#[tool(
|
||||
name = "find_files",
|
||||
description = "Fuzzy file search by name. Searches FILE NAMES, not file contents. Use it when you need to find a file, not a definition. Use grep instead for searching code content (definitions, usage patterns). Supports fuzzy matching, path prefixes ('src/'), and glob constraints ('name **/src/*.{ts,tsx} !test/'). IMPORTANT: Keep queries SHORT — prefer 1-2 terms max. Multiple words are a waterfall (each narrows results), NOT OR. If unsure, start broad with 1 term and refine."
|
||||
description = "Fuzzy file search by name. Searches FILE NAMES, not file contents. Use it when you need to find a file, not a definition. Use grep instead for searching code content (definitions, usage patterns). Supports fuzzy matching, path prefixes ('src/'), and glob constraints ('name **/src/*.{ts,tsx} !test/'). IMPORTANT: Keep queries SHORT — prefer 1-2 terms max. Multiple words are a waterfall (each narrows results), NOT OR. If unsure, start broad with 1 term and refine.",
|
||||
annotations(
|
||||
read_only_hint = true,
|
||||
destructive_hint = false,
|
||||
open_world_hint = false
|
||||
)
|
||||
)]
|
||||
fn find_files(
|
||||
&self,
|
||||
@@ -536,7 +540,12 @@ impl FffServer {
|
||||
/// Prefer plain text over regex. Filter files with constraints.
|
||||
#[tool(
|
||||
name = "grep",
|
||||
description = "Search file contents. Search for bare identifiers (e.g. 'InProgressQuote', 'ActorAuth'), NOT code syntax or regex. Filter files with constraints (e.g. '*.rs query', 'src/ query'). Use filename, directory (ending with /) or glob expressions to prefilter. See server instructions for constraint syntax and core rules."
|
||||
description = "Search file contents. Search for bare identifiers (e.g. 'InProgressQuote', 'ActorAuth'), NOT code syntax or regex. Filter files with constraints (e.g. '*.rs query', 'src/ query'). Use filename, directory (ending with /) or glob expressions to prefilter. See server instructions for constraint syntax and core rules.",
|
||||
annotations(
|
||||
read_only_hint = true,
|
||||
destructive_hint = false,
|
||||
open_world_hint = false
|
||||
)
|
||||
)]
|
||||
fn grep(
|
||||
&self,
|
||||
@@ -573,7 +582,12 @@ impl FffServer {
|
||||
/// Patterns are literal text — NEVER escape special characters.
|
||||
#[tool(
|
||||
name = "multi_grep",
|
||||
description = "Search file contents for lines matching ANY of multiple patterns (OR logic). IMPORTANT: This returns files where ANY query matches, NOT all patterns. Patterns are literal text — NEVER escape special characters (no \\( \\) \\. etc). Faster than regex alternation for literal text. See server instructions for constraint syntax."
|
||||
description = "Search file contents for lines matching ANY of multiple patterns (OR logic). IMPORTANT: This returns files where ANY query matches, NOT all patterns. Patterns are literal text — NEVER escape special characters (no \\( \\) \\. etc). Faster than regex alternation for literal text. See server instructions for constraint syntax.",
|
||||
annotations(
|
||||
read_only_hint = true,
|
||||
destructive_hint = false,
|
||||
open_world_hint = false
|
||||
)
|
||||
)]
|
||||
fn multi_grep(
|
||||
&self,
|
||||
@@ -613,61 +627,12 @@ impl FffServer {
|
||||
.ok_or_else(|| ErrorData::internal_error("File picker not initialized", None))?;
|
||||
let patterns_refs: Vec<&str> = params.patterns.iter().map(|s| s.as_str()).collect();
|
||||
|
||||
let parser = fff_query_parser::QueryParser::new(fff_query_parser::AiGrepConfig);
|
||||
let parsed_constraints = parser.parse(constraint_query);
|
||||
let constraints = parsed_constraints.constraints.as_slice();
|
||||
let parser = QueryParser::new(AiGrepConfig);
|
||||
let constraints = parser.parse_constraints(constraint_query);
|
||||
|
||||
let result = picker.multi_grep(&patterns_refs, constraints, &options);
|
||||
let result = picker.multi_grep(&patterns_refs, &constraints, &options);
|
||||
let file_refs: Vec<&FileItem> = result.files.to_vec();
|
||||
|
||||
if result.matches.is_empty() && file_offset == 0 {
|
||||
// Fallback: try individual patterns with plain grep
|
||||
let (fallback_options, _) =
|
||||
make_grep_options(output_mode, GrepMode::PlainText, 0, context);
|
||||
|
||||
let fallback_options = GrepSearchOptions {
|
||||
time_budget_ms: 3000,
|
||||
before_context: 0,
|
||||
..fallback_options
|
||||
};
|
||||
|
||||
for pat in ¶ms.patterns {
|
||||
let full_query: Cow<str> = if !constraint_query.is_empty() {
|
||||
Cow::Owned(format!("{} {}", constraint_query, pat))
|
||||
} else {
|
||||
Cow::Borrowed(pat)
|
||||
};
|
||||
|
||||
let parsed = parser.parse(&full_query);
|
||||
let fb_result = picker.grep(&parsed, &fallback_options);
|
||||
|
||||
if !fb_result.matches.is_empty() {
|
||||
let fb_file_refs: Vec<&FileItem> = fb_result.files.to_vec();
|
||||
let mut cs = self.lock_cursors()?;
|
||||
let text = &GrepFormatter {
|
||||
matches: &fb_result.matches,
|
||||
files: &fb_file_refs,
|
||||
total_matched: fb_result.matches.len(),
|
||||
next_file_offset: fb_result.next_file_offset,
|
||||
output_mode,
|
||||
max_results,
|
||||
show_context: false,
|
||||
auto_expand_defs: auto_expand,
|
||||
picker,
|
||||
}
|
||||
.format(&mut cs);
|
||||
return Ok(CallToolResult::success(vec![Content::text(format!(
|
||||
"0 multi-pattern matches. Plain grep fallback for \"{}\":\n{}",
|
||||
pat, text
|
||||
))]));
|
||||
}
|
||||
}
|
||||
|
||||
return Ok(CallToolResult::success(vec![Content::text(
|
||||
"0 matches.".to_string(),
|
||||
)]));
|
||||
}
|
||||
|
||||
if result.matches.is_empty() {
|
||||
return Ok(CallToolResult::success(vec![Content::text(
|
||||
"0 matches.".to_string(),
|
||||
|
||||
@@ -1,20 +1,14 @@
|
||||
//! Background update checker — compares the embedded build hash against
|
||||
//! the latest GitHub release tag to surface upgrade notices in MCP instructions.
|
||||
|
||||
use std::sync::OnceLock;
|
||||
|
||||
const REPO: &str = "dmtrKovalenko/fff.nvim";
|
||||
const BUILD_HASH: &str = env!("FFF_GIT_HASH");
|
||||
const REPO: &str = "dmtrKovalenko/fff";
|
||||
const CURRENT_VERSION: &str = env!("CARGO_PKG_VERSION");
|
||||
|
||||
/// Holds the result of the update check (empty string = up to date or check failed).
|
||||
static UPDATE_NOTICE: OnceLock<String> = OnceLock::new();
|
||||
|
||||
/// Returns the update notice if the check has completed, empty string otherwise.
|
||||
pub fn get_update_notice() -> &'static str {
|
||||
UPDATE_NOTICE.get().map(|s| s.as_str()).unwrap_or("")
|
||||
}
|
||||
|
||||
/// Kick off the update check in a background thread so it never blocks the server.
|
||||
pub fn spawn_update_check() {
|
||||
std::thread::spawn(|| {
|
||||
let notice = check_latest_release();
|
||||
@@ -22,34 +16,28 @@ pub fn spawn_update_check() {
|
||||
});
|
||||
}
|
||||
|
||||
/// Fetch the latest release tag from GitHub and compare against the build hash.
|
||||
fn check_latest_release() -> String {
|
||||
match fetch_latest_tag() {
|
||||
Ok(tag) => compare_versions(BUILD_HASH, &tag),
|
||||
match fetch_latest_stable_tag() {
|
||||
Ok(tag) => compare_versions(CURRENT_VERSION, &tag),
|
||||
Err(_) => String::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Compare a build hash against a release tag.
|
||||
/// Returns an update notice string, or empty if up-to-date.
|
||||
fn compare_versions(build_hash: &str, release_tag: &str) -> String {
|
||||
fn compare_versions(current_version: &str, release_tag: &str) -> String {
|
||||
let tag = release_tag.trim();
|
||||
if tag.is_empty() || build_hash == "unknown" {
|
||||
return String::new();
|
||||
}
|
||||
|
||||
let our_short = &build_hash[..build_hash.len().min(tag.len())];
|
||||
if our_short == tag {
|
||||
let tag_version = tag.strip_prefix('v').unwrap_or(tag);
|
||||
if tag.is_empty() || tag_version == current_version {
|
||||
return String::new();
|
||||
}
|
||||
|
||||
format!(
|
||||
"\n[fff update available: `curl -fsSL https://raw.githubusercontent.com/{REPO}/main/install-mcp.sh | bash`]\n"
|
||||
"\n[fff update available ({current_version} -> {tag_version}): `curl -fsSL https://raw.githubusercontent.com/{REPO}/main/install-mcp.sh | bash`]\n"
|
||||
)
|
||||
}
|
||||
|
||||
/// Shell out to curl to fetch the latest release tag name from GitHub API.
|
||||
fn fetch_latest_tag() -> Result<String, Box<dyn std::error::Error>> {
|
||||
// Uses /releases/latest — GitHub excludes prereleases here, matching the
|
||||
// stable channel that install-mcp.sh installs from.
|
||||
fn fetch_latest_stable_tag() -> Result<String, Box<dyn std::error::Error>> {
|
||||
let output = std::process::Command::new("curl")
|
||||
.args([
|
||||
"-fsSL",
|
||||
@@ -57,7 +45,7 @@ fn fetch_latest_tag() -> Result<String, Box<dyn std::error::Error>> {
|
||||
"5",
|
||||
"-H",
|
||||
"Accept: application/vnd.github.v3+json",
|
||||
&format!("https://api.github.com/repos/{REPO}/releases?per_page=1"),
|
||||
&format!("https://api.github.com/repos/{REPO}/releases/latest"),
|
||||
])
|
||||
.output()?;
|
||||
|
||||
@@ -66,13 +54,47 @@ fn fetch_latest_tag() -> Result<String, Box<dyn std::error::Error>> {
|
||||
}
|
||||
|
||||
let body = String::from_utf8(output.stdout)?;
|
||||
let releases: Vec<serde_json::Value> = serde_json::from_str(&body)?;
|
||||
let tag = releases
|
||||
.first()
|
||||
.and_then(|r| r.get("tag_name"))
|
||||
let release: serde_json::Value = serde_json::from_str(&body)?;
|
||||
let tag = release
|
||||
.get("tag_name")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
|
||||
Ok(tag)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::compare_versions;
|
||||
|
||||
#[test]
|
||||
fn same_version_with_v_prefix_is_silent() {
|
||||
assert_eq!(compare_versions("0.10.1", "v0.10.1"), "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn same_version_without_v_prefix_is_silent() {
|
||||
assert_eq!(compare_versions("0.10.1", "0.10.1"), "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_tag_is_silent() {
|
||||
assert_eq!(compare_versions("0.10.1", ""), "");
|
||||
assert_eq!(compare_versions("0.10.1", " "), "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn older_current_reports_update() {
|
||||
let notice = compare_versions("0.10.0", "v0.10.1");
|
||||
assert!(notice.contains("0.10.0 -> 0.10.1"), "got: {notice}");
|
||||
assert!(notice.contains("install-mcp.sh"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nightly_tag_never_equals_stable_current() {
|
||||
let notice = compare_versions("0.10.1", "0.10.2-nightly.6a239e9");
|
||||
assert!(!notice.is_empty());
|
||||
assert!(notice.contains("0.10.1 -> 0.10.2-nightly.6a239e9"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,181 @@
|
||||
use std::io::{BufRead, BufReader, Write};
|
||||
use std::process::{Child, ChildStdin, Command, Stdio};
|
||||
use std::sync::mpsc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
const BIN: &str = env!("CARGO_BIN_EXE_fff-mcp");
|
||||
|
||||
#[test]
|
||||
fn stays_alive_while_parent_alive_despite_idle_timeout() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
std::fs::write(dir.path().join("a.txt"), "hello").unwrap();
|
||||
|
||||
let mut child = Command::new(BIN)
|
||||
.arg(dir.path())
|
||||
.args([
|
||||
"--no-update-check",
|
||||
"--no-warmup",
|
||||
"--no-watch",
|
||||
"--idle-timeout-secs",
|
||||
"1",
|
||||
])
|
||||
.arg("--log-file")
|
||||
.arg(dir.path().join("test.log"))
|
||||
.env("FFF_MCP_TEST_WATCHDOG_INTERVAL_MS", "100")
|
||||
.stdin(Stdio::piped())
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::null())
|
||||
.spawn()
|
||||
.unwrap();
|
||||
|
||||
let mut stdin = child.stdin.take().unwrap();
|
||||
let stdout_lines = spawn_line_reader(child.stdout.take().unwrap());
|
||||
do_handshake(&mut stdin, &stdout_lines);
|
||||
|
||||
// Wait past the idle timeout and several watchdog ticks.
|
||||
std::thread::sleep(Duration::from_secs(2));
|
||||
assert!(
|
||||
child.try_wait().unwrap().is_none(),
|
||||
"fff-mcp exited on idle timeout even though its parent is alive"
|
||||
);
|
||||
|
||||
// Closing stdin ends the transport; the server must still shut down cleanly.
|
||||
drop(stdin);
|
||||
wait_for_exit(&mut child, Duration::from_secs(15));
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn exits_when_parent_dies_even_without_idle_timeout() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
std::fs::write(dir.path().join("a.txt"), "hello").unwrap();
|
||||
let log_path = dir.path().join("test.log");
|
||||
let exit_signal = dir.path().join("exit-parent");
|
||||
|
||||
// Intermediary parent: sh backgrounds fff-mcp and waits until the handshake
|
||||
// completes before dying and orphaning it.
|
||||
let mut sh = Command::new("sh")
|
||||
.arg("-c")
|
||||
.arg(
|
||||
// Preserve stdin before POSIX shells assign /dev/null to background jobs.
|
||||
r#"exec 3<&0
|
||||
"$1" "$2" --no-update-check --no-warmup --no-watch \
|
||||
--idle-timeout-secs 0 --log-file "$3" <&3 &
|
||||
while [ ! -e "$4" ]; do sleep 0.1; done"#,
|
||||
)
|
||||
.arg("sh")
|
||||
.arg(BIN)
|
||||
.arg(dir.path())
|
||||
.arg(&log_path)
|
||||
.arg(&exit_signal)
|
||||
.env("FFF_MCP_TEST_WATCHDOG_INTERVAL_MS", "100")
|
||||
.stdin(Stdio::piped())
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::null())
|
||||
.spawn()
|
||||
.unwrap();
|
||||
|
||||
let mut stdin = sh.stdin.take().unwrap();
|
||||
let stdout_lines = spawn_line_reader(sh.stdout.take().unwrap());
|
||||
do_handshake(&mut stdin, &stdout_lines);
|
||||
|
||||
std::fs::write(exit_signal, "").unwrap();
|
||||
sh.wait().unwrap();
|
||||
|
||||
// We still hold the stdin write end, so the only exit path is the parent
|
||||
// liveness check. EOF on stdout means fff-mcp closed it by exiting.
|
||||
let deadline = Instant::now() + Duration::from_secs(5);
|
||||
loop {
|
||||
match stdout_lines.recv_timeout(deadline.saturating_duration_since(Instant::now())) {
|
||||
Ok(_) => continue,
|
||||
Err(mpsc::RecvTimeoutError::Disconnected) => break,
|
||||
Err(mpsc::RecvTimeoutError::Timeout) => {
|
||||
panic!("fff-mcp did not exit within 5s of its parent dying")
|
||||
}
|
||||
}
|
||||
}
|
||||
drop(stdin);
|
||||
|
||||
let logs = read_session_logs(dir.path());
|
||||
assert!(
|
||||
logs.contains("Parent process") && logs.contains("exited, shutting down"),
|
||||
"expected parent-death exit reason in logs, got:\n{}",
|
||||
logs
|
||||
);
|
||||
}
|
||||
|
||||
fn do_handshake(stdin: &mut ChildStdin, stdout_lines: &mpsc::Receiver<String>) {
|
||||
let initialize = serde_json::json!({
|
||||
"jsonrpc": "2.0",
|
||||
"id": 1,
|
||||
"method": "initialize",
|
||||
"params": {
|
||||
"protocolVersion": "2024-11-05",
|
||||
"capabilities": {},
|
||||
"clientInfo": { "name": "parent-liveness-test", "version": "0.0.0" }
|
||||
}
|
||||
});
|
||||
writeln!(stdin, "{}", initialize).unwrap();
|
||||
stdin.flush().unwrap();
|
||||
|
||||
let response = stdout_lines
|
||||
.recv_timeout(Duration::from_secs(30))
|
||||
.expect("no initialize response within 30s");
|
||||
assert!(
|
||||
response.contains("\"serverInfo\""),
|
||||
"unexpected initialize response: {}",
|
||||
response
|
||||
);
|
||||
|
||||
writeln!(
|
||||
stdin,
|
||||
"{}",
|
||||
serde_json::json!({ "jsonrpc": "2.0", "method": "notifications/initialized" })
|
||||
)
|
||||
.unwrap();
|
||||
stdin.flush().unwrap();
|
||||
}
|
||||
|
||||
fn spawn_line_reader(stdout: std::process::ChildStdout) -> mpsc::Receiver<String> {
|
||||
let (tx, rx) = mpsc::channel();
|
||||
std::thread::spawn(move || {
|
||||
for line in BufReader::new(stdout).lines() {
|
||||
match line {
|
||||
Ok(line) => {
|
||||
if tx.send(line).is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
Err(_) => break,
|
||||
}
|
||||
}
|
||||
});
|
||||
rx
|
||||
}
|
||||
|
||||
fn wait_for_exit(child: &mut Child, timeout: Duration) {
|
||||
let deadline = Instant::now() + timeout;
|
||||
while Instant::now() < deadline {
|
||||
if child.try_wait().unwrap().is_some() {
|
||||
return;
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(100));
|
||||
}
|
||||
child.kill().ok();
|
||||
panic!(
|
||||
"fff-mcp did not exit within {:?} after stdin closed",
|
||||
timeout
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
fn read_session_logs(dir: &std::path::Path) -> String {
|
||||
let mut combined = String::new();
|
||||
for entry in std::fs::read_dir(dir).unwrap().flatten() {
|
||||
let name = entry.file_name().to_string_lossy().to_string();
|
||||
if name.starts_with("test") && name.ends_with(".log") {
|
||||
combined.push_str(&std::fs::read_to_string(entry.path()).unwrap_or_default());
|
||||
}
|
||||
}
|
||||
combined
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "fff-nvim"
|
||||
version = "0.10.1"
|
||||
version = "0.10.4"
|
||||
edition = "2024"
|
||||
|
||||
[lints]
|
||||
@@ -15,6 +15,8 @@ crate-type = ["cdylib", "rlib"]
|
||||
default = ["ripgrep"]
|
||||
ripgrep = ["fff/ripgrep", "fff-query-parser/ripgrep", "dep:ignore"]
|
||||
zlob = ["fff/zlob", "fff-query-parser/zlob", "dep:zlob"]
|
||||
# Keep full-rescan accounting in a release build; required by rescan_probe.
|
||||
rescan-stats = ["fff/rescan-stats"]
|
||||
|
||||
[dependencies]
|
||||
# Workspace dependencies
|
||||
@@ -22,10 +24,10 @@ ahash = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
|
||||
# Local crates
|
||||
fff = { package = "fff-search", path = "../fff-core", version = "0.10.1", default-features = false, features = [
|
||||
fff = { package = "fff-search", path = "../fff-core", version = "0.10.4", default-features = false, features = [
|
||||
"mimalloc-collect",
|
||||
] }
|
||||
fff-query-parser = { path = "../fff-query-parser", version = "0.10.1", default-features = false }
|
||||
fff-query-parser = { path = "../fff-query-parser", version = "0.10.4", default-features = false }
|
||||
chrono = { version = "0.4", features = ["serde"] }
|
||||
ctrlc = "3.4.2"
|
||||
git2 = { workspace = true }
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
use fff::file_picker::FilePicker;
|
||||
use fff::{
|
||||
FFFMode, FilePickerOptions, RESCAN_STATS_ENABLED, RescanReason, RescanStats, SharedFilePicker,
|
||||
SharedFrecency,
|
||||
};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
const POLL: Duration = Duration::from_millis(250);
|
||||
|
||||
fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let (base_path, run_for) = parse_args()?;
|
||||
|
||||
if !RESCAN_STATS_ENABLED {
|
||||
return Err(
|
||||
"this build has rescan accounting compiled out; rebuild with \
|
||||
`--features rescan-stats` (or drop `--release`)"
|
||||
.into(),
|
||||
);
|
||||
}
|
||||
|
||||
let picker = SharedFilePicker::default();
|
||||
let frecency = SharedFrecency::noop();
|
||||
|
||||
println!("indexing {base_path} ...");
|
||||
let started = Instant::now();
|
||||
FilePicker::new_with_shared_state(
|
||||
picker.clone(),
|
||||
frecency.clone(),
|
||||
FilePickerOptions {
|
||||
base_path: base_path.clone(),
|
||||
enable_mmap_cache: false,
|
||||
mode: FFFMode::default(),
|
||||
watch: true,
|
||||
..Default::default()
|
||||
},
|
||||
)?;
|
||||
|
||||
if !picker.wait_for_scan(Duration::from_secs(600)) {
|
||||
return Err("timed out waiting for the initial scan".into());
|
||||
}
|
||||
if !picker.wait_for_watcher(Duration::from_secs(600)) {
|
||||
return Err("timed out waiting for the watcher".into());
|
||||
}
|
||||
|
||||
println!(
|
||||
"indexed {} files in {:.2}s; watching for rescan requests.\n",
|
||||
live_files(&picker),
|
||||
started.elapsed().as_secs_f64()
|
||||
);
|
||||
picker.reset_rescan_stats();
|
||||
|
||||
let running = Arc::new(AtomicBool::new(true));
|
||||
let stop = Arc::clone(&running);
|
||||
ctrlc::set_handler(move || stop.store(false, Ordering::SeqCst))?;
|
||||
|
||||
let watching_since = Instant::now();
|
||||
let mut last = RescanStats::default();
|
||||
|
||||
while running.load(Ordering::SeqCst) {
|
||||
std::thread::sleep(POLL);
|
||||
|
||||
let stats = picker.rescan_stats();
|
||||
let delta = stats.since(&last);
|
||||
if delta.total > 0 || delta.throttled > 0 {
|
||||
let now = watching_since.elapsed().as_secs_f64();
|
||||
let files = live_files(&picker);
|
||||
let overflow = overflow_files(&picker);
|
||||
|
||||
for reason in RescanReason::ALL {
|
||||
for _ in 0..delta.count(reason) {
|
||||
println!(
|
||||
"[{now:>8.2}s] request {reason:<21} files={files} overflow={overflow}"
|
||||
);
|
||||
}
|
||||
let suppressed = delta.count_throttled(reason);
|
||||
if suppressed > 0 {
|
||||
println!("[{now:>8.2}s] throttled {reason:<21} x{suppressed}");
|
||||
}
|
||||
}
|
||||
last = stats;
|
||||
}
|
||||
|
||||
if run_for.is_some_and(|limit| watching_since.elapsed() >= limit) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
let elapsed = watching_since.elapsed();
|
||||
let stats = picker.rescan_stats();
|
||||
println!("\n{:.1}s watched", elapsed.as_secs_f64());
|
||||
println!("{stats}");
|
||||
if stats.watcher_triggered() > 0 {
|
||||
println!(
|
||||
"{:.1} watcher rescan requests/minute",
|
||||
stats.watcher_triggered() as f64 / elapsed.as_secs_f64().max(1.0) * 60.0
|
||||
);
|
||||
} else {
|
||||
println!("no full rescans: every change was applied incrementally");
|
||||
}
|
||||
if stats.throttled > 0 {
|
||||
println!(
|
||||
"{} additional request(s) were throttled; {} total requests observed",
|
||||
stats.throttled,
|
||||
stats.total + stats.throttled
|
||||
);
|
||||
}
|
||||
|
||||
if let Ok(mut guard) = picker.write() {
|
||||
guard.take();
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn parse_args() -> Result<(String, Option<Duration>), Box<dyn std::error::Error>> {
|
||||
let mut base_path = None;
|
||||
let mut run_for = None;
|
||||
let mut args = std::env::args().skip(1);
|
||||
|
||||
while let Some(arg) = args.next() {
|
||||
match arg.as_str() {
|
||||
"--seconds" | "-s" => {
|
||||
let value = args.next().ok_or("--seconds needs a value")?;
|
||||
run_for = Some(Duration::from_secs(value.parse()?));
|
||||
}
|
||||
"--help" | "-h" => {
|
||||
println!("usage: rescan_probe [path] [--seconds N]");
|
||||
std::process::exit(0);
|
||||
}
|
||||
other => base_path = Some(other.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
let base_path = match base_path {
|
||||
Some(path) => path,
|
||||
None => std::env::current_dir()?.to_string_lossy().into_owned(),
|
||||
};
|
||||
|
||||
Ok((base_path, run_for))
|
||||
}
|
||||
|
||||
fn live_files(picker: &SharedFilePicker) -> usize {
|
||||
picker
|
||||
.read()
|
||||
.ok()
|
||||
.and_then(|g| g.as_ref().map(|p| p.live_file_count()))
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
fn overflow_files(picker: &SharedFilePicker) -> usize {
|
||||
picker
|
||||
.read()
|
||||
.ok()
|
||||
.and_then(|g| g.as_ref().map(|p| p.get_overflow_files().len()))
|
||||
.unwrap_or(0)
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "fff-python"
|
||||
version = "0.10.1"
|
||||
version = "0.10.4"
|
||||
edition = "2024"
|
||||
|
||||
[lints]
|
||||
@@ -17,7 +17,7 @@ ripgrep = ["fff/ripgrep", "fff-query-parser/ripgrep"]
|
||||
zlob = ["fff/zlob", "fff-query-parser/zlob"]
|
||||
|
||||
[dependencies]
|
||||
fff = { package = "fff-search", path = "../fff-core", version = "0.10.1", default-features = false }
|
||||
fff-query-parser = { path = "../fff-query-parser", version = "0.10.1", default-features = false }
|
||||
fff = { package = "fff-search", path = "../fff-core", version = "0.10.4", default-features = false }
|
||||
fff-query-parser = { path = "../fff-query-parser", version = "0.10.4", default-features = false }
|
||||
git2 = { workspace = true }
|
||||
pyo3 = { version = "0.24.0", features = ["extension-module", "abi3-py310"] }
|
||||
|
||||
@@ -693,15 +693,11 @@ impl FileFinder {
|
||||
}
|
||||
let pattern_refs: Vec<&str> = patterns.iter().map(|s| s.as_str()).collect();
|
||||
|
||||
let parsed_constraints = constraints.as_ref().map(|c| {
|
||||
if picker.mode().is_ai() {
|
||||
QueryParser::new(fff_query_parser::AiGrepConfig).parse(c)
|
||||
} else {
|
||||
fff::grep::parse_grep_query(c)
|
||||
}
|
||||
});
|
||||
let parsed_constraints = constraints
|
||||
.as_ref()
|
||||
.map(|c| QueryParser::new(fff_query_parser::AiGrepConfig).parse_constraints(c));
|
||||
let constraint_refs: &[fff::Constraint<'_>] = match &parsed_constraints {
|
||||
Some(q) => &q.constraints,
|
||||
Some(constraints) => constraints,
|
||||
None => &[],
|
||||
};
|
||||
let options = grep_options(
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "fff-query-parser"
|
||||
version = "0.10.1"
|
||||
version = "0.10.4"
|
||||
edition = "2024"
|
||||
description = "Query parser for fff file finder - includes specific syntax for various constraints like globs, extensions, regex etc"
|
||||
license = "MIT"
|
||||
|
||||
@@ -49,6 +49,14 @@ impl<C: ParserConfig> QueryParser<C> {
|
||||
Self { config }
|
||||
}
|
||||
|
||||
/// Parse a field containing only constraints.
|
||||
pub fn parse_constraints<'a>(&self, query: &'a str) -> ConstraintVec<'a> {
|
||||
query
|
||||
.split_whitespace()
|
||||
.filter_map(|token| parse_token(token, &self.config))
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn parse<'a>(&self, query: &'a str) -> FFFQuery<'a> {
|
||||
let raw_query = query;
|
||||
let config: &C = &self.config;
|
||||
@@ -498,7 +506,7 @@ fn parse_git_status(value: &str) -> Option<Constraint<'_>> {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::{FileSearchConfig, GrepConfig};
|
||||
use crate::{AiGrepConfig, FileSearchConfig, GrepConfig};
|
||||
|
||||
/// File-picker-like config with filename-constraint detection enabled,
|
||||
/// mirroring the Neovim layer's opt-in behavior.
|
||||
@@ -1019,6 +1027,42 @@ mod tests {
|
||||
assert_eq!(result.grep_text(), "pattern");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_standalone_constraints_preserve_directory() {
|
||||
let result = QueryParser::new(AiGrepConfig).parse_constraints("scope-a/");
|
||||
assert_eq!(result.as_slice(), &[Constraint::PathSegment("scope-a")]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_plain_constraints_preserve_directory_only() {
|
||||
let directory = QueryParser::new(GrepConfig).parse_constraints("scope-a/");
|
||||
assert_eq!(directory.as_slice(), &[Constraint::PathSegment("scope-a")]);
|
||||
|
||||
let file = QueryParser::new(GrepConfig).parse_constraints("scope-a/one.txt");
|
||||
assert!(file.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_standalone_constraints_preserve_file() {
|
||||
let result = QueryParser::new(AiGrepConfig).parse_constraints("scope-a/one.txt");
|
||||
assert_eq!(
|
||||
result.as_slice(),
|
||||
&[Constraint::FilePath("scope-a/one.txt")]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_standalone_constraints_preserve_file_without_search_text() {
|
||||
let result = QueryParser::new(AiGrepConfig).parse_constraints("scope-a/ scope-a/one.txt");
|
||||
assert_eq!(
|
||||
result.as_slice(),
|
||||
&[
|
||||
Constraint::PathSegment("scope-a"),
|
||||
Constraint::FilePath("scope-a/one.txt")
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_ai_grep_filename_with_pathsegment_only_promotes_to_text() {
|
||||
// When the ONLY non-text constraints are path-scoping (PathSegment,
|
||||
|
||||
+13
-6
@@ -1,5 +1,5 @@
|
||||
*fff.nvim.txt*
|
||||
For Neovim >= 0.10.0 Last change: 2026 July 16
|
||||
For Neovim >= 0.10.0 Last change: 2026 July 30
|
||||
|
||||
==============================================================================
|
||||
Table of Contents *fff.nvim-table-of-contents*
|
||||
@@ -24,8 +24,9 @@ INSTALLATION ~
|
||||
LAZY.NVIM
|
||||
|
||||
>lua
|
||||
-- Package name changed from `fff.nvim` to `fff`. If you installed fff.nvim before, clean with `:Lazy clean`
|
||||
{
|
||||
'dmtrKovalenko/fff.nvim',
|
||||
'dmtrKovalenko/fff',
|
||||
build = function()
|
||||
-- downloads a prebuilt binary or falls back to cargo build
|
||||
require("fff.download").download_or_build_binary()
|
||||
@@ -59,13 +60,14 @@ LAZY.NVIM
|
||||
VIM.PACK
|
||||
|
||||
>lua
|
||||
vim.pack.add({ 'https://github.com/dmtrKovalenko/fff.nvim' })
|
||||
-- Package name changed from `fff.nvim` to `fff`. If you installed fff.nvim before, clean with `:packdel fff.nvim`
|
||||
vim.pack.add({ 'https://github.com/dmtrKovalenko/fff' })
|
||||
|
||||
vim.api.nvim_create_autocmd('PackChanged', {
|
||||
callback = function(ev)
|
||||
local name, kind = ev.data.spec.name, ev.data.kind
|
||||
if name == 'fff.nvim' and (kind == 'install' or kind == 'update') then
|
||||
if not ev.data.active then vim.cmd.packadd('fff.nvim') end
|
||||
if name == 'fff' and (kind == 'install' or kind == 'update') then
|
||||
if not ev.data.active then vim.cmd.packadd('fff') end
|
||||
require('fff.download').download_or_build_binary()
|
||||
end
|
||||
end,
|
||||
@@ -199,6 +201,11 @@ Defaults are sensible. Override only what you care about.
|
||||
-- Border style for the picker windows. Leave unset (nil) to follow the
|
||||
-- global `vim.o.winborder`; set it to override fff's borders independently.
|
||||
border = nil, -- 'single' | 'double' | 'rounded' | 'solid' | 'shadow' | 'none'
|
||||
-- border = {
|
||||
-- { ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ' },
|
||||
-- { ' ', ' ', ' ', ' ', ' ' },
|
||||
-- },
|
||||
|
||||
flex = { size = 130, wrap = 'top' },
|
||||
min_list_height = 10, -- do not display anything except the list below this threshold
|
||||
show_scrollbar = true,
|
||||
@@ -319,7 +326,7 @@ Both find and grep accept these tokens to refine a query:
|
||||
|
||||
- `git:modified`. One of `modified`, `staged`, `deleted`, `renamed`, `untracked`, `ignored`.
|
||||
- `test/`. Any deeply nested children of `test/`.
|
||||
- `!something`, `!test/`, `!git:modified`. Exclusion.
|
||||
- `!something`, `!test/`, `!git:modified`. Exclusion. Text exclusions need at least 3 alphanumeric-containing characters, so operators like `!=` or `!==` work.
|
||||
- `./**/*.{rs,lua}`. Any valid glob, powered by zlob <https://github.com/dmtrKovalenko/zlob>.
|
||||
|
||||
Grep-only:
|
||||
|
||||
+6
-6
@@ -1,5 +1,5 @@
|
||||
-- Single file Neovim config for testing fff.nvim locally
|
||||
-- Usage: nvim -u /Users/neogoose/dev/fff.nvim/init.lua
|
||||
-- Single file Neovim config for testing fff locally
|
||||
-- Usage: nvim -u /Users/neogoose/dev/fff/init.lua
|
||||
|
||||
-- Set up lazy.nvim plugin manager
|
||||
local lazypath = vim.fn.stdpath('data') .. '/lazy/lazy.nvim'
|
||||
@@ -17,8 +17,8 @@ vim.opt.rtp:prepend(lazypath)
|
||||
|
||||
require('lazy').setup({
|
||||
{
|
||||
dir = '~/dev/fff.nvim',
|
||||
'https://github.com/dmtrKovalenko/fff.nvim',
|
||||
dir = '~/dev/fff',
|
||||
'https://github.com/dmtrKovalenko/fff',
|
||||
build = function()
|
||||
-- this will download prebuild binary or try to use existing rustup toolchain to build from source
|
||||
-- (if you are using lazy you can use gb for rebuilding a plugin if needed)
|
||||
@@ -34,7 +34,7 @@ require('lazy').setup({
|
||||
},
|
||||
config = function()
|
||||
require('fff').setup({
|
||||
-- Configure fff.nvim here
|
||||
-- Configure fff here
|
||||
ui = {
|
||||
width = 0.8,
|
||||
height = 0.8,
|
||||
@@ -59,4 +59,4 @@ vim.keymap.set('n', 'fg', function() require('fff').find_in_git_root() end, { de
|
||||
vim.keymap.set('n', 'fr', function() require('fff').scan_files() end, { desc = 'Rescan files' })
|
||||
vim.keymap.set('n', 'fs', function() require('fff').refresh_git_status() end, { desc = 'Refresh git status' })
|
||||
|
||||
vim.notify('FFF.nvim local config loaded! Press ff', vim.log.levels.INFO)
|
||||
vim.notify('FFF local config loaded! Press ff', vim.log.levels.INFO)
|
||||
|
||||
Generated
+12
-12
@@ -2,11 +2,11 @@
|
||||
"nodes": {
|
||||
"crane": {
|
||||
"locked": {
|
||||
"lastModified": 1773857772,
|
||||
"narHash": "sha256-5xsK26KRHf0WytBtsBnQYC/lTWDhQuT57HJ7SzuqZcM=",
|
||||
"lastModified": 1785782307,
|
||||
"narHash": "sha256-MPaRdVkf6zZP5fCPxYCi8Dr4pZzgmXzg8T9nVEbp3Mw=",
|
||||
"owner": "ipetkov",
|
||||
"repo": "crane",
|
||||
"rev": "b556d7bbae5ff86e378451511873dfd07e4504cd",
|
||||
"rev": "2c71e194474d13de031d729b729c968ddbe3507f",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@@ -51,11 +51,11 @@
|
||||
},
|
||||
"nixpkgs": {
|
||||
"locked": {
|
||||
"lastModified": 1776329215,
|
||||
"narHash": "sha256-a8BYi3mzoJ/AcJP8UldOx8emoPRLeWqALZWu4ZvjPXw=",
|
||||
"lastModified": 1786593342,
|
||||
"narHash": "sha256-smTKQXMLLStzc8zJevMCckbk3My7SvbbLmPYZUJJKW4=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "b86751bc4085f48661017fa226dee99fab6c651b",
|
||||
"rev": "6b5e5b7a6631f065bf6908986990b37d845f847f",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@@ -81,11 +81,11 @@
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1773803479,
|
||||
"narHash": "sha256-GD6i1F2vrSxbsmbS92+8+x3DbHOJ+yrS78Pm4xigW4M=",
|
||||
"lastModified": 1786762605,
|
||||
"narHash": "sha256-iQpYIhInh8gRx+cSnPtX+Yp2Gg9kr2h+MehETCqRgDo=",
|
||||
"owner": "oxalica",
|
||||
"repo": "rust-overlay",
|
||||
"rev": "f17186f52e82ec5cf40920b58eac63b78692ac7c",
|
||||
"rev": "ad8ebb59d84bcf3780c46f107e1d99eb4ca2fe7f",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@@ -134,11 +134,11 @@
|
||||
"systems": "systems_2"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1776789209,
|
||||
"narHash": "sha256-G6B7Q4TXn7MZ1mB+f9rymjsYF5PLWoSvmbxijb/99bw=",
|
||||
"lastModified": 1786753251,
|
||||
"narHash": "sha256-7UGNbG2MwgYCdsknKBL6wMXQQeiTmCiz+JS9zvcmqv4=",
|
||||
"owner": "mitchellh",
|
||||
"repo": "zig-overlay",
|
||||
"rev": "14fe971844e841297ddd2ce9783d6892b467af39",
|
||||
"rev": "a07cc99cee149044f27d95500717161fca2afdf6",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
description = "fff.nvim";
|
||||
description = "fff";
|
||||
|
||||
inputs = {
|
||||
nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
|
||||
@@ -63,6 +63,10 @@
|
||||
];
|
||||
LIBCLANG_PATH = "${pkgs.llvmPackages.libclang.lib}/lib";
|
||||
|
||||
# in zlob we compile by default with -target=native unless CI is detected, because
|
||||
# nix is different we have to make sure that the build target is a baseline cpu
|
||||
CI = "1";
|
||||
|
||||
# Zig 0.16 insists on writing to its global cache even when the
|
||||
# zlob build.rs passes --global-cache-dir. In the nix sandbox $HOME
|
||||
# is /homeless-shelter (unwritable), so redirect to $TMPDIR before
|
||||
|
||||
+2
-2
@@ -4,7 +4,7 @@
|
||||
FFF MCP Server installer for Windows.
|
||||
.DESCRIPTION
|
||||
Pipe usage:
|
||||
irm https://raw.githubusercontent.com/dmtrKovalenko/fff.nvim/main/install-mcp.ps1 | iex
|
||||
irm https://raw.githubusercontent.com/dmtrKovalenko/fff/main/install-mcp.ps1 | iex
|
||||
Direct usage (supports params):
|
||||
iwr https://.../install-mcp.ps1 -OutFile install-mcp.ps1; .\install-mcp.ps1 -Version v0.1.2
|
||||
Env-var fallbacks (for the piped form):
|
||||
@@ -29,7 +29,7 @@ $ErrorActionPreference = 'Stop'
|
||||
# Force TLS 1.2 — PS 5.1 on older Win10 may default to SSL3/TLS1.0 which GitHub rejects.
|
||||
[Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12
|
||||
|
||||
$Repo = 'dmtrKovalenko/fff.nvim'
|
||||
$Repo = 'dmtrKovalenko/fff'
|
||||
$BinaryName = 'fff-mcp'
|
||||
if (-not $InstallDir) { $InstallDir = Join-Path $env:LOCALAPPDATA 'fff-mcp\bin' }
|
||||
|
||||
|
||||
+9
-9
@@ -2,20 +2,20 @@
|
||||
set -eo pipefail
|
||||
|
||||
# FFF MCP Server installer
|
||||
# Usage: curl -fsSL https://raw.githubusercontent.com/dmtrKovalenko/fff.nvim/main/install-mcp.sh | bash
|
||||
# Usage: curl -fsSL https://raw.githubusercontent.com/dmtrKovalenko/fff/main/install-mcp.sh | bash
|
||||
|
||||
REPO="dmtrKovalenko/fff.nvim"
|
||||
REPO="dmtrKovalenko/fff"
|
||||
BINARY_NAME="fff-mcp"
|
||||
INSTALL_DIR="${FFF_MCP_INSTALL_DIR:-$HOME/.local/bin}"
|
||||
|
||||
PINNED_RELEASE_TAG="v0.10.1"
|
||||
PINNED_RELEASE_TAG="v0.10.4"
|
||||
|
||||
SHA256_X86_64_UNKNOWN_LINUX_MUSL="c17637c333afbbbdea4b03cf3e1573240c4147ae121756e363aafa3c9d0efb58"
|
||||
SHA256_AARCH64_UNKNOWN_LINUX_MUSL="2a25019101ee9373327dabd7ac1d4800638688e95b67bfb3bec441bf8d938f28"
|
||||
SHA256_X86_64_APPLE_DARWIN="d3c8d70d47ab80afa2287e1b4546b23f494bb4604b80e5a706a0af714ee25674"
|
||||
SHA256_AARCH64_APPLE_DARWIN="eff6660a9c48e3e1972d557c1003e057e5ff99d6039f5f810671f2123093fdfc"
|
||||
SHA256_X86_64_PC_WINDOWS_MSVC="e341b78464095c349b0c6b0a32b146fd217b542d973917b89645a5aa511640d8"
|
||||
SHA256_AARCH64_PC_WINDOWS_MSVC="c2fc1e55395f314a9493340f11f9b75501a6cd2c1e8fea646d343c7eaf3cd556"
|
||||
SHA256_X86_64_UNKNOWN_LINUX_MUSL="68384121a9173b214190c594ea938cb5a1c52ca3069a3f3b1a0eecc28fd8ea82"
|
||||
SHA256_AARCH64_UNKNOWN_LINUX_MUSL="334ff6f28d5ee47b9652a6c590cf47b908541ff25cb1bd3c7c3303d5f5342c40"
|
||||
SHA256_X86_64_APPLE_DARWIN="821aedd5873292841f32d7584eb68fea085b248e5c970ff5d4f6dca790e9b18c"
|
||||
SHA256_AARCH64_APPLE_DARWIN="f55ce2945cd30fd252a4b1e7d8674ff7ce41b104cc009a8d3d52d08bfccca2c0"
|
||||
SHA256_X86_64_PC_WINDOWS_MSVC="2ff065dc7636c3d50d93a6ea55209760ed382bb6c096c01a3ba7357d6fa6b59e"
|
||||
SHA256_AARCH64_PC_WINDOWS_MSVC="42244ce562fec2ff13d97d819430e028f1f2e5a0d08418f9bd24f97d060741eb"
|
||||
|
||||
expected_sha_for() {
|
||||
case "$1" in
|
||||
|
||||
+1
-1
@@ -9,7 +9,7 @@ local M = {}
|
||||
--- @field min_list_height number
|
||||
--- @field show_scrollbar boolean
|
||||
--- @field path_shorten_strategy string
|
||||
--- @field border? 'single'|'double'|'rounded'|'solid'|'shadow'|'none' Border preset; falls back to `vim.o.winborder` when nil
|
||||
--- @field border? 'single'|'double'|'rounded'|'solid'|'shadow'|'none'|table<string[],string[]> Border preset; falls back to `vim.o.winborder` when nil
|
||||
|
||||
--- @class FffPreviewConfig
|
||||
--- @field enabled boolean
|
||||
|
||||
+84
-35
@@ -3,6 +3,8 @@ if not fuzzy then error('Failed to load fff.fuzzy module. Ensure the Rust backen
|
||||
|
||||
local M = {}
|
||||
|
||||
local fs_scanning_refusal
|
||||
|
||||
---@class fff.core.State
|
||||
local state = {
|
||||
---@type boolean
|
||||
@@ -111,6 +113,13 @@ M.change_indexing_directory = function(new_path)
|
||||
|
||||
local fff_rust = M.ensure_initialized()
|
||||
local config = require('fff.conf').get()
|
||||
|
||||
local refusal = fs_scanning_refusal(vim.tbl_extend('force', config, { base_path = expanded_path }))
|
||||
if refusal then
|
||||
vim.notify('FFF: ' .. refusal, vim.log.levels.WARN)
|
||||
return false
|
||||
end
|
||||
|
||||
local ok, err = pcall(fff_rust.restart_index_in_path, expanded_path, {
|
||||
follow_symlinks = config.follow_symlinks,
|
||||
enable_fs_root_scanning = config.enable_fs_root_scanning,
|
||||
@@ -126,51 +135,91 @@ M.change_indexing_directory = function(new_path)
|
||||
return true
|
||||
end
|
||||
|
||||
--- Reset the file-picker flag so the next `ensure_initialized` recreates the
|
||||
--- Rust picker. Call after `cleanup_file_picker` drops it (`FFFClearCache`);
|
||||
--- otherwise the flag stays set and every later call operates on a dropped
|
||||
--- picker (see #772).
|
||||
M.mark_file_picker_uninitialized = function() state.file_picker_initialized = false end
|
||||
|
||||
M.ensure_initialized = function()
|
||||
if state.initialized then return fuzzy end
|
||||
state.initialized = true
|
||||
|
||||
local config = require('fff.conf').get()
|
||||
if config.logging.enabled then
|
||||
local log_success, log_error =
|
||||
pcall(fuzzy.init_tracing, config.logging.log_file, config.logging.log_level, config.logging.retain_runs)
|
||||
if log_success then
|
||||
M.log_file_path = log_error
|
||||
else
|
||||
vim.notify('Failed to initialize logging: ' .. (tostring(log_error) or 'unknown error'), vim.log.levels.WARN)
|
||||
end
|
||||
end
|
||||
|
||||
local frecency_db_path = config.frecency.db_path or (vim.fn.stdpath('cache') .. '/fff_frecency')
|
||||
local history_db_path = config.history.db_path or (vim.fn.stdpath('data') .. '/fff_history')
|
||||
|
||||
local ok, result = pcall(fuzzy.init_db, frecency_db_path, history_db_path, true)
|
||||
if not ok then vim.notify('Failed to databases: ' .. tostring(result), vim.log.levels.WARN) end
|
||||
|
||||
ok, result = pcall(fuzzy.init_file_picker, config.base_path, {
|
||||
follow_symlinks = config.follow_symlinks,
|
||||
enable_fs_root_scanning = config.enable_fs_root_scanning,
|
||||
enable_home_dir_scanning = config.enable_home_dir_scanning,
|
||||
enable_filename_constraint = config.grep and config.grep.enable_filename_constraint,
|
||||
})
|
||||
if not ok then
|
||||
vim.notify('Failed to initialize file picker: ' .. tostring(result), vim.log.levels.ERROR)
|
||||
-- Refusal gates both one-time setup and (re)creating the picker so we never
|
||||
-- index fs-root / home, even after a cache clear.
|
||||
-- Some folks are complaining that neovim instance is closing if ffi returns error on startup (via lazy=false)
|
||||
-- I can't repro so just precheck on lua side to prevent crashing neovim instance
|
||||
local refusal = fs_scanning_refusal(config)
|
||||
if refusal then
|
||||
state.initialized = true
|
||||
vim.notify('FFF: ' .. refusal, vim.log.levels.WARN)
|
||||
return fuzzy
|
||||
end
|
||||
|
||||
state.file_picker_initialized = true
|
||||
setup_global_autocmds(config)
|
||||
if not state.initialized then
|
||||
state.initialized = true
|
||||
if config.logging.enabled then
|
||||
local log_success, log_error =
|
||||
pcall(fuzzy.init_tracing, config.logging.log_file, config.logging.log_level, config.logging.retain_runs)
|
||||
if log_success then
|
||||
M.log_file_path = log_error
|
||||
else
|
||||
vim.notify('Failed to initialize logging: ' .. (tostring(log_error) or 'unknown error'), vim.log.levels.WARN)
|
||||
end
|
||||
end
|
||||
|
||||
local highlights = require('fff.highlights')
|
||||
highlights.setup()
|
||||
local frecency_db_path = config.frecency.db_path or (vim.fn.stdpath('cache') .. '/fff_frecency')
|
||||
local history_db_path = config.history.db_path or (vim.fn.stdpath('data') .. '/fff_history')
|
||||
|
||||
vim.api.nvim_create_autocmd('ColorScheme', {
|
||||
group = vim.api.nvim_create_augroup('fff_highlights', { clear = true }),
|
||||
callback = function() highlights.setup() end,
|
||||
desc = 'Re-apply FFF highlights on colorscheme change',
|
||||
})
|
||||
local ok, result = pcall(fuzzy.init_db, frecency_db_path, history_db_path, true)
|
||||
if not ok then vim.notify('Failed to databases: ' .. tostring(result), vim.log.levels.WARN) end
|
||||
|
||||
setup_global_autocmds(config)
|
||||
|
||||
local highlights = require('fff.highlights')
|
||||
highlights.setup()
|
||||
|
||||
vim.api.nvim_create_autocmd('ColorScheme', {
|
||||
group = vim.api.nvim_create_augroup('fff_highlights', { clear = true }),
|
||||
callback = function() highlights.setup() end,
|
||||
desc = 'Re-apply FFF highlights on colorscheme change',
|
||||
})
|
||||
end
|
||||
|
||||
-- Recreated whenever the picker was torn down (e.g. `FFFClearCache files`).
|
||||
-- Guarded separately from one-time setup so a cache clear rebuilds the
|
||||
-- picker instead of leaving a dropped one behind (#772).
|
||||
if not state.file_picker_initialized then
|
||||
local ok, result = pcall(fuzzy.init_file_picker, config.base_path, {
|
||||
follow_symlinks = config.follow_symlinks,
|
||||
enable_fs_root_scanning = config.enable_fs_root_scanning,
|
||||
enable_home_dir_scanning = config.enable_home_dir_scanning,
|
||||
enable_filename_constraint = config.grep and config.grep.enable_filename_constraint,
|
||||
})
|
||||
if not ok then
|
||||
vim.notify('Failed to initialize file picker: ' .. tostring(result), vim.log.levels.ERROR)
|
||||
return fuzzy
|
||||
end
|
||||
state.file_picker_initialized = true
|
||||
end
|
||||
|
||||
return fuzzy
|
||||
end
|
||||
|
||||
function fs_scanning_refusal(config)
|
||||
local path = vim.fn.fnamemodify(vim.fn.expand(config.base_path), ':p'):gsub('/+$', '')
|
||||
|
||||
if not config.enable_fs_root_scanning and (path == '' or path:match('^%a:$')) then
|
||||
return 'Refusing to index filesystem root. Set enable_fs_root_scanning = true to override.'
|
||||
end
|
||||
|
||||
if not config.enable_home_dir_scanning then
|
||||
local home = (vim.fn.expand('$HOME') or ''):gsub('/+$', '')
|
||||
if home ~= '' and path == home then
|
||||
return 'Refusing to index home directory. Set enable_home_dir_scanning = true to override.'
|
||||
end
|
||||
end
|
||||
|
||||
return nil
|
||||
end
|
||||
|
||||
return M
|
||||
|
||||
@@ -3,7 +3,7 @@ local system = require('fff.utils.system')
|
||||
local fs_utils = require('fff.utils.fs')
|
||||
local fff_version = require('fff.utils.version')
|
||||
|
||||
local GITHUB_REPO = 'dmtrKovalenko/fff.nvim'
|
||||
local GITHUB_REPO = 'dmtrKovalenko/fff'
|
||||
|
||||
local function get_binary_dir(plugin_dir) return plugin_dir .. '/../target/release' end
|
||||
|
||||
|
||||
@@ -29,6 +29,9 @@ local function get_border_chars(config)
|
||||
if border == nil or border == '' then border = vim.o.winborder end
|
||||
if border == nil or border == '' then border = 'single' end
|
||||
|
||||
if type(border) == 'table' then
|
||||
if #border == 2 and type(border[1]) == 'table' and type(border[2]) == 'table' then return border[1], border[2] end
|
||||
end
|
||||
if BORDER_PRESETS[border] then return BORDER_PRESETS[border], T_JUNCTION_PRESETS[border] end
|
||||
return BORDER_PRESETS.single, T_JUNCTION_PRESETS.single
|
||||
end
|
||||
|
||||
+7
-1
@@ -101,7 +101,13 @@ function M.clear_cache(scope)
|
||||
|
||||
if scope == 'all' or scope == 'files' then
|
||||
local ok, err = pcall(fuzzy.cleanup_file_picker)
|
||||
if not ok then table.insert(errors, 'cleanup file picker: ' .. tostring(err)) end
|
||||
if not ok then
|
||||
table.insert(errors, 'cleanup file picker: ' .. tostring(err))
|
||||
else
|
||||
-- Rust picker is gone; clear the core flag so the next ensure_initialized
|
||||
-- rebuilds it instead of operating on a dropped picker (#772).
|
||||
require('fff.core').mark_file_picker_uninitialized()
|
||||
end
|
||||
end
|
||||
|
||||
if scope == 'all' or scope == 'frecency' then
|
||||
|
||||
@@ -174,8 +174,8 @@ If prebuilt binaries aren't available for your platform:
|
||||
|
||||
```bash
|
||||
# Clone the repository
|
||||
git clone https://github.com/dmtrKovalenko/fff.nvim
|
||||
cd fff.nvim
|
||||
git clone https://github.com/dmtrKovalenko/fff
|
||||
cd fff
|
||||
|
||||
# Build the C library
|
||||
cargo build --release -p fff-c
|
||||
|
||||
@@ -4,19 +4,24 @@
|
||||
"private": false,
|
||||
"description": "High-performance fuzzy file finder for Bun - perfect for LLM agent tools",
|
||||
"type": "module",
|
||||
"main": "src/index.ts",
|
||||
"types": "src/index.ts",
|
||||
"main": "dist/index.js",
|
||||
"types": "dist/index.d.ts",
|
||||
"exports": {
|
||||
".": {
|
||||
"import": "./src/index.ts",
|
||||
"types": "./src/index.ts"
|
||||
}
|
||||
"types": "./dist/index.d.ts",
|
||||
"bun": "./src/index.ts",
|
||||
"import": "./dist/index.js",
|
||||
"default": "./dist/index.js"
|
||||
},
|
||||
"./package.json": "./package.json"
|
||||
},
|
||||
"files": [
|
||||
"dist",
|
||||
"src",
|
||||
"examples"
|
||||
],
|
||||
"scripts": {
|
||||
"build": "bun build ./src/index.ts --format=esm --target=bun --sourcemap=external --outdir dist && tsc -p tsconfig.build.json",
|
||||
"test": "bun test test/",
|
||||
"typecheck": "tsc --noEmit",
|
||||
"demo": "bun ./examples/search.ts"
|
||||
|
||||
@@ -573,16 +573,10 @@ export interface FileFinderApi {
|
||||
glob(pattern: string, options?: GlobOptions): Result<SearchResult>;
|
||||
|
||||
/** Fuzzy directory search. */
|
||||
directorySearch(
|
||||
query: string,
|
||||
options?: DirSearchOptions,
|
||||
): Result<DirSearchResult>;
|
||||
directorySearch(query: string, options?: DirSearchOptions): Result<DirSearchResult>;
|
||||
|
||||
/** Fuzzy search over files and directories interleaved by score. */
|
||||
mixedSearch(
|
||||
query: string,
|
||||
options?: SearchOptions,
|
||||
): Result<MixedSearchResult>;
|
||||
mixedSearch(query: string, options?: SearchOptions): Result<MixedSearchResult>;
|
||||
|
||||
/** Content search (live grep). */
|
||||
grep(query: string, options?: GrepOptions): Result<GrepResult>;
|
||||
@@ -647,10 +641,7 @@ export interface FileFinderApi {
|
||||
* Events are debounced and submitted in batches per 100-ms window at most 128 events.
|
||||
* Gitignored and other ignored files are never triggering watcher.
|
||||
*/
|
||||
watch(
|
||||
callback: WatchBatchCallback,
|
||||
options?: WatchOptions,
|
||||
): Result<WatchUnsubscribe>;
|
||||
watch(callback: WatchBatchCallback, options?: WatchOptions): Result<WatchUnsubscribe>;
|
||||
watch(
|
||||
pattern: string,
|
||||
callback: WatchBatchCallback,
|
||||
|
||||
@@ -0,0 +1,230 @@
|
||||
import { afterEach, describe, expect, mock, test } from "bun:test";
|
||||
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { FileFinder } from "../src/index";
|
||||
|
||||
// Real-native tests for #700/#760. Lives here, not in pi-fff/test: that suite
|
||||
// mocks @ff-labs/fff-bun process-globally and bun module mocks can't be undone.
|
||||
mock.module("@earendil-works/pi-tui", () => ({
|
||||
Text: class Text {
|
||||
text: string;
|
||||
constructor(text: string) {
|
||||
this.text = text;
|
||||
}
|
||||
setText(text: string) {
|
||||
this.text = text;
|
||||
}
|
||||
},
|
||||
}));
|
||||
|
||||
const schema = (type: string) => (options?: unknown) => ({ type, options });
|
||||
mock.module("@sinclair/typebox", () => ({
|
||||
Type: {
|
||||
Array: (items: unknown, options?: unknown) => ({ type: "array", items, options }),
|
||||
Boolean: schema("boolean"),
|
||||
Number: schema("number"),
|
||||
Object: (properties: unknown, options?: unknown) => ({
|
||||
type: "object",
|
||||
properties,
|
||||
options,
|
||||
}),
|
||||
Optional: (value: unknown) => ({ ...(value as object), optional: true }),
|
||||
String: schema("string"),
|
||||
Union: (items: unknown[], options?: unknown) => ({ type: "union", items, options }),
|
||||
},
|
||||
}));
|
||||
|
||||
const { default: fffExtension } = await import("../../pi-fff/src/index");
|
||||
|
||||
// Inject this package as the extension's SDK through the cache hook sdk.ts
|
||||
// already uses for reloads: CI has no node_modules to resolve "@ff-labs/fff-bun"
|
||||
// from pi-fff, and the finder stays the real native one either way.
|
||||
(globalThis as Record<string, unknown>).__fffSdkPromiseGlobal = Promise.resolve({
|
||||
FileFinder,
|
||||
});
|
||||
|
||||
const cleanups: Array<() => void> = [];
|
||||
|
||||
afterEach(() => {
|
||||
while (cleanups.length) cleanups.pop()?.();
|
||||
});
|
||||
|
||||
function makeWorkspace(name: string): string {
|
||||
const dir = mkdtempSync(join(tmpdir(), `pi-fff-native-${name}-`));
|
||||
cleanups.push(() => rmSync(dir, { recursive: true, force: true }));
|
||||
writeFileSync(join(dir, "alpha.ts"), "export const alpha = 1;\n");
|
||||
writeFileSync(join(dir, "beta.ts"), "export const beta = 2;\n");
|
||||
mkdirSync(join(dir, "src"));
|
||||
writeFileSync(join(dir, "src", "gamma.ts"), "export const gamma = 3;\n");
|
||||
return dir;
|
||||
}
|
||||
|
||||
function makeDbPaths() {
|
||||
const root = mkdtempSync(join(tmpdir(), "pi-fff-native-dbs-"));
|
||||
cleanups.push(() => rmSync(root, { recursive: true, force: true }));
|
||||
return { frecencyDbPath: join(root, "frecency"), historyDbPath: join(root, "history") };
|
||||
}
|
||||
|
||||
function createFinder(options: Parameters<typeof FileFinder.create>[0]) {
|
||||
const result = FileFinder.create(options);
|
||||
if (result.ok) {
|
||||
const finder = result.value;
|
||||
cleanups.push(() => {
|
||||
if (!finder.isDestroyed) finder.destroy();
|
||||
});
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
type SearchOk = { ok: true; value: { items: Array<{ fileName: string }> } };
|
||||
type SearchResult = SearchOk | { ok: false; error: string };
|
||||
|
||||
function fileNames(
|
||||
finder: { fileSearch: (q: string, o: { pageSize: number }) => SearchResult },
|
||||
query: string,
|
||||
): string[] {
|
||||
const search = finder.fileSearch(query, { pageSize: 10 });
|
||||
expect(search.ok).toBe(true);
|
||||
return search.ok ? search.value.items.map((i) => i.fileName) : [];
|
||||
}
|
||||
|
||||
describe("fff-bun: many finders share one LMDB env per path (#700/#760)", () => {
|
||||
test("a second finder on the same db paths works and searches", async () => {
|
||||
const dbs = makeDbPaths();
|
||||
const main = createFinder({ basePath: makeWorkspace("main"), ...dbs });
|
||||
expect(main.ok).toBe(true);
|
||||
if (!main.ok) return;
|
||||
await main.value.waitForScan(15_000);
|
||||
expect(fileNames(main.value, "alpha")).toContain("alpha.ts");
|
||||
|
||||
// The createAgentSession scenario: same process, same db paths.
|
||||
const sub = createFinder({ basePath: makeWorkspace("subagent"), ...dbs });
|
||||
expect(sub.ok).toBe(true);
|
||||
if (!sub.ok) return;
|
||||
await sub.value.waitForScan(15_000);
|
||||
expect(fileNames(sub.value, "gamma")).toContain("gamma.ts");
|
||||
}, 30_000);
|
||||
|
||||
test("destroying one finder keeps the shared env alive for the other", async () => {
|
||||
const dbs = makeDbPaths();
|
||||
const first = createFinder({ basePath: makeWorkspace("first"), ...dbs });
|
||||
const second = createFinder({ basePath: makeWorkspace("second"), ...dbs });
|
||||
expect(first.ok).toBe(true);
|
||||
expect(second.ok).toBe(true);
|
||||
if (!first.ok || !second.ok) return;
|
||||
|
||||
first.value.destroy();
|
||||
await second.value.waitForScan(15_000);
|
||||
expect(fileNames(second.value, "alpha")).toContain("alpha.ts");
|
||||
|
||||
// And once the survivor is gone too, the paths are reusable.
|
||||
second.value.destroy();
|
||||
const third = createFinder({ basePath: makeWorkspace("third"), ...dbs });
|
||||
expect(third.ok).toBe(true);
|
||||
}, 30_000);
|
||||
|
||||
test("a db-less aux finder coexists with the main finder (#700)", async () => {
|
||||
const main = createFinder({ basePath: makeWorkspace("main"), ...makeDbPaths() });
|
||||
expect(main.ok).toBe(true);
|
||||
|
||||
const aux = createFinder({ basePath: makeWorkspace("aux") });
|
||||
expect(aux.ok).toBe(true);
|
||||
if (!aux.ok) return;
|
||||
await aux.value.waitForScan(15_000);
|
||||
expect(fileNames(aux.value, "gamma")).toContain("gamma.ts");
|
||||
}, 30_000);
|
||||
});
|
||||
|
||||
type EventHandler = (...args: unknown[]) => unknown;
|
||||
type RegisteredTool = {
|
||||
name: string;
|
||||
execute: (
|
||||
toolCallId: string,
|
||||
params: unknown,
|
||||
signal?: AbortSignal,
|
||||
) => Promise<unknown>;
|
||||
};
|
||||
|
||||
function startSession(
|
||||
cwd: string,
|
||||
dbs: { frecencyDbPath: string; historyDbPath: string },
|
||||
) {
|
||||
const events = new Map<string, EventHandler>();
|
||||
const tools = new Map<string, RegisteredTool>();
|
||||
const notifications: Array<{ message: string; level?: string }> = [];
|
||||
|
||||
const flags: Record<string, unknown> = {
|
||||
"fff-frecency-db": dbs.frecencyDbPath,
|
||||
"fff-history-db": dbs.historyDbPath,
|
||||
};
|
||||
|
||||
const pi = {
|
||||
getFlag: (name: string) => flags[name],
|
||||
on: (event: string, handler: EventHandler) => events.set(event, handler),
|
||||
registerCommand: () => undefined,
|
||||
registerFlag: () => undefined,
|
||||
registerTool: (tool: RegisteredTool) => tools.set(tool.name, tool),
|
||||
appendEntry: () => undefined,
|
||||
};
|
||||
|
||||
const ctx = {
|
||||
cwd,
|
||||
ui: {
|
||||
notify: (message: string, level?: string) => notifications.push({ message, level }),
|
||||
setStatus: () => undefined,
|
||||
},
|
||||
};
|
||||
|
||||
fffExtension(pi as never);
|
||||
cleanups.push(() => {
|
||||
void events.get("session_shutdown")?.({}, undefined);
|
||||
});
|
||||
|
||||
return {
|
||||
start: async () => events.get("session_start")?.({ reason: "startup" }, ctx),
|
||||
shutdown: async () => events.get("session_shutdown")?.({}, undefined),
|
||||
find: async (pattern: string, params?: Record<string, unknown>) =>
|
||||
JSON.stringify(
|
||||
await tools.get("fffind")?.execute("test-call", { pattern, ...params }),
|
||||
),
|
||||
errors: () => notifications.filter((n) => n.level === "error").map((n) => n.message),
|
||||
};
|
||||
}
|
||||
|
||||
describe("pi-fff: in-process double activation works (#760)", () => {
|
||||
test("two sessions in one process both search against the same dbs", async () => {
|
||||
const dbs = makeDbPaths();
|
||||
const first = startSession(makeWorkspace("session1"), dbs);
|
||||
await first.start();
|
||||
expect(first.errors()).toEqual([]);
|
||||
expect(await first.find("alpha")).toContain("alpha.ts");
|
||||
|
||||
// What createAgentSession does: activate the extension again in-process.
|
||||
const second = startSession(makeWorkspace("session2"), dbs);
|
||||
await second.start();
|
||||
expect(second.errors()).toEqual([]);
|
||||
expect(await second.find("gamma")).toContain("gamma.ts");
|
||||
|
||||
// And the first session keeps working alongside it.
|
||||
expect(await first.find("beta")).toContain("beta.ts");
|
||||
|
||||
await second.shutdown();
|
||||
await first.shutdown();
|
||||
}, 40_000);
|
||||
|
||||
test("aux finder over an external root shares the session dbs (#700)", async () => {
|
||||
const dbs = makeDbPaths();
|
||||
const session = startSession(makeWorkspace("aux-session"), dbs);
|
||||
await session.start();
|
||||
expect(session.errors()).toEqual([]);
|
||||
|
||||
// An absolute out-of-workspace path constraint routes to an aux finder,
|
||||
// which now opens the same frecency/history LMDB paths as the main finder.
|
||||
const external = makeWorkspace("aux-external");
|
||||
expect(await session.find("gamma", { path: external })).toContain("gamma.ts");
|
||||
expect(session.errors()).toEqual([]);
|
||||
|
||||
await session.shutdown();
|
||||
}, 40_000);
|
||||
});
|
||||
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"extends": "./tsconfig.json",
|
||||
"compilerOptions": {
|
||||
"rootDir": "./src",
|
||||
"outDir": "./dist",
|
||||
"emitDeclarationOnly": true,
|
||||
"noEmit": false
|
||||
},
|
||||
"include": ["src/**/*"]
|
||||
}
|
||||
@@ -152,8 +152,8 @@ If prebuilt binaries aren't available for your platform:
|
||||
|
||||
```bash
|
||||
# Clone the repository
|
||||
git clone https://github.com/dmtrKovalenko/fff.nvim
|
||||
cd fff.nvim
|
||||
git clone https://github.com/dmtrKovalenko/fff
|
||||
cd fff
|
||||
|
||||
# Build the C library
|
||||
cargo build --release -p fff-c
|
||||
|
||||
@@ -4,19 +4,26 @@
|
||||
"private": false,
|
||||
"description": "High-performance fuzzy file finder for Node.js - perfect for LLM agent tools",
|
||||
"type": "module",
|
||||
"main": "dist/src/index.js",
|
||||
"types": "dist/src/index.d.ts",
|
||||
"main": "dist/index.cjs",
|
||||
"module": "dist/index.js",
|
||||
"types": "dist/index.d.ts",
|
||||
"exports": {
|
||||
".": {
|
||||
"import": "./dist/src/index.js",
|
||||
"types": "./dist/src/index.d.ts"
|
||||
}
|
||||
"types": "./dist/index.d.ts",
|
||||
"import": "./dist/index.js",
|
||||
"require": "./dist/index.cjs",
|
||||
"default": "./dist/index.js"
|
||||
},
|
||||
"./package.json": "./package.json"
|
||||
},
|
||||
"files": [
|
||||
"dist"
|
||||
],
|
||||
"scripts": {
|
||||
"build": "tsc",
|
||||
"build": "npm run build:esm && npm run build:cjs && npm run build:types",
|
||||
"build:esm": "bun build ./src/index.ts --format=esm --target=node --external ffi-rs --sourcemap=external --outdir dist",
|
||||
"build:cjs": "bun build ./src/index.ts --format=cjs --target=node --external ffi-rs --sourcemap=external --outdir dist --entry-naming \"[name].cjs\"",
|
||||
"build:types": "tsc -p tsconfig.build.json",
|
||||
"test": "node test/e2e.mjs && node test/watch.mjs",
|
||||
"typecheck": "tsc --noEmit"
|
||||
},
|
||||
|
||||
@@ -16,6 +16,9 @@ import { getLibFilename, getNpmPackageName } from "./platform.js";
|
||||
* Get the current file's directory
|
||||
*/
|
||||
function getCurrentDir(): string {
|
||||
// CJS build: import.meta.url is inlined at bundle time, __dirname is the truth
|
||||
if (typeof __dirname !== "undefined") return __dirname;
|
||||
|
||||
const url = import.meta.url;
|
||||
|
||||
if (url.startsWith("file://")) {
|
||||
@@ -30,7 +33,7 @@ function getCurrentDir(): string {
|
||||
function getPackageDir(): string {
|
||||
const currentDir = getCurrentDir();
|
||||
// In dev: src/ -> package root
|
||||
// In dist: dist/src/ -> package root
|
||||
// In dist: dist/ -> package root
|
||||
// We look for package.json to find the actual root
|
||||
let dir = currentDir;
|
||||
for (let i = 0; i < 5; i++) {
|
||||
|
||||
@@ -573,16 +573,10 @@ export interface FileFinderApi {
|
||||
glob(pattern: string, options?: GlobOptions): Result<SearchResult>;
|
||||
|
||||
/** Fuzzy directory search. */
|
||||
directorySearch(
|
||||
query: string,
|
||||
options?: DirSearchOptions,
|
||||
): Result<DirSearchResult>;
|
||||
directorySearch(query: string, options?: DirSearchOptions): Result<DirSearchResult>;
|
||||
|
||||
/** Fuzzy search over files and directories interleaved by score. */
|
||||
mixedSearch(
|
||||
query: string,
|
||||
options?: SearchOptions,
|
||||
): Result<MixedSearchResult>;
|
||||
mixedSearch(query: string, options?: SearchOptions): Result<MixedSearchResult>;
|
||||
|
||||
/** Content search (live grep). */
|
||||
grep(query: string, options?: GrepOptions): Result<GrepResult>;
|
||||
@@ -647,10 +641,7 @@ export interface FileFinderApi {
|
||||
* Events are debounced and submitted in batches per 100-ms window at most 128 events.
|
||||
* Gitignored and other ignored files are never triggering watcher.
|
||||
*/
|
||||
watch(
|
||||
callback: WatchBatchCallback,
|
||||
options?: WatchOptions,
|
||||
): Result<WatchUnsubscribe>;
|
||||
watch(callback: WatchBatchCallback, options?: WatchOptions): Result<WatchUnsubscribe>;
|
||||
watch(
|
||||
pattern: string,
|
||||
callback: WatchBatchCallback,
|
||||
|
||||
@@ -246,11 +246,7 @@ function readResultEnvelope(
|
||||
paramsValue: unknown[],
|
||||
): { rawPtr: JsExternal; struct: FffResultRaw } | Result<never> {
|
||||
loadLibrary();
|
||||
const { rawPtr, struct: structData } = callRaw(
|
||||
funcName,
|
||||
paramsType,
|
||||
paramsValue,
|
||||
);
|
||||
const { rawPtr, struct: structData } = callRaw(funcName, paramsType, paramsValue);
|
||||
|
||||
if (structData.success === 0) {
|
||||
const errorStr = readCString(structData.error);
|
||||
@@ -328,8 +324,7 @@ function callJsonResult<T>(
|
||||
if (isNullPointer(handlePtr)) return { ok: true, value: undefined as T };
|
||||
const jsonStr = readCString(handlePtr);
|
||||
freeString(handlePtr);
|
||||
if (jsonStr === null || jsonStr === "")
|
||||
return { ok: true, value: undefined as T };
|
||||
if (jsonStr === null || jsonStr === "") return { ok: true, value: undefined as T };
|
||||
try {
|
||||
return { ok: true, value: snakeToCamel(JSON.parse(jsonStr)) as T };
|
||||
} catch {
|
||||
@@ -849,16 +844,10 @@ function readGrepMatchFromRaw(raw: FffGrepMatchRaw): GrepMatch {
|
||||
match.fuzzyScore = raw.fuzzy_score;
|
||||
}
|
||||
if (raw.context_before_count > 0) {
|
||||
match.contextBefore = readCStringArray(
|
||||
raw.context_before,
|
||||
raw.context_before_count,
|
||||
);
|
||||
match.contextBefore = readCStringArray(raw.context_before, raw.context_before_count);
|
||||
}
|
||||
if (raw.context_after_count > 0) {
|
||||
match.contextAfter = readCStringArray(
|
||||
raw.context_after,
|
||||
raw.context_after_count,
|
||||
);
|
||||
match.contextAfter = readCStringArray(raw.context_after, raw.context_after_count);
|
||||
}
|
||||
if (raw.is_definition !== 0) {
|
||||
match.isDefinition = true;
|
||||
@@ -927,8 +916,7 @@ function parseGrepResult(rawPtr: JsExternal): Result<GrepResult> {
|
||||
totalFilesSearched: gr.total_files_searched,
|
||||
totalFiles: gr.total_files,
|
||||
filteredFileCount: gr.filtered_file_count,
|
||||
nextCursor:
|
||||
gr.next_file_offset > 0 ? createGrepCursor(gr.next_file_offset) : null,
|
||||
nextCursor: gr.next_file_offset > 0 ? createGrepCursor(gr.next_file_offset) : null,
|
||||
};
|
||||
if (regexFallbackError) {
|
||||
grepResult.regexFallbackError = regexFallbackError;
|
||||
@@ -1280,14 +1268,7 @@ export function ffiGlob(
|
||||
DataType.U32, // page_index
|
||||
DataType.U32, // page_size
|
||||
],
|
||||
paramsValue: [
|
||||
handle,
|
||||
pattern,
|
||||
currentFile,
|
||||
maxThreads,
|
||||
pageIndex,
|
||||
pageSize,
|
||||
],
|
||||
paramsValue: [handle, pattern, currentFile, maxThreads, pageIndex, pageSize],
|
||||
freeResultMemory: false,
|
||||
}) as JsExternal;
|
||||
|
||||
@@ -1319,14 +1300,7 @@ export function ffiSearchDirectories(
|
||||
DataType.U32, // page_index
|
||||
DataType.U32, // page_size
|
||||
],
|
||||
paramsValue: [
|
||||
handle,
|
||||
query,
|
||||
currentFile ?? "",
|
||||
maxThreads,
|
||||
pageIndex,
|
||||
pageSize,
|
||||
],
|
||||
paramsValue: [handle, query, currentFile ?? "", maxThreads, pageIndex, pageSize],
|
||||
freeResultMemory: false,
|
||||
}) as JsExternal;
|
||||
|
||||
@@ -1545,11 +1519,7 @@ export function ffiGetScanProgress(handle: NativeHandle): Result<{
|
||||
isWarmupComplete: boolean;
|
||||
}> {
|
||||
loadLibrary();
|
||||
const res = readResultEnvelope(
|
||||
"fff_get_scan_progress",
|
||||
[DataType.External],
|
||||
[handle],
|
||||
);
|
||||
const res = readResultEnvelope("fff_get_scan_progress", [DataType.External], [handle]);
|
||||
if ("ok" in res) return res;
|
||||
|
||||
const handlePtr = res.struct.handle;
|
||||
@@ -1584,10 +1554,7 @@ export function ffiGetScanProgress(handle: NativeHandle): Result<{
|
||||
/**
|
||||
* Wait for a tree scan to complete.
|
||||
*/
|
||||
export function ffiWaitForScan(
|
||||
handle: NativeHandle,
|
||||
timeoutMs: number,
|
||||
): Result<boolean> {
|
||||
export function ffiWaitForScan(handle: NativeHandle, timeoutMs: number): Result<boolean> {
|
||||
return callBoolResult(
|
||||
"fff_wait_for_scan",
|
||||
[DataType.External, DataType.U64],
|
||||
@@ -1598,10 +1565,7 @@ export function ffiWaitForScan(
|
||||
/**
|
||||
* Restart index in new path.
|
||||
*/
|
||||
export function ffiRestartIndex(
|
||||
handle: NativeHandle,
|
||||
newPath: string,
|
||||
): Result<void> {
|
||||
export function ffiRestartIndex(handle: NativeHandle, newPath: string): Result<void> {
|
||||
return callVoidResult(
|
||||
"fff_restart_index",
|
||||
[DataType.External, DataType.String],
|
||||
@@ -1772,8 +1736,7 @@ function ensureWatchTrampoline(): JsExternal {
|
||||
// fff watcher uses a single cross-boundary FFI callback to deliver all events which we then manually
|
||||
// mapping to the user's javascript functions
|
||||
function ensureWatchCallbackRegistered(handle: NativeHandle): Result<void> {
|
||||
if (watchInstances.has(handle as unknown))
|
||||
return { ok: true, value: undefined };
|
||||
if (watchInstances.has(handle as unknown)) return { ok: true, value: undefined };
|
||||
const trampoline = ensureWatchTrampoline();
|
||||
const registered = callVoidResult(
|
||||
"fff_set_watch_callback",
|
||||
@@ -1785,11 +1748,7 @@ function ensureWatchCallbackRegistered(handle: NativeHandle): Result<void> {
|
||||
}
|
||||
|
||||
function releaseWatchTrampolineIfIdle(): void {
|
||||
if (
|
||||
watchHandlers.size > 0 ||
|
||||
watchInstances.size > 0 ||
|
||||
watchTrampoline === null
|
||||
)
|
||||
if (watchHandlers.size > 0 || watchInstances.size > 0 || watchTrampoline === null)
|
||||
return;
|
||||
freePointer({
|
||||
paramsType: [WATCH_TRAMPOLINE_TYPE],
|
||||
@@ -1835,10 +1794,7 @@ export function ffiWatch(
|
||||
* this returns the callback can never run again (a late native tail batch
|
||||
* misses the map lookup and is dropped).
|
||||
*/
|
||||
export function ffiUnwatch(
|
||||
handle: NativeHandle,
|
||||
watchId: number,
|
||||
): Result<boolean> {
|
||||
export function ffiUnwatch(handle: NativeHandle, watchId: number): Result<boolean> {
|
||||
const result = callBoolResult(
|
||||
"fff_unwatch",
|
||||
[DataType.External, DataType.U64],
|
||||
|
||||
@@ -2,13 +2,13 @@ import { after, before, describe, it } from "node:test";
|
||||
import { strict as assert } from "node:assert";
|
||||
import { dirname, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { FileFinder } from "../dist/src/index.js";
|
||||
import { FileFinder } from "../dist/index.js";
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const REPO_ROOT = resolve(__dirname, "..", "..", "..");
|
||||
const normalizePath = (p) => p.replace(/\\/g, "/");
|
||||
|
||||
/** @type {import("../dist/src/finder.js").FileFinder | null} */
|
||||
/** @type {import("../dist/finder.js").FileFinder | null} */
|
||||
let finder = null;
|
||||
|
||||
describe("fff-node", { concurrency: 1 }, () => {
|
||||
|
||||
@@ -13,7 +13,7 @@ import { existsSync } from "node:fs";
|
||||
import { dirname, resolve } from "node:path";
|
||||
import process from "node:process";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { FileFinder } from "../dist/src/index.js";
|
||||
import { FileFinder } from "../dist/index.js";
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const REPO_ROOT = resolve(__dirname, "..", "..", "..");
|
||||
|
||||
@@ -15,7 +15,7 @@ import { mkdtempSync, realpathSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join, sep } from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
import { FileFinder } from "../dist/src/index.js";
|
||||
import { FileFinder } from "../dist/index.js";
|
||||
|
||||
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
@@ -33,7 +33,7 @@ async function waitFor(cond, timeoutMs = 10_000) {
|
||||
/** All events delivered to a batch-callback mock, flattened across calls. */
|
||||
const deliveredEvents = (fn) => fn.mock.calls.flatMap((call) => call.arguments[0]);
|
||||
|
||||
/** @type {import("../dist/src/finder.js").FileFinder | null} */
|
||||
/** @type {import("../dist/finder.js").FileFinder | null} */
|
||||
let finder = null;
|
||||
/** @type {string} */
|
||||
let baseDir = "";
|
||||
@@ -237,15 +237,21 @@ describe("fff-node watch", { concurrency: 1 }, () => {
|
||||
import { mkdtempSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { FileFinder } from ${JSON.stringify(new URL("../dist/src/index.js", import.meta.url).href)};
|
||||
import { FileFinder } from ${JSON.stringify(new URL("../dist/index.js", import.meta.url).href)};
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "fff-watch-exit-"));
|
||||
writeFileSync(join(dir, "seed.txt"), "seed");
|
||||
const created = FileFinder.create({ basePath: dir });
|
||||
if (!created.ok) throw new Error(created.error);
|
||||
const finder = created.value;
|
||||
await finder.waitForScan(10_000);
|
||||
const sub = finder.watch("**/*.txt", () => {});
|
||||
const wait = await finder.waitForScan(10_000);
|
||||
if (!wait.ok || !wait.value) throw new Error("waitForScan failed");
|
||||
const deadline = Date.now() + 2_000;
|
||||
let sub = finder.watch("**/*.txt", () => {});
|
||||
while (!sub.ok && Date.now() < deadline) {
|
||||
await new Promise((r) => setTimeout(r, 50));
|
||||
sub = finder.watch("**/*.txt", () => {});
|
||||
}
|
||||
if (!sub.ok) throw new Error(sub.error);
|
||||
await new Promise((r) => setTimeout(r, 300));
|
||||
sub.value();
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"extends": "./tsconfig.json",
|
||||
"compilerOptions": {
|
||||
"rootDir": "./src",
|
||||
"outDir": "./dist",
|
||||
"emitDeclarationOnly": true,
|
||||
"noEmit": false
|
||||
},
|
||||
"include": ["src/**/*"]
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
# fff-search
|
||||
|
||||
Python bindings for [FFF (Fast File Finder)](https://github.com/dmtrKovalenko/fff.nvim), built with [PyO3](https://pyo3.rs/) and [Maturin](https://www.maturin.rs/). Install with `pip install fff-search`; import as `fff`.
|
||||
Python bindings for [FFF (Fast File Finder)](https://github.com/dmtrKovalenko/fff), built with [PyO3](https://pyo3.rs/) and [Maturin](https://www.maturin.rs/). Install with `pip install fff-search`; import as `fff`.
|
||||
|
||||
## Requirements
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[project]
|
||||
name = "fff-search"
|
||||
version = "0.10.1"
|
||||
version = "0.10.4"
|
||||
description = "Python bindings for FFF (Fast File Finder)"
|
||||
readme = "README.md"
|
||||
license = { text = "MIT" }
|
||||
@@ -15,8 +15,8 @@ dev = [
|
||||
]
|
||||
|
||||
[project.urls]
|
||||
Repository = "https://github.com/dmtrKovalenko/fff.nvim"
|
||||
Issues = "https://github.com/dmtrKovalenko/fff.nvim/issues"
|
||||
Repository = "https://github.com/dmtrKovalenko/fff"
|
||||
Issues = "https://github.com/dmtrKovalenko/fff/issues"
|
||||
|
||||
[build-system]
|
||||
requires = ["maturin>=1.0,<2.0"]
|
||||
|
||||
@@ -50,7 +50,7 @@ class FileFinder(_FileFinder):
|
||||
return True
|
||||
|
||||
|
||||
__version__ = "0.10.1"
|
||||
__version__ = "0.10.4"
|
||||
|
||||
__all__ = [
|
||||
"FFFException",
|
||||
|
||||
Generated
+1
-1
@@ -34,7 +34,7 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "fff-search"
|
||||
version = "0.10.1"
|
||||
version = "0.10.4"
|
||||
source = { editable = "." }
|
||||
|
||||
[package.optional-dependencies]
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# @ff-labs/pi-fff
|
||||
|
||||
A [pi](https://github.com/badlogic/pi-mono) extension that replaces the built-in `find` and `grep` tools with [FFF](https://github.com/dmtrKovalenko/fff.nvim) — a Rust-native, SIMD-accelerated file finder with built-in memory.
|
||||
A [pi](https://github.com/badlogic/pi-mono) extension that replaces the built-in `find` and `grep` tools with [FFF](https://github.com/dmtrKovalenko/fff) — a Rust-native, SIMD-accelerated file finder with built-in memory.
|
||||
|
||||
## What it does
|
||||
|
||||
@@ -44,20 +44,20 @@ pi install -l npm:@ff-labs/pi-fff
|
||||
**Via git:**
|
||||
|
||||
```bash
|
||||
pi install git:github.com/dmtrKovalenko/fff.nvim
|
||||
pi install git:github.com/dmtrKovalenko/fff
|
||||
```
|
||||
|
||||
Pin to a release:
|
||||
|
||||
```bash
|
||||
pi install git:github.com/dmtrKovalenko/fff.nvim@v0.3.0
|
||||
pi install git:github.com/dmtrKovalenko/fff@v0.3.0
|
||||
```
|
||||
|
||||
### Local development / manual install
|
||||
|
||||
```bash
|
||||
git clone https://github.com/dmtrKovalenko/fff.nvim.git
|
||||
cd fff.nvim/packages/pi-fff
|
||||
git clone https://github.com/dmtrKovalenko/fff.git
|
||||
cd fff/packages/pi-fff
|
||||
npm install
|
||||
```
|
||||
|
||||
@@ -65,14 +65,14 @@ Then add to your pi `settings.json`:
|
||||
|
||||
```json
|
||||
{
|
||||
"extensions": ["/path/to/fff.nvim/packages/pi-fff/src/index.ts"]
|
||||
"extensions": ["/path/to/fff/packages/pi-fff/src/index.ts"]
|
||||
}
|
||||
```
|
||||
|
||||
Or test directly:
|
||||
|
||||
```bash
|
||||
pi -e /path/to/fff.nvim/packages/pi-fff/src/index.ts
|
||||
pi -e /path/to/fff/packages/pi-fff/src/index.ts
|
||||
```
|
||||
|
||||
This extension registers FFF-powered tools (`fffind`, `ffgrep`, `fff-multi-grep`) alongside pi's built-in tools.
|
||||
@@ -134,7 +134,8 @@ Mode precedence:
|
||||
- `--fff-mode <mode>` — set mode (see above)
|
||||
- `--fff-frecency-db <path>` — path to frecency database (also: `FFF_FRECENCY_DB` env)
|
||||
- `--fff-history-db <path>` — path to query history database (also: `FFF_HISTORY_DB` env)
|
||||
- `--fff-enable-root-scan` — allow indexing when launched from `/` (also: `FFF_ENABLE_ROOT_SCAN=1` env). FFF refuses to init at the filesystem root by default. Home directory scanning is always enabled for pi.
|
||||
- `--fff-enable-root-scan` — allow indexing when launched from `/` (also: `FFF_ENABLE_ROOT_SCAN=1` env). FFF refuses to init at the filesystem root by default.
|
||||
- `--fff-enable-home-scan` — index the home directory when launched from `$HOME` (also: `FFF_ENABLE_HOME_SCAN` env). Enabled by default. Disable with `--fff-enable-home-scan=false` or `FFF_ENABLE_HOME_SCAN=0` if your `$HOME` contains huge trees (toolchains, kernel sources, build outputs) that make the background index run for a long time. When launched from `$HOME` with this enabled, pi shows a warning that the whole home tree is being indexed.
|
||||
|
||||
## Data
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import type { FileFinderApi } from "@ff-labs/fff-node";
|
||||
import { HOME_DIR } from "./paths";
|
||||
import { loadSdk, SCAN_TIMEOUT_MS } from "./sdk";
|
||||
|
||||
export const MAX_AUX = 3;
|
||||
@@ -15,10 +15,19 @@ interface AuxPicker {
|
||||
|
||||
export interface AuxOpts {
|
||||
enableFsRootScanning: boolean;
|
||||
enableHomeDirScanning?: boolean;
|
||||
// Called before a newly spawned aux picker starts a scan that covers $HOME.
|
||||
onHomeDirScan?: (root: string) => void;
|
||||
frecencyDbPath?: string;
|
||||
historyDbPath?: string;
|
||||
}
|
||||
|
||||
export class AuxFinderPool {
|
||||
private entries: AuxPicker[] = [];
|
||||
// In-flight creations keyed by root. Concurrent acquire() calls for the same
|
||||
// (or a covering) root share one finder/scan instead of each starting a full
|
||||
// duplicate traversal — issue #746. Mirrors the main finder's finderPromise.
|
||||
private pending = new Map<string, Promise<AuxPicker>>();
|
||||
constructor(private opts: AuxOpts) {}
|
||||
|
||||
destroy(): void {
|
||||
@@ -27,6 +36,7 @@ export class AuxFinderPool {
|
||||
}
|
||||
|
||||
this.entries = [];
|
||||
this.pending.clear();
|
||||
}
|
||||
|
||||
private sweepIdle(now = Date.now()): void {
|
||||
@@ -58,32 +68,61 @@ export class AuxFinderPool {
|
||||
return { finder: covering.finder, root: covering.root };
|
||||
}
|
||||
|
||||
// Coalesce concurrent creations for the same root so we scan once. A slow
|
||||
// full-home scan started by one call is awaited by the others instead of
|
||||
// each spawning its own traversal (#746).
|
||||
const inflight = this.pending.get(maybeRoot);
|
||||
if (inflight) {
|
||||
const e = await inflight;
|
||||
e.lastUsed = Date.now();
|
||||
return { finder: e.finder, root: e.root };
|
||||
}
|
||||
|
||||
const creation = this.create(maybeRoot).finally(() => {
|
||||
this.pending.delete(maybeRoot);
|
||||
});
|
||||
this.pending.set(maybeRoot, creation);
|
||||
const entry = await creation;
|
||||
return { finder: entry.finder, root: entry.root };
|
||||
}
|
||||
|
||||
private async create(root: string): Promise<AuxPicker> {
|
||||
if (this.entries.length >= MAX_AUX) {
|
||||
let oldest = this.entries[0];
|
||||
for (const e of this.entries)
|
||||
if (e.lastUsed < oldest.lastUsed) oldest = e;
|
||||
for (const e of this.entries) if (e.lastUsed < oldest.lastUsed) oldest = e;
|
||||
if (!oldest.finder.isDestroyed) oldest.finder.destroy();
|
||||
this.entries = this.entries.filter((e) => e !== oldest);
|
||||
}
|
||||
|
||||
const enableHomeDirScanning = this.opts.enableHomeDirScanning ?? true;
|
||||
// A fresh picker rooted at (or above) $HOME walks the whole home tree, so
|
||||
// the user gets told every time the agent spawns one — see issue #743.
|
||||
if (enableHomeDirScanning && rootCovers(root, HOME_DIR)) {
|
||||
this.opts.onHomeDirScan?.(root);
|
||||
}
|
||||
|
||||
const { FileFinder } = await loadSdk();
|
||||
// LMDB env can only be opened once per process; the main finder already
|
||||
// owns the frecency/history DBs. Aux finders are transient and run without
|
||||
// persistent scoring — see issue #700.
|
||||
const result = FileFinder.create({
|
||||
basePath: maybeRoot,
|
||||
basePath: root,
|
||||
frecencyDbPath: this.opts.frecencyDbPath,
|
||||
historyDbPath: this.opts.historyDbPath,
|
||||
aiMode: true,
|
||||
enableHomeDirScanning: true,
|
||||
enableHomeDirScanning,
|
||||
enableFsRootScanning: this.opts.enableFsRootScanning,
|
||||
});
|
||||
if (!result.ok)
|
||||
throw new Error(
|
||||
`Failed to create aux file finder for ${maybeRoot}: ${result.error}`,
|
||||
);
|
||||
|
||||
if (!result.ok) {
|
||||
throw new Error(`Failed to create aux file finder for ${root}: ${result.error}`);
|
||||
}
|
||||
|
||||
await result.value.waitForScan(SCAN_TIMEOUT_MS);
|
||||
this.entries.push({ root: maybeRoot, finder: result.value, lastUsed: Date.now() });
|
||||
return { finder: result.value, root: maybeRoot };
|
||||
const entry: AuxPicker = {
|
||||
root,
|
||||
finder: result.value,
|
||||
lastUsed: Date.now(),
|
||||
};
|
||||
this.entries.push(entry);
|
||||
return entry;
|
||||
}
|
||||
|
||||
size(): number {
|
||||
@@ -96,9 +135,7 @@ export class AuxFinderPool {
|
||||
// remainder usable as a fuzzy path constraint relative to that root. Glob and
|
||||
// nonexistent segments both go into the suffix: we walk up to the nearest
|
||||
// existing ancestor so partially-wrong paths still resolve to a search root.
|
||||
export function resolveAuxRoot(
|
||||
absPath: string,
|
||||
): { root: string; suffix: string } | null {
|
||||
export function resolveAuxRoot(absPath: string): { root: string; suffix: string } | null {
|
||||
const trimmed = path.normalize(absPath.trim()).replace(/\/+$/, "") || "/";
|
||||
if (!path.isAbsolute(trimmed)) return null;
|
||||
if (trimmed === path.sep) return { root: path.sep, suffix: "" };
|
||||
@@ -147,7 +184,7 @@ export function routePathConstraint(
|
||||
let candidate = pathConstraint.trim();
|
||||
if (!candidate) return null;
|
||||
if (candidate === "~" || candidate.startsWith("~/"))
|
||||
candidate = path.join(os.homedir(), candidate.slice(1));
|
||||
candidate = path.join(HOME_DIR, candidate.slice(1));
|
||||
if (!path.isAbsolute(candidate)) {
|
||||
// Plain workspace-relative constraints stay on the workspace finder.
|
||||
if (candidate !== ".." && !candidate.startsWith("../")) return null;
|
||||
@@ -158,7 +195,6 @@ export function routePathConstraint(
|
||||
return resolveAuxRoot(candidate);
|
||||
}
|
||||
|
||||
|
||||
export function rootCovers(root: string, target: string): boolean {
|
||||
if (root === target) return true;
|
||||
const prefix = root.endsWith(path.sep) ? root : root + path.sep;
|
||||
|
||||
+187
-113
@@ -23,6 +23,7 @@ import type {
|
||||
import { Type } from "@sinclair/typebox";
|
||||
import { AuxFinderPool, routePathConstraint } from "./aux-finders";
|
||||
import { buildQuery } from "./query";
|
||||
import { isHomeDir } from "./paths";
|
||||
import { loadSdk, SCAN_TIMEOUT_MS } from "./sdk";
|
||||
|
||||
export { SCAN_TIMEOUT_MS } from "./sdk";
|
||||
@@ -33,9 +34,19 @@ export { SCAN_TIMEOUT_MS } from "./sdk";
|
||||
|
||||
const DEFAULT_GREP_LIMIT = 20;
|
||||
const DEFAULT_FIND_LIMIT = 30;
|
||||
const GREP_PAGE_SIZE_MAX = 50;
|
||||
const GREP_CONTEXT_MAX = 20;
|
||||
const GREP_MAX_LINE_LENGTH = 500;
|
||||
const MENTION_MAX_RESULTS = 20;
|
||||
|
||||
// If we exceed 10 seconds for indexed grep - something is definitely off
|
||||
const GREP_TIME_BUDGET_MS = 10_000;
|
||||
|
||||
const HOME_SCAN_STATUS_KEY = "fff";
|
||||
const HOME_SCAN_POLL_MS = 1_000;
|
||||
const HOME_SCAN_DISABLE_HINT =
|
||||
"You can prevent home dir indexing with --fff-enable-home-scan=false (or FFF_ENABLE_HOME_SCAN=0).";
|
||||
|
||||
type FffMode = "tools-and-ui" | "tools-only" | "override";
|
||||
|
||||
const VALID_MODES: FffMode[] = ["tools-and-ui", "tools-only", "override"];
|
||||
@@ -119,6 +130,13 @@ function truncateLine(line: string, max = GREP_MAX_LINE_LENGTH): string {
|
||||
return trimmed.length <= max ? trimmed : `${trimmed.slice(0, max)}...`;
|
||||
}
|
||||
|
||||
// Clamp caller-supplied context to a non-negative bounded integer so a large
|
||||
// value cannot multiply output size past the model window.
|
||||
function clampContext(context: number | undefined): number {
|
||||
if (!context || context < 0) return 0;
|
||||
return Math.min(Math.floor(context), GREP_CONTEXT_MAX);
|
||||
}
|
||||
|
||||
const HOT_FRECENCY = 25;
|
||||
const WARM_FRECENCY = 20;
|
||||
|
||||
@@ -261,9 +279,7 @@ function createFffMentionProvider(
|
||||
|
||||
const query = prefix.startsWith('@"') ? prefix.slice(2) : prefix.slice(1);
|
||||
const items = await getItems(query, options.signal);
|
||||
return options.signal.aborted || items.length === 0
|
||||
? null
|
||||
: { items, prefix };
|
||||
return options.signal.aborted || items.length === 0 ? null : { items, prefix };
|
||||
},
|
||||
applyCompletion(_lines, cursorLine, cursorCol, item, prefix) {
|
||||
const currentLine = _lines[cursorLine] || "";
|
||||
@@ -272,11 +288,7 @@ function createFffMentionProvider(
|
||||
const newLine = before + item.value + after;
|
||||
const newCursorCol = cursorCol - prefix.length + item.value.length;
|
||||
return {
|
||||
lines: [
|
||||
..._lines.slice(0, cursorLine),
|
||||
newLine,
|
||||
..._lines.slice(cursorLine + 1),
|
||||
],
|
||||
lines: [..._lines.slice(0, cursorLine), newLine, ..._lines.slice(cursorLine + 1)],
|
||||
cursorLine,
|
||||
cursorCol: newCursorCol,
|
||||
};
|
||||
@@ -306,7 +318,7 @@ export default function fffExtension(pi: ExtensionAPI) {
|
||||
|
||||
const toolNames = resolveToolNames(currentMode);
|
||||
|
||||
// DB path resolution: flag > env > undefined (use fff-node defaults)
|
||||
// DB path resolution: flag > env > undefined (no persistent DBs)
|
||||
const frecencyDbPath =
|
||||
(pi.getFlag("fff-frecency-db") as string | undefined) ??
|
||||
process.env.FFF_FRECENCY_DB ??
|
||||
@@ -316,20 +328,28 @@ export default function fffExtension(pi: ExtensionAPI) {
|
||||
process.env.FFF_HISTORY_DB ??
|
||||
undefined;
|
||||
|
||||
// Root scanning opt-in: flag (boolean) > env ("1"/"true") > false.
|
||||
// FFF refuses to init at / unless this is set. Home dir scanning is on by
|
||||
// default for pi — launching pi from $HOME is a normal flow.
|
||||
function resolveBoolOpt(flagName: string, envName: string): boolean {
|
||||
// flag (boolean) > env ("1"/"true", or "0"/"false") > default.
|
||||
function resolveBoolOpt(flagName: string, envName: string, fallback = false): boolean {
|
||||
const flag = pi.getFlag(flagName);
|
||||
if (typeof flag === "boolean") return flag;
|
||||
if (typeof flag === "string") return flag === "true" || flag === "1";
|
||||
const env = process.env[envName];
|
||||
return env === "1" || env === "true";
|
||||
if (env === "1" || env === "true") return true;
|
||||
if (env === "0" || env === "false") return false;
|
||||
return fallback;
|
||||
}
|
||||
// Root scanning opt-in: FFF refuses to init at / unless this is set.
|
||||
const enableFsRootScanning = resolveBoolOpt(
|
||||
"fff-enable-root-scan",
|
||||
"FFF_ENABLE_ROOT_SCAN",
|
||||
);
|
||||
// Home dir scanning is on by default (launching pi from $HOME is a normal
|
||||
// flow), but configurable so users with huge $HOME trees can opt out.
|
||||
const enableHomeDirScanning = resolveBoolOpt(
|
||||
"fff-enable-home-scan",
|
||||
"FFF_ENABLE_HOME_SCAN",
|
||||
true,
|
||||
);
|
||||
|
||||
function getMode(): FffMode {
|
||||
return currentMode;
|
||||
@@ -343,8 +363,29 @@ export default function fffExtension(pi: ExtensionAPI) {
|
||||
return currentMode !== "tools-only";
|
||||
}
|
||||
|
||||
let auxPool = new AuxFinderPool({
|
||||
// Set on session_start; the only handle to the UI outside an event handler.
|
||||
// setStatus is TUI/RPC-only, hence optional.
|
||||
let uiCtx: {
|
||||
ui: {
|
||||
notify: (message: string, type?: "info" | "warning" | "error") => void;
|
||||
setStatus?: (key: string, text: string | undefined) => void;
|
||||
};
|
||||
} | null = null;
|
||||
let homeScanTimer: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
function warnHomeDirScan(root: string): void {
|
||||
uiCtx?.ui.notify(
|
||||
`(fff): Your cwd (${root}) is too large. Indexing will take additional time and resources.\n${HOME_SCAN_DISABLE_HINT}`,
|
||||
"warning",
|
||||
);
|
||||
}
|
||||
|
||||
const auxPool = new AuxFinderPool({
|
||||
enableFsRootScanning,
|
||||
enableHomeDirScanning,
|
||||
onHomeDirScan: warnHomeDirScan,
|
||||
frecencyDbPath,
|
||||
historyDbPath,
|
||||
});
|
||||
|
||||
// in case cwd changes we need to figure this out
|
||||
@@ -367,7 +408,7 @@ export default function fffExtension(pi: ExtensionAPI) {
|
||||
frecencyDbPath,
|
||||
historyDbPath,
|
||||
aiMode: true,
|
||||
enableHomeDirScanning: true,
|
||||
enableHomeDirScanning,
|
||||
enableFsRootScanning,
|
||||
});
|
||||
|
||||
@@ -385,7 +426,40 @@ export default function fffExtension(pi: ExtensionAPI) {
|
||||
return finderPromise;
|
||||
}
|
||||
|
||||
function stopHomeScanStatus(): void {
|
||||
if (homeScanTimer) {
|
||||
clearInterval(homeScanTimer);
|
||||
homeScanTimer = null;
|
||||
}
|
||||
uiCtx?.ui.setStatus?.(HOME_SCAN_STATUS_KEY, undefined);
|
||||
}
|
||||
|
||||
// waitForScan() resolves on timeout too, so the scan can still be running.
|
||||
// Poll the live progress until it settles, then clear the footer.
|
||||
function trackHomeScanStatus(): void {
|
||||
stopHomeScanStatus();
|
||||
if (!uiCtx?.ui.setStatus) return;
|
||||
|
||||
const tick = () => {
|
||||
const progress = mainFinder?.getScanProgress?.();
|
||||
if (!progress?.ok || !progress.value.isScanning) {
|
||||
stopHomeScanStatus();
|
||||
return;
|
||||
}
|
||||
uiCtx?.ui.setStatus?.(
|
||||
HOME_SCAN_STATUS_KEY,
|
||||
`Agent is indexing $HOME (${progress.value.scannedFilesCount} files), this can lead to high CPU`,
|
||||
);
|
||||
};
|
||||
|
||||
homeScanTimer = setInterval(tick, HOME_SCAN_POLL_MS);
|
||||
// Must not hold the process open once pi is done.
|
||||
(homeScanTimer as { unref?: () => void }).unref?.();
|
||||
tick();
|
||||
}
|
||||
|
||||
function destroyFinder() {
|
||||
stopHomeScanStatus();
|
||||
if (mainFinder && !mainFinder.isDestroyed) {
|
||||
mainFinder.destroy();
|
||||
mainFinder = null;
|
||||
@@ -407,9 +481,7 @@ export default function fffExtension(pi: ExtensionAPI) {
|
||||
const aux = await auxPool.acquire(route.root);
|
||||
// A broader covering picker may have been reused; rebase the suffix so the
|
||||
// constraint stays relative to the picker's actual root.
|
||||
const rebase = nodePath
|
||||
.relative(aux.root, route.root)
|
||||
.replaceAll(nodePath.sep, "/");
|
||||
const rebase = nodePath.relative(aux.root, route.root).replaceAll(nodePath.sep, "/");
|
||||
const suffix = [rebase, route.suffix].filter(Boolean).join("/");
|
||||
const query = buildQuery(suffix || undefined, pattern, exclude, aux.root);
|
||||
return { finder: aux.finder, query, root: aux.root };
|
||||
@@ -426,22 +498,20 @@ export default function fffExtension(pi: ExtensionAPI) {
|
||||
const result = f.mixedSearch(query, { pageSize: MENTION_MAX_RESULTS });
|
||||
if (!result.ok) return [];
|
||||
|
||||
return result.value.items
|
||||
.slice(0, MENTION_MAX_RESULTS)
|
||||
.map((mixed: MixedItem) => {
|
||||
if (mixed.type === "directory") {
|
||||
return {
|
||||
value: buildAtCompletionValue(mixed.item.relativePath),
|
||||
label: mixed.item.dirName,
|
||||
description: mixed.item.relativePath,
|
||||
};
|
||||
}
|
||||
return result.value.items.slice(0, MENTION_MAX_RESULTS).map((mixed: MixedItem) => {
|
||||
if (mixed.type === "directory") {
|
||||
return {
|
||||
value: buildAtCompletionValue(mixed.item.relativePath),
|
||||
label: mixed.item.fileName,
|
||||
label: mixed.item.dirName,
|
||||
description: mixed.item.relativePath,
|
||||
};
|
||||
});
|
||||
}
|
||||
return {
|
||||
value: buildAtCompletionValue(mixed.item.relativePath),
|
||||
label: mixed.item.fileName,
|
||||
description: mixed.item.relativePath,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function registerAutocompleteProvider(ctx: {
|
||||
@@ -477,21 +547,11 @@ export default function fffExtension(pi: ExtensionAPI) {
|
||||
return current.getSuggestions(lines, cursorLine, cursorCol, options);
|
||||
},
|
||||
applyCompletion(lines, cursorLine, cursorCol, item, prefix) {
|
||||
return current.applyCompletion(
|
||||
lines,
|
||||
cursorLine,
|
||||
cursorCol,
|
||||
item,
|
||||
prefix,
|
||||
);
|
||||
return current.applyCompletion(lines, cursorLine, cursorCol, item, prefix);
|
||||
},
|
||||
shouldTriggerFileCompletion(lines, cursorLine, cursorCol) {
|
||||
return (
|
||||
current.shouldTriggerFileCompletion?.(
|
||||
lines,
|
||||
cursorLine,
|
||||
cursorCol,
|
||||
) ?? true
|
||||
current.shouldTriggerFileCompletion?.(lines, cursorLine, cursorCol) ?? true
|
||||
);
|
||||
},
|
||||
};
|
||||
@@ -506,14 +566,12 @@ export default function fffExtension(pi: ExtensionAPI) {
|
||||
});
|
||||
|
||||
pi.registerFlag("fff-frecency-db", {
|
||||
description:
|
||||
"Path to the frecency database (overrides FFF_FRECENCY_DB env)",
|
||||
description: "Path to the frecency database (overrides FFF_FRECENCY_DB env)",
|
||||
type: "string",
|
||||
});
|
||||
|
||||
pi.registerFlag("fff-history-db", {
|
||||
description:
|
||||
"Path to the query history database (overrides FFF_HISTORY_DB env)",
|
||||
description: "Path to the query history database (overrides FFF_HISTORY_DB env)",
|
||||
type: "string",
|
||||
});
|
||||
|
||||
@@ -523,9 +581,16 @@ export default function fffExtension(pi: ExtensionAPI) {
|
||||
type: "boolean",
|
||||
});
|
||||
|
||||
pi.registerFlag("fff-enable-home-scan", {
|
||||
description:
|
||||
"Index the home dir when launched from $HOME (default true; disable with --fff-enable-home-scan=false or FFF_ENABLE_HOME_SCAN=0)",
|
||||
type: "boolean",
|
||||
});
|
||||
|
||||
pi.on("session_start", async (_event, ctx) => {
|
||||
try {
|
||||
activeCwd = ctx.cwd;
|
||||
uiCtx = ctx as unknown as typeof uiCtx;
|
||||
|
||||
// Restore persisted mode from session entries. This handles session
|
||||
// resume after process restart where env vars are lost, and ensures
|
||||
@@ -552,6 +617,21 @@ export default function fffExtension(pi: ExtensionAPI) {
|
||||
|
||||
registerAutocompleteProvider(ctx);
|
||||
await ensureFinder(activeCwd);
|
||||
|
||||
// Warn when launched from $HOME with home scanning on: indexing a large
|
||||
// home tree can run for a long time in the background (issue #743).
|
||||
const atHome = enableHomeDirScanning && isHomeDir(activeCwd);
|
||||
if (atHome) {
|
||||
warnHomeDirScan(activeCwd);
|
||||
ctx.ui.setStatus?.(
|
||||
HOME_SCAN_STATUS_KEY,
|
||||
"Agent is indexing $HOME, this can lead to high CPU",
|
||||
);
|
||||
}
|
||||
|
||||
// waitForScan() also resolves on timeout, so poll until the scan really
|
||||
// settles before clearing the footer.
|
||||
if (atHome) trackHomeScanStatus();
|
||||
} catch (e: unknown) {
|
||||
ctx.ui.notify(
|
||||
`FFF init failed: ${e instanceof Error ? e.message : String(e)}`,
|
||||
@@ -573,20 +653,15 @@ export default function fffExtension(pi: ExtensionAPI) {
|
||||
context: any,
|
||||
maxLines = 15,
|
||||
) => {
|
||||
const text =
|
||||
(context.lastComponent as Text | undefined) ?? new Text("", 0, 0);
|
||||
const output =
|
||||
result.content?.find((c) => c.type === "text")?.text?.trim() ?? "";
|
||||
const text = (context.lastComponent as Text | undefined) ?? new Text("", 0, 0);
|
||||
const output = result.content?.find((c) => c.type === "text")?.text?.trim() ?? "";
|
||||
if (!output) {
|
||||
text.setText(theme.fg("muted", "No output"));
|
||||
return text;
|
||||
}
|
||||
|
||||
const lines = output.split("\n");
|
||||
const displayLines = lines.slice(
|
||||
0,
|
||||
options.expanded ? lines.length : maxLines,
|
||||
);
|
||||
const displayLines = lines.slice(0, options.expanded ? lines.length : maxLines);
|
||||
let content = `\n${displayLines.map((line: string) => theme.fg("toolOutput", line)).join("\n")}`;
|
||||
if (lines.length > displayLines.length) {
|
||||
content += theme.fg(
|
||||
@@ -623,7 +698,9 @@ export default function fffExtension(pi: ExtensionAPI) {
|
||||
}),
|
||||
),
|
||||
context: Type.Optional(
|
||||
Type.Number({ description: "Context lines before+after each match" }),
|
||||
Type.Number({
|
||||
description: `Context lines before+after each match (0-${GREP_CONTEXT_MAX})`,
|
||||
}),
|
||||
),
|
||||
limit: Type.Optional(
|
||||
Type.Number({
|
||||
@@ -641,10 +718,10 @@ export default function fffExtension(pi: ExtensionAPI) {
|
||||
description: `Grep file contents. Smart-case, auto-detects regex vs literal, git-aware. Results are ranked by frecency (most-accessed files first); matches within a file stay in source order. Default limit ${DEFAULT_GREP_LIMIT}.`,
|
||||
promptSnippet: "Grep contents",
|
||||
promptGuidelines: [
|
||||
"Prefer bare identifiers as patterns. Literal queries are most efficient.",
|
||||
"Use path for include ('src/', '*.ts') and exclude for noise ('test/,*.min.js').",
|
||||
"caseSensitive: true when you need exact case (smart-case otherwise).",
|
||||
"After 1-2 greps, read the top match instead of more greps.",
|
||||
`${toolNames.grep}: prefer bare identifiers as patterns. Literal queries are most efficient.`,
|
||||
`${toolNames.grep}: use path for include ('src/', '*.ts') and exclude for noise ('test/,*.min.js').`,
|
||||
`${toolNames.grep}: caseSensitive: true when you need exact case (smart-case otherwise).`,
|
||||
`${toolNames.grep}: after 1-2 greps, read the top match instead of more greps.`,
|
||||
],
|
||||
parameters: grepSchema,
|
||||
|
||||
@@ -652,14 +729,14 @@ export default function fffExtension(pi: ExtensionAPI) {
|
||||
if (signal?.aborted) throw new Error("Operation aborted");
|
||||
|
||||
const pattern = params.pattern;
|
||||
const aux = await resolveFinderForPath(
|
||||
params.path,
|
||||
pattern,
|
||||
params.exclude,
|
||||
);
|
||||
const aux = await resolveFinderForPath(params.path, pattern, params.exclude);
|
||||
|
||||
const picker = aux ? aux.finder : await ensureFinder(activeCwd);
|
||||
const effectiveLimit = Math.max(1, params.limit ?? DEFAULT_GREP_LIMIT);
|
||||
// pageSize caps TOTAL matches across all files; maxMatchesPerFile alone
|
||||
// only caps per-file, so limit=5 could still return a full SDK page.
|
||||
const pageSize = Math.min(effectiveLimit, GREP_PAGE_SIZE_MAX);
|
||||
const context = clampContext(params.context);
|
||||
const query = aux
|
||||
? aux.query
|
||||
: buildQuery(params.path, pattern, params.exclude, activeCwd);
|
||||
@@ -667,8 +744,7 @@ export default function fffExtension(pi: ExtensionAPI) {
|
||||
// Auto-detect: regex if the pattern has regex metacharacters AND parses
|
||||
// as a valid regex, otherwise plain literal. The fuzzy fallback below
|
||||
// only kicks in for plain mode — regex queries are intentional.
|
||||
const hasRegexSyntax =
|
||||
pattern !== pattern.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
const hasRegexSyntax = pattern !== pattern.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
|
||||
let mode: GrepMode = hasRegexSyntax ? "regex" : "plain";
|
||||
if (mode === "regex") {
|
||||
@@ -708,11 +784,13 @@ export default function fffExtension(pi: ExtensionAPI) {
|
||||
const grepResult = picker.grep(query, {
|
||||
mode,
|
||||
smartCase,
|
||||
maxMatchesPerFile: Math.min(effectiveLimit, 50),
|
||||
maxMatchesPerFile: pageSize,
|
||||
pageSize,
|
||||
cursor: (params.cursor ? getCursor(params.cursor) : null) ?? null,
|
||||
beforeContext: params.context ?? 0,
|
||||
afterContext: params.context ?? 0,
|
||||
beforeContext: context,
|
||||
afterContext: context,
|
||||
classifyDefinitions: true,
|
||||
timeBudgetMs: GREP_TIME_BUDGET_MS,
|
||||
});
|
||||
|
||||
if (!grepResult.ok) throw new Error(grepResult.error);
|
||||
@@ -720,8 +798,14 @@ export default function fffExtension(pi: ExtensionAPI) {
|
||||
let result = grepResult.value;
|
||||
let fuzzyNotice: string | null = null;
|
||||
|
||||
// automatic fuzzy fallback allows to broad the queries and find different cases
|
||||
if (result.items.length === 0 && !params.cursor && mode !== "regex") {
|
||||
// if we hit the timeout do not run the fuzzy fallback
|
||||
// cause it will only consumer more time
|
||||
if (
|
||||
result.items.length === 0 &&
|
||||
!result.nextCursor &&
|
||||
!params.cursor &&
|
||||
mode !== "regex"
|
||||
) {
|
||||
// When the caller pinned a specific file (path has an extension), the
|
||||
// fuzzy fallback broadens across the whole picker — the file may just
|
||||
// be misnamed. For directory constraints (or no path), we keep the
|
||||
@@ -733,11 +817,13 @@ export default function fffExtension(pi: ExtensionAPI) {
|
||||
const fuzzy = picker.grep(fuzzyQuery, {
|
||||
mode: "fuzzy",
|
||||
smartCase,
|
||||
maxMatchesPerFile: Math.min(effectiveLimit, 50),
|
||||
maxMatchesPerFile: pageSize,
|
||||
pageSize,
|
||||
cursor: null,
|
||||
beforeContext: 0,
|
||||
afterContext: 0,
|
||||
classifyDefinitions: true,
|
||||
timeBudgetMs: GREP_TIME_BUDGET_MS,
|
||||
});
|
||||
|
||||
if (fuzzy.ok && fuzzy.value.items.length > 0) {
|
||||
@@ -749,14 +835,10 @@ export default function fffExtension(pi: ExtensionAPI) {
|
||||
let output = formatGrepOutput(result);
|
||||
const notices: string[] = [];
|
||||
if (result.regexFallbackError) {
|
||||
notices.push(
|
||||
`Invalid regex: ${result.regexFallbackError}, used literal match`,
|
||||
);
|
||||
notices.push(`Invalid regex: ${result.regexFallbackError}, used literal match`);
|
||||
}
|
||||
if (result.nextCursor) {
|
||||
notices.push(
|
||||
`Continue with cursor="${storeCursor(result.nextCursor)}"`,
|
||||
);
|
||||
notices.push(`Continue with cursor="${storeCursor(result.nextCursor)}"`);
|
||||
}
|
||||
|
||||
if (notices.length > 0) output += `\n\n[${notices.join(". ")}]`;
|
||||
@@ -772,8 +854,7 @@ export default function fffExtension(pi: ExtensionAPI) {
|
||||
},
|
||||
|
||||
renderCall(args, theme, context) {
|
||||
const text =
|
||||
(context.lastComponent as Text | undefined) ?? new Text("", 0, 0);
|
||||
const text = (context.lastComponent as Text | undefined) ?? new Text("", 0, 0);
|
||||
const pattern = args?.pattern ?? "";
|
||||
const path = args?.path ?? ".";
|
||||
let content =
|
||||
@@ -828,12 +909,12 @@ export default function fffExtension(pi: ExtensionAPI) {
|
||||
description: `Fuzzy path search and glob search. Matches against the whole repo-relative path, not just the filename. Frecency-ranked, git-aware. Multi-word = narrower (AND). Default limit ${DEFAULT_FIND_LIMIT}.`,
|
||||
promptSnippet: "Find files by path or glob",
|
||||
promptGuidelines: [
|
||||
"Matches the WHOLE path, not just the filename — `profile` hits `chrome/browser/profiles/x.cc` too.",
|
||||
"Keep queries to 1-2 terms; extra words narrow.",
|
||||
"Use for paths, not content. Use grep for content.",
|
||||
"For exact path matches use a glob in `path` — e.g. path: '**/profile.h' for exact filename, or path: 'src/**/profile.h' scoped to a subtree. Bare patterns are fuzzy.",
|
||||
"To list everything inside a directory, pass path: 'dir/**' with an empty or wildcard pattern instead of using pattern alone.",
|
||||
"Use exclude: 'test/,*.min.js' to cut noise in large repos.",
|
||||
`${toolNames.find}: matches the WHOLE path, not just the filename — \`profile\` hits \`chrome/browser/profiles/x.cc\` too.`,
|
||||
`${toolNames.find}: keep queries to 1-2 terms; extra words narrow.`,
|
||||
`${toolNames.find}: use for paths, not content. Use ${toolNames.grep} for content.`,
|
||||
`${toolNames.find}: for exact path matches use a glob in \`path\` — e.g. path: '**/profile.h' for exact filename, or path: 'src/**/profile.h' scoped to a subtree. Bare patterns are fuzzy.`,
|
||||
`${toolNames.find}: to list everything inside a directory, pass path: 'dir/**' with an empty or wildcard pattern instead of using pattern alone.`,
|
||||
`${toolNames.find}: use exclude: 'test/,*.min.js' to cut noise in large repos.`,
|
||||
],
|
||||
parameters: findSchema,
|
||||
|
||||
@@ -845,16 +926,11 @@ export default function fffExtension(pi: ExtensionAPI) {
|
||||
const aux = resumed
|
||||
? resumed.auxRoot
|
||||
? {
|
||||
finder: (await auxPool.acquire(resumed.auxRoot, { exact: true }))
|
||||
.finder,
|
||||
finder: (await auxPool.acquire(resumed.auxRoot, { exact: true })).finder,
|
||||
root: resumed.auxRoot,
|
||||
}
|
||||
: null
|
||||
: await resolveFinderForPath(
|
||||
params.path,
|
||||
params.pattern,
|
||||
params.exclude,
|
||||
);
|
||||
: await resolveFinderForPath(params.path, params.pattern, params.exclude);
|
||||
|
||||
const picker = aux ? aux.finder : await ensureFinder(activeCwd);
|
||||
const effectiveLimit = resumed
|
||||
@@ -886,8 +962,7 @@ export default function fffExtension(pi: ExtensionAPI) {
|
||||
// shown so far there's another page to fetch.
|
||||
const shownSoFar = pageIndex * effectiveLimit + result.items.length;
|
||||
const hasMore =
|
||||
result.items.length >= effectiveLimit &&
|
||||
result.totalMatched > shownSoFar;
|
||||
result.items.length >= effectiveLimit && result.totalMatched > shownSoFar;
|
||||
|
||||
const notices: string[] = [];
|
||||
if (formatted.weak && formatted.shownCount > 0)
|
||||
@@ -922,8 +997,7 @@ export default function fffExtension(pi: ExtensionAPI) {
|
||||
},
|
||||
|
||||
renderCall(args, theme, context) {
|
||||
const text =
|
||||
(context.lastComponent as Text | undefined) ?? new Text("", 0, 0);
|
||||
const text = (context.lastComponent as Text | undefined) ?? new Text("", 0, 0);
|
||||
const pattern = args?.pattern ?? "";
|
||||
const path = args?.path ?? ".";
|
||||
let content =
|
||||
@@ -957,7 +1031,9 @@ export default function fffExtension(pi: ExtensionAPI) {
|
||||
Type.String({ description: "File filter, e.g. '*.{ts,tsx} !test/'" }),
|
||||
),
|
||||
context: Type.Optional(
|
||||
Type.Number({ description: "Context lines before+after" }),
|
||||
Type.Number({
|
||||
description: `Context lines before+after (0-${GREP_CONTEXT_MAX})`,
|
||||
}),
|
||||
),
|
||||
limit: Type.Optional(
|
||||
Type.Number({
|
||||
@@ -974,9 +1050,9 @@ export default function fffExtension(pi: ExtensionAPI) {
|
||||
"Search file contents for ANY of multiple literal patterns (OR, SIMD Aho-Corasick). Faster than regex alternation.",
|
||||
promptSnippet: "Multi-pattern OR content search",
|
||||
promptGuidelines: [
|
||||
"Use when searching for several identifiers at once.",
|
||||
"Include all naming-convention variants (snake/camel/Pascal).",
|
||||
"Patterns are literal. Use constraints for file filters.",
|
||||
`${toolNames.multiGrep}: use when searching for several identifiers at once.`,
|
||||
`${toolNames.multiGrep}: include all naming-convention variants (snake/camel/Pascal).`,
|
||||
`${toolNames.multiGrep}: patterns are literal. Use constraints for file filters.`,
|
||||
],
|
||||
parameters: multiGrepSchema,
|
||||
|
||||
@@ -987,15 +1063,18 @@ export default function fffExtension(pi: ExtensionAPI) {
|
||||
|
||||
const f = await ensureFinder(activeCwd);
|
||||
const effectiveLimit = Math.max(1, params.limit ?? DEFAULT_GREP_LIMIT);
|
||||
const pageSize = Math.min(effectiveLimit, GREP_PAGE_SIZE_MAX);
|
||||
const context = clampContext(params.context);
|
||||
|
||||
const grepResult = f.multiGrep({
|
||||
patterns: params.patterns,
|
||||
constraints: params.constraints,
|
||||
maxMatchesPerFile: Math.min(effectiveLimit, 50),
|
||||
maxMatchesPerFile: pageSize,
|
||||
pageSize,
|
||||
smartCase: true,
|
||||
cursor: (params.cursor ? getCursor(params.cursor) : null) ?? null,
|
||||
beforeContext: params.context ?? 0,
|
||||
afterContext: params.context ?? 0,
|
||||
beforeContext: context,
|
||||
afterContext: context,
|
||||
});
|
||||
|
||||
if (!grepResult.ok) throw new Error(grepResult.error);
|
||||
@@ -1024,8 +1103,7 @@ export default function fffExtension(pi: ExtensionAPI) {
|
||||
},
|
||||
|
||||
renderCall(args, theme, context) {
|
||||
const text =
|
||||
(context.lastComponent as Text | undefined) ?? new Text("", 0, 0);
|
||||
const text = (context.lastComponent as Text | undefined) ?? new Text("", 0, 0);
|
||||
const patterns = args?.patterns ?? [];
|
||||
const constraints = args?.constraints;
|
||||
let content =
|
||||
@@ -1047,8 +1125,7 @@ export default function fffExtension(pi: ExtensionAPI) {
|
||||
// --- commands ---
|
||||
|
||||
pi.registerCommand("fff-mode", {
|
||||
description:
|
||||
"Show or set FFF mode: /fff-mode [tools-and-ui | tools-only | override]",
|
||||
description: "Show or set FFF mode: /fff-mode [tools-and-ui | tools-only | override]",
|
||||
handler: async (args, ctx) => {
|
||||
const arg = (args || "").trim();
|
||||
|
||||
@@ -1062,10 +1139,7 @@ export default function fffExtension(pi: ExtensionAPI) {
|
||||
|
||||
// Validate and set mode
|
||||
if (!VALID_MODES.includes(arg as FffMode)) {
|
||||
ctx.ui.notify(
|
||||
`Usage: /fff-mode [${VALID_MODES.join(" | ")}]`,
|
||||
"warning",
|
||||
);
|
||||
ctx.ui.notify(`Usage: /fff-mode [${VALID_MODES.join(" | ")}]`, "warning");
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
// Resolved once per process: os.homedir() hits the env/passwd on every call.
|
||||
export const HOME_DIR = path.resolve(os.homedir());
|
||||
|
||||
export function isHomeDir(dir: string): boolean {
|
||||
return path.resolve(dir) === HOME_DIR;
|
||||
}
|
||||
@@ -10,11 +10,7 @@ export function normalizePathConstraint(
|
||||
if (path.isAbsolute(trimmed)) {
|
||||
const relative = path.relative(cwd, trimmed).replaceAll(path.sep, "/");
|
||||
if (relative === "") return null;
|
||||
if (
|
||||
relative.startsWith("../") ||
|
||||
relative === ".." ||
|
||||
path.isAbsolute(relative)
|
||||
) {
|
||||
if (relative.startsWith("../") || relative === ".." || path.isAbsolute(relative)) {
|
||||
throw new Error(
|
||||
`Path constraint must be relative to the workspace: ${pathConstraint}`,
|
||||
);
|
||||
|
||||
@@ -22,8 +22,21 @@ function detectRuntime(): "bun" | "node" {
|
||||
export function loadSdk(): Promise<{ FileFinder: FileFinderStatic }> {
|
||||
if (sdkPromise) return sdkPromise;
|
||||
|
||||
// Pi reloads extension modules with jiti moduleCache:false, so this module
|
||||
// is re-executed on every /reload. Re-importing the fff-bun module graph
|
||||
// (which top-level awaits a `type: "file"` import of the native .so) hangs
|
||||
// forever inside the Bun-compiled pi binary. Cache the first import on
|
||||
// globalThis so reloads reuse the resolved module instead of re-importing.
|
||||
const g = globalThis as Record<string, unknown>;
|
||||
if (g.__fffSdkPromiseGlobal) {
|
||||
sdkPromise = g.__fffSdkPromiseGlobal as Promise<{ FileFinder: FileFinderStatic }>;
|
||||
return sdkPromise;
|
||||
}
|
||||
|
||||
// default to node as it seems like default option
|
||||
const pkg = detectRuntime() === "bun" ? "@ff-labs/fff-bun" : "@ff-labs/fff-node";
|
||||
sdkPromise = import(pkg) as Promise<{ FileFinder: FileFinderStatic }>;
|
||||
return sdkPromise;
|
||||
const p = import(pkg) as Promise<{ FileFinder: FileFinderStatic }>;
|
||||
sdkPromise = p;
|
||||
(globalThis as Record<string, unknown>).__fffSdkPromiseGlobal = p;
|
||||
return p;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
import { describe, expect, mock, test } from "bun:test";
|
||||
|
||||
interface MockFinder {
|
||||
isDestroyed: boolean;
|
||||
basePath: string;
|
||||
waitForScan: (ms: number) => Promise<void>;
|
||||
destroy: () => void;
|
||||
}
|
||||
|
||||
const created: MockFinder[] = [];
|
||||
|
||||
function createMockFinder(basePath: string): MockFinder {
|
||||
const finder: MockFinder = {
|
||||
isDestroyed: false,
|
||||
basePath,
|
||||
// simulate a slow scan so concurrent acquires overlap
|
||||
waitForScan: () => new Promise((r) => setTimeout(r, 50)),
|
||||
destroy: () => {
|
||||
finder.isDestroyed = true;
|
||||
},
|
||||
};
|
||||
created.push(finder);
|
||||
return finder;
|
||||
}
|
||||
|
||||
const finderModule = {
|
||||
FileFinder: {
|
||||
create: (options: Record<string, unknown>) => ({
|
||||
ok: true,
|
||||
value: createMockFinder(options.basePath as string),
|
||||
}),
|
||||
},
|
||||
};
|
||||
|
||||
mock.module("@ff-labs/fff-node", () => finderModule);
|
||||
mock.module("@ff-labs/fff-bun", () => finderModule);
|
||||
|
||||
const { AuxFinderPool } = await import("../src/aux-finders");
|
||||
|
||||
describe("AuxFinderPool concurrent dedup (#746)", () => {
|
||||
test("two concurrent acquires for same root share one finder", async () => {
|
||||
created.length = 0;
|
||||
const pool = new AuxFinderPool({ enableFsRootScanning: false });
|
||||
const [a, b] = await Promise.all([
|
||||
pool.acquire("/Users/x"),
|
||||
pool.acquire("/Users/x"),
|
||||
]);
|
||||
expect(created.length).toBe(1);
|
||||
expect(a.finder).toBe(b.finder);
|
||||
});
|
||||
|
||||
test("sequential acquire after in-flight one resolves still reuses", async () => {
|
||||
created.length = 0;
|
||||
const pool = new AuxFinderPool({ enableFsRootScanning: false });
|
||||
const first = pool.acquire("/Users/x");
|
||||
const second = pool.acquire("/Users/x");
|
||||
await Promise.all([first, second]);
|
||||
const third = await pool.acquire("/Users/x");
|
||||
expect(created.length).toBe(1);
|
||||
expect(third.root).toBe("/Users/x");
|
||||
});
|
||||
});
|
||||
@@ -98,9 +98,7 @@ describe("routePathConstraint", () => {
|
||||
});
|
||||
|
||||
test("returns null when .. resolves back inside the workspace", () => {
|
||||
expect(
|
||||
routePathConstraint("../workspace/src", workspace),
|
||||
).toBeNull();
|
||||
expect(routePathConstraint("../workspace/src", workspace)).toBeNull();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import { describe, expect, mock, test } from "bun:test";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
interface MockFinder {
|
||||
isDestroyed: boolean;
|
||||
@@ -40,10 +42,10 @@ mock.module("@ff-labs/fff-bun", () => finderModule);
|
||||
|
||||
const { AuxFinderPool } = await import("../src/aux-finders");
|
||||
|
||||
function makePool() {
|
||||
function makePool(opts: Record<string, unknown> = {}) {
|
||||
created.length = 0;
|
||||
createOptions.length = 0;
|
||||
return new AuxFinderPool({ enableFsRootScanning: false });
|
||||
return new AuxFinderPool({ enableFsRootScanning: false, ...opts });
|
||||
}
|
||||
|
||||
describe("AuxFinderPool covering reuse", () => {
|
||||
@@ -92,16 +94,51 @@ describe("AuxFinderPool covering reuse", () => {
|
||||
expect(created.length).toBe(2);
|
||||
});
|
||||
|
||||
// Regression for #700: aux finders must not reopen the main frecency/history
|
||||
// LMDB envs, or heed fails with "environment already open in this program".
|
||||
test("aux finders are created without frecency/history db paths", async () => {
|
||||
const pool = makePool();
|
||||
// Regression for #743: the agent spawning an aux picker over $HOME must warn
|
||||
// the user every time, not silently walk the home tree.
|
||||
test("notifies on every aux picker that covers $HOME", async () => {
|
||||
const onHomeDirScan = mock(() => undefined);
|
||||
const pool = makePool({ onHomeDirScan });
|
||||
const home = os.homedir();
|
||||
|
||||
await pool.acquire(home);
|
||||
await pool.acquire(path.dirname(home));
|
||||
expect(onHomeDirScan.mock.calls).toEqual([[home], [path.dirname(home)]]);
|
||||
|
||||
// Project-scoped roots below $HOME do not walk the whole home tree.
|
||||
await pool.acquire(path.join(home, "dev", "some-project"));
|
||||
expect(onHomeDirScan).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
test("no aux notification when home scanning is disabled", async () => {
|
||||
const onHomeDirScan = mock(() => undefined);
|
||||
const pool = makePool({ enableHomeDirScanning: false, onHomeDirScan });
|
||||
|
||||
await pool.acquire(os.homedir());
|
||||
expect(onHomeDirScan).not.toHaveBeenCalled();
|
||||
expect(createOptions[0].enableHomeDirScanning).toBe(false);
|
||||
});
|
||||
|
||||
// #700 is fixed by the process-wide LMDB env pool: same-path opens share one
|
||||
// env, so aux finders now reuse the session's frecency/history DBs.
|
||||
test("aux finders receive the pool's frecency/history db paths", async () => {
|
||||
const pool = makePool({
|
||||
frecencyDbPath: "/dbs/frecency",
|
||||
historyDbPath: "/dbs/history",
|
||||
});
|
||||
await pool.acquire("/a/b/c");
|
||||
await pool.acquire("/x/y");
|
||||
expect(createOptions.length).toBe(2);
|
||||
for (const opts of createOptions) {
|
||||
expect(opts.frecencyDbPath).toBeUndefined();
|
||||
expect(opts.historyDbPath).toBeUndefined();
|
||||
expect(opts.frecencyDbPath).toBe("/dbs/frecency");
|
||||
expect(opts.historyDbPath).toBe("/dbs/history");
|
||||
}
|
||||
});
|
||||
|
||||
test("aux finders stay db-less when the session has no db paths", async () => {
|
||||
const pool = makePool();
|
||||
await pool.acquire("/a/b/c");
|
||||
expect(createOptions[0].frecencyDbPath).toBeUndefined();
|
||||
expect(createOptions[0].historyDbPath).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,20 +1,35 @@
|
||||
import { beforeEach, describe, expect, mock, test } from "bun:test";
|
||||
import os from "node:os";
|
||||
|
||||
type MockFinder = {
|
||||
isDestroyed: boolean;
|
||||
waitForScan: ReturnType<typeof mock>;
|
||||
mixedSearch: ReturnType<typeof mock>;
|
||||
getScanProgress: ReturnType<typeof mock>;
|
||||
destroy: ReturnType<typeof mock>;
|
||||
};
|
||||
|
||||
const createCalls: unknown[] = [];
|
||||
let finders: MockFinder[] = [];
|
||||
let mixedSearchImpl: ((query: string, options: unknown) => unknown) | undefined;
|
||||
let scanProgressImpl: (() => unknown) | undefined;
|
||||
|
||||
function createMockFinder(): MockFinder {
|
||||
return {
|
||||
isDestroyed: false,
|
||||
waitForScan: mock(async () => undefined),
|
||||
getScanProgress: mock(() => {
|
||||
if (scanProgressImpl) return scanProgressImpl();
|
||||
return {
|
||||
ok: true,
|
||||
value: {
|
||||
scannedFilesCount: 0,
|
||||
isScanning: false,
|
||||
isWatcherReady: true,
|
||||
isWarmupComplete: true,
|
||||
},
|
||||
};
|
||||
}),
|
||||
mixedSearch: mock((query: string, options: unknown) => {
|
||||
if (mixedSearchImpl) return mixedSearchImpl(query, options);
|
||||
return {
|
||||
@@ -102,20 +117,21 @@ function createPi(mode?: string) {
|
||||
return { pi, events, commands };
|
||||
}
|
||||
|
||||
function createContext() {
|
||||
function createContext(cwd = "/tmp/workspace") {
|
||||
return {
|
||||
cwd: "/tmp/workspace",
|
||||
cwd,
|
||||
ui: {
|
||||
addAutocompleteProvider: mock(() => undefined),
|
||||
notify: mock(() => undefined),
|
||||
setEditorComponent: mock(() => undefined),
|
||||
setStatus: mock(() => undefined),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function start(mode?: string) {
|
||||
async function start(mode?: string, cwd?: string) {
|
||||
const setup = createPi(mode);
|
||||
const ctx = createContext();
|
||||
const ctx = createContext(cwd);
|
||||
fffExtension(setup.pi as any);
|
||||
|
||||
const sessionStart = setup.events.get("session_start");
|
||||
@@ -125,6 +141,10 @@ async function start(mode?: string) {
|
||||
return { ...setup, ctx };
|
||||
}
|
||||
|
||||
async function shutdown(setup: { events: Map<string, EventHandler> }) {
|
||||
await setup.events.get("session_shutdown")?.({}, undefined);
|
||||
}
|
||||
|
||||
function currentProvider(
|
||||
result = { items: [{ value: "base", label: "base" }], prefix: "ba" },
|
||||
) {
|
||||
@@ -143,7 +163,71 @@ beforeEach(() => {
|
||||
createCalls.length = 0;
|
||||
finders = [];
|
||||
mixedSearchImpl = undefined;
|
||||
scanProgressImpl = undefined;
|
||||
delete process.env.PI_FFF_MODE;
|
||||
delete process.env.FFF_ENABLE_HOME_SCAN;
|
||||
});
|
||||
|
||||
// Regression for #743: launching from $HOME must be visible and interruptible.
|
||||
describe("pi-fff $HOME scan warning", () => {
|
||||
test("warns and pins a status when cwd is $HOME", async () => {
|
||||
const setup = await start(undefined, os.homedir());
|
||||
|
||||
expect(setup.ctx.ui.notify).toHaveBeenCalledTimes(1);
|
||||
const [message, level] = setup.ctx.ui.notify.mock.calls[0];
|
||||
expect(message).toContain(os.homedir());
|
||||
expect(level).toBe("warning");
|
||||
expect(setup.ctx.ui.setStatus).toHaveBeenCalledWith(
|
||||
"fff",
|
||||
"Agent is indexing $HOME, this can lead to high CPU",
|
||||
);
|
||||
await shutdown(setup);
|
||||
});
|
||||
|
||||
test("stays silent outside $HOME", async () => {
|
||||
const { ctx } = await start();
|
||||
|
||||
expect(ctx.ui.notify).not.toHaveBeenCalled();
|
||||
expect(ctx.ui.setStatus).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test("clears the status once the scan settles", async () => {
|
||||
const setup = await start(undefined, os.homedir());
|
||||
|
||||
expect(setup.ctx.ui.setStatus).toHaveBeenLastCalledWith("fff", undefined);
|
||||
await shutdown(setup);
|
||||
});
|
||||
|
||||
// waitForScan resolves on timeout, so a slow $HOME walk keeps the footer up.
|
||||
test("keeps reporting live progress while the scan is still running", async () => {
|
||||
scanProgressImpl = () => ({
|
||||
ok: true,
|
||||
value: {
|
||||
scannedFilesCount: 12345,
|
||||
isScanning: true,
|
||||
isWatcherReady: false,
|
||||
isWarmupComplete: false,
|
||||
},
|
||||
});
|
||||
const setup = await start(undefined, os.homedir());
|
||||
|
||||
const [key, text] = setup.ctx.ui.setStatus.mock.calls.at(-1) as [string, string];
|
||||
expect(key).toBe("fff");
|
||||
expect(text).toContain("12345 files");
|
||||
|
||||
// session_shutdown must stop the poller and clear the footer.
|
||||
await shutdown(setup);
|
||||
expect(setup.ctx.ui.setStatus).toHaveBeenLastCalledWith("fff", undefined);
|
||||
});
|
||||
|
||||
test("no warning when home scanning is disabled", async () => {
|
||||
process.env.FFF_ENABLE_HOME_SCAN = "0";
|
||||
const setup = await start(undefined, os.homedir());
|
||||
|
||||
expect(setup.ctx.ui.notify).not.toHaveBeenCalled();
|
||||
expect(setup.ctx.ui.setStatus).not.toHaveBeenCalled();
|
||||
await shutdown(setup);
|
||||
});
|
||||
});
|
||||
|
||||
describe("pi-fff autocomplete registration", () => {
|
||||
@@ -164,6 +248,14 @@ describe("pi-fff autocomplete registration", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
test("FFF_ENABLE_HOME_SCAN=0 disables home dir scanning", async () => {
|
||||
process.env.FFF_ENABLE_HOME_SCAN = "0";
|
||||
await start();
|
||||
|
||||
const opts = createCalls[0] as { enableHomeDirScanning: boolean };
|
||||
expect(opts.enableHomeDirScanning).toBe(false);
|
||||
});
|
||||
|
||||
test("session_start survives hosts without addAutocompleteProvider", async () => {
|
||||
const setup = createPi();
|
||||
const ctx = {
|
||||
|
||||
@@ -567,16 +567,10 @@ export interface FileFinderApi {
|
||||
glob(pattern: string, options?: GlobOptions): Result<SearchResult>;
|
||||
|
||||
/** Fuzzy directory search. */
|
||||
directorySearch(
|
||||
query: string,
|
||||
options?: DirSearchOptions,
|
||||
): Result<DirSearchResult>;
|
||||
directorySearch(query: string, options?: DirSearchOptions): Result<DirSearchResult>;
|
||||
|
||||
/** Fuzzy search over files and directories interleaved by score. */
|
||||
mixedSearch(
|
||||
query: string,
|
||||
options?: SearchOptions,
|
||||
): Result<MixedSearchResult>;
|
||||
mixedSearch(query: string, options?: SearchOptions): Result<MixedSearchResult>;
|
||||
|
||||
/** Content search (live grep). */
|
||||
grep(query: string, options?: GrepOptions): Result<GrepResult>;
|
||||
@@ -641,10 +635,7 @@ export interface FileFinderApi {
|
||||
* Events are debounced and submitted in batches per 100-ms window at most 128 events.
|
||||
* Gitignored and other ignored files are never triggering watcher.
|
||||
*/
|
||||
watch(
|
||||
callback: WatchBatchCallback,
|
||||
options?: WatchOptions,
|
||||
): Result<WatchUnsubscribe>;
|
||||
watch(callback: WatchBatchCallback, options?: WatchOptions): Result<WatchUnsubscribe>;
|
||||
watch(
|
||||
pattern: string,
|
||||
callback: WatchBatchCallback,
|
||||
|
||||
+1
-1
@@ -77,4 +77,4 @@ git push origin "$TAG"
|
||||
|
||||
echo ""
|
||||
echo "Release $VERSION created and pushed."
|
||||
echo "CI will build and publish: https://github.com/dmtrKovalenko/fff.nvim/actions"
|
||||
echo "CI will build and publish: https://github.com/dmtrKovalenko/fff/actions"
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user