chore(fff-core): clippy 1.98 (#808)

clippy 1.98 added chunks_exact_to_as_chunks, which fails CI under
-D warnings. SEEN_WORDS is a multiple of 8, so the remainder is
always empty and the fixed-size chunk keeps codegen identical or
better.

Co-authored-by: gustav-fff <286169375+gustav-fff@users.noreply.github.com>

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,58 @@
+name: Bug report
+description: Report a crash, hang, or incorrect behavior in fff (any frontend — nvim plugin, Node/Bun SDK, MCP server, C SDK).
+title: "[Bug]: "
+labels: ["bug"]
+body:
+  - type: dropdown
+    id: frontend
+    attributes:
+      label: Which fff frontend?
+      options:
+        - Neovim plugin (fff.nvim)
+        - MCP server (fff-mcp)
+        - Node SDK (@ff-labs/fff-node)
+        - Bun SDK
+        - C SDK (libfff)
+        - Other / multiple
+    validations:
+      required: true
+
+  - type: textarea
+    id: logs
+    attributes:
+      label: has logs
+      description: |
+        Attach your fff log file — the single most useful thing for debugging.
+
+        fff writes a fresh log file on every process startup, named `fff+<unix-ts>+<pid>.log`, and keeps the last 20. Find the file matching your crashed/buggy run and drag-and-drop it here (or paste its contents).
+
+        Where the log files live:
+
+        | Frontend | Linux / macOS | Windows |
+        |---|---|---|
+        | Neovim plugin | `~/.local/state/nvim/log/fff+*.log` | `%LOCALAPPDATA%\nvim-data\log\fff+*.log` |
+        | MCP server (`fff-mcp`) | `~/.cache/fff_mcp+*.log` (override with `--log-file`) | `%LOCALAPPDATA%\fff_mcp+*.log` |
+        | Node / Bun SDK | path you passed as `logFilePath` to `FileFinder.create({...})` | same |
+        | C SDK | path you passed as `log_file_path` in `FffCreateOptions` | same |
+
+        Neovim users: run `:FFFOpenLog` to open the current session's log directly.
+
+        Also some useful commands:
+
+        ```sh
+        # Neovim plugin
+        ls -t ~/.local/state/nvim/log/fff+*.log | head -1
+
+        # MCP server
+        ls -t ~/.cache/fff_mcp+*.log | head -1
+        ```
+    validations:
+      required: false
+
+  - type: textarea
+    id: body
+    attributes:
+      label: Description
+      description: Please provide as much helpful information as you can
+    validations:
+      required: true

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,5 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Discussion / question
+    url: https://github.com/dmtrKovalenko/fff/discussions
+    about: For usage questions, design discussion, or anything that's not a bug or feature request.

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,34 @@
+name: Feature request
+description: Suggest something new for fff
+title: "[Suggestion]: "
+labels: ["enhancement"]
+body:
+  - type: dropdown
+    id: frontend
+    attributes:
+      label: Which fff frontend(s)?
+      multiple: true
+      options:
+        - Neovim plugin (fff.nvim)
+        - MCP server (fff-mcp)
+        - Node SDK (@ff-labs/fff-node)
+        - Bun SDK (@ff-labs/fff-bun)
+        - C lib (libfff)
+        - Core or Rust crate
+    validations:
+      required: true
+
+  - type: textarea
+    id: problem
+    attributes:
+      label: What problem are you trying to solve?
+    validations:
+      required: true
+
+  - type: textarea
+    id: proposal
+    attributes:
+      label: Proposed solution
+      description: If you have an idea of the shape of the API, describe it here.
+    validations:
+      required: false

.github/workflows/external-tests.yml

@@ -3,8 +3,14 @@ name: e2e Tests
 on:
   push:
     branches: [main]
+    paths-ignore:
+      - '**.md'
+      - 'doc/**'
   pull_request:
     branches: [main]
+    paths-ignore:
+      - '**.md'
+      - 'doc/**'
 
 env:
   CARGO_TERM_COLOR: always
@@ -12,6 +18,14 @@ env:
   # Force Node 24 for all JS-based actions to avoid the libuv
   # process_title assertion crash on Windows (known Node 20 bug).
   FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
+  # e2e only needs a working binary, so skip fat LTO (same settings as the `ci`
+  # profile releases ship). Overriding release keeps artifacts in target/release.
+  CARGO_PROFILE_RELEASE_LTO: thin
+  CARGO_PROFILE_RELEASE_CODEGEN_UNITS: 16
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' && !startsWith(github.ref, 'refs/tags/') }}
 
 jobs:
   lua-tests:
@@ -19,6 +33,13 @@ jobs:
     runs-on: ${{ matrix.os }}
     # e2e tests could be flaky on CI so we do not block release creation if they failed
     continue-on-error: ${{ github.ref == 'refs/heads/main' && github.event_name == 'push' }}
+    # alpine-musl installs rust via apk and has no sccache, so keep this scoped.
+    env:
+      SCCACHE_GHA_ENABLED: "true"
+      RUSTC_WRAPPER: sccache
+      # fff-search alone exceeds the 600s default on windows, and the server
+      # sees no new requests while it compiles, so it would idle out mid-unit.
+      SCCACHE_IDLE_TIMEOUT: "0"
     strategy:
       fail-fast: false
       matrix:
@@ -26,15 +47,14 @@ jobs:
           - os: ubuntu-latest
           - os: macos-latest
           - os: windows-latest
-            target: x86_64-pc-windows-msvc
     steps:
       - uses: actions/checkout@v5
       - uses: oven-sh/setup-bun@v2
-
+      - uses: actions/setup-node@v6
       - name: Install Zig
         uses: mlugg/setup-zig@v2
         with:
-          version: 0.15.2
+          version: 0.16.0
 
       - name: Install Rust
         uses: actions-rust-lang/setup-rust-toolchain@v1.15.4
@@ -43,20 +63,23 @@ jobs:
           cache-on-failure: false
           cache-key: "v2-lua-e2e"
           rustflags: ""
-          target: ${{ matrix.target || '' }}
 
-      - name: Build Rust binary (Windows)
-        if: matrix.target
-        run: cargo build --release --target ${{ matrix.target }} -p fff-nvim --features zlob
+      - name: Setup sccache
+        uses: mozilla-actions/sccache-action@v0.0.11
+        with:
+          version: v0.17.0
 
-      - name: Copy binary to target/release (Windows)
-        if: matrix.target
+      - name: Build Rust binary
         shell: bash
-        run: |
-          cp target/${{ matrix.target }}/release/fff_nvim.dll target/release/fff_nvim.dll
+        run: make build-e2e
+
+      - name: sccache stats
+        if: always()
+        shell: bash
+        run: sccache --show-stats
 
       - name: Verify Windows DLL has no unexpected dependencies
-        if: matrix.target
+        if: matrix.os == 'windows-latest'
         shell: pwsh
         run: |
           # Find dumpbin via vswhere (always available on GitHub Actions Windows runners)
@@ -72,10 +95,6 @@ jobs:
             exit 1
           }
 
-      - name: Build Rust binary
-        if: ${{ !matrix.target }}
-        run: cargo build --release -p fff-nvim --features zlob
-
       - name: Install Neovim
         uses: rhysd/action-setup-vim@v1
         with:
@@ -90,27 +109,128 @@ jobs:
         shell: bash
         run: make test-lua
 
+      - name: Dump fff trace log on failure
+        if: failure()
+        shell: bash
+        run: |
+          # init_tracing writes session files named fff-test+<ts>+<pid>.log
+          found=0
+          for f in fff-test*.log; do
+            [ -f "$f" ] || continue
+            found=1
+            echo "=== $f ==="
+            cat "$f"
+          done
+          if [ "$found" = 0 ]; then
+            echo "(no log file produced)"
+          fi
+
       - name: Run version resolution tests
         shell: bash
         run: make test-version
 
-      - name: Run bun tests
+      - name: Run non windows tests
         shell: bash
-        if: ${{ matrix.os == 'macos-latest' }}
-        run: make test-bun
+        if: ${{ matrix.os != 'windows-latest' }}
+        run: |
+          make test-bun
+          make test-c-api
+
+      - name: Verify bun --compile 
+        shell: bash
+        run: make test-bun-compile
 
       - name: Install Node.js
         if: ${{ matrix.os != 'ubuntu-latest' }}
         uses: actions/setup-node@v6
         with:
-          node-version: "24"
+          node-version: "25"
 
-      - name: Install node deps
-        if: ${{ matrix.os != 'ubuntu-latest' }}
+      - name: Install node dependencies
         shell: bash
-        run: npm install
+        run: cd packages && npm ci
 
       - name: Run node tests
-        if: ${{ matrix.os != 'ubuntu-latest' }}
         shell: bash
         run: make test-node
+
+  # Regression for https://github.com/dmtrKovalenko/fff/issues/480: build &
+  # run @ff-labs/fff-node end-to-end on real Alpine Linux (musl). Forces
+  # findBinary() through the npm-package resolver so detectLinuxLibc()
+  # actually runs.
+  alpine-musl:
+    name: e2e (alpine-musl)
+    runs-on: ubuntu-latest
+    container: node:22-alpine
+    continue-on-error: ${{ github.ref == 'refs/heads/main' && github.event_name == 'push' }}
+    defaults:
+      run:
+        shell: sh
+    steps:
+      - name: Install build deps
+        run: apk add --no-cache git rust cargo musl-dev
+
+      - uses: actions/checkout@v5
+
+      # libgit2 refuses repos owned by a different user; checkout in a
+      # container can land at a uid mismatch, so opt every dir in.
+      - name: Mark workspace safe for git
+        run: git config --global --add safe.directory '*'
+
+      - name: Sanity check libc is musl
+        run: |
+          if ! ldd --version 2>&1 | grep -qi musl; then
+            echo "FAIL: container is not running musl libc"
+            exit 1
+          fi
+
+      - uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            target
+          key: alpine-musl-cargo-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: |
+            alpine-musl-cargo-
+
+      - name: Build libfff_c (musl)
+        run: cargo build --release -p fff-c
+
+      - name: Install workspace npm deps
+        working-directory: packages
+        run: npm ci
+
+      # Upstream @yuuang/ffi-rs-linux-x64-musl ships with libc:"glibc" in
+      # its package.json (a publishing bug in ffi-rs), so npm filters it
+      # out. Force-install it so the FFI runtime is present on Alpine.
+      - name: Install ffi-rs musl runtime
+        working-directory: packages
+        run: |
+          FFI_RS_VERSION=$(node -p "require('ffi-rs/package.json').version")
+          npm install --no-package-lock --no-save --force \
+            "@yuuang/ffi-rs-linux-x64-musl@${FFI_RS_VERSION}"
+
+      # Stage the freshly built libfff_c.so as the platform npm package
+      # so findBinary() resolves through the @ff-labs/fff-bin-* path —
+      # this is what exercises detectLinuxLibc().
+      - name: Stage musl bin package
+        run: |
+          PKG_DIR=packages/node_modules/@ff-labs/fff-bin-linux-x64-musl
+          mkdir -p "$PKG_DIR"
+          cp target/release/libfff_c.so "$PKG_DIR/libfff_c.so"
+          cat >"$PKG_DIR/package.json" <<'JSON'
+          { "name": "@ff-labs/fff-bin-linux-x64-musl", "version": "0.0.0" }
+          JSON
+
+      # fff-node builds with bun; the npm `bun` package ships musl binaries
+      - name: Install bun
+        run: npm install -g bun
+
+      - name: Build fff-node
+        working-directory: packages/fff-node
+        run: npm run build
+
+      - name: Run fff-node e2e suite
+        working-directory: packages/fff-node
+        run: node test/e2e.mjs

.github/workflows/lua.yml

@@ -3,8 +3,18 @@ name: Lua CI
 on:
   push:
     branches: [main]
+    paths-ignore:
+      - '**.md'
+      - 'doc/**'
   pull_request:
     branches: [main]
+    paths-ignore:
+      - '**.md'
+      - 'doc/**'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' && !startsWith(github.ref, 'refs/tags/') }}
 
 jobs:
   lua-ls:

.github/workflows/nix.yml

@@ -3,8 +3,18 @@ name: Nix CI
 on:
   push:
     branches: [main]
+    paths-ignore:
+      - '**.md'
+      - 'doc/**'
   pull_request:
     branches: [main]
+    paths-ignore:
+      - '**.md'
+      - 'doc/**'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' && !startsWith(github.ref, 'refs/tags/') }}
 
 jobs:
   check:

.github/workflows/oxc.yml

@@ -0,0 +1,46 @@
+name: Oxc
+
+permissions:
+  contents: read
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - "packages/**"
+      - .github/workflows/oxc.yml
+  pull_request:
+    paths:
+      - "packages/**"
+      - .github/workflows/oxc.yml
+
+env:
+  CLICOLOR: 1
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' && !startsWith(github.ref, 'refs/tags/') }}
+
+jobs:
+  oxc:
+    name: Check packages with oxfmt and oxlint
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: packages
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          persist-credentials: false
+
+      - uses: oven-sh/setup-bun@v2
+
+      - name: Install
+        run: bun install --frozen-lockfile
+
+      - name: oxfmt
+        run: bun run format:check
+
+      - name: oxlint
+        run: bun run lint -- -f github --deny-warnings

.github/workflows/panvimdoc.yaml

@@ -1,7 +1,8 @@
 on:
-  push:
-    branches-ignore:
-      - main
+  schedule:
+    - cron: "0 4 * * *"
+  workflow_dispatch:
+
 name: docs
 
 jobs:
@@ -9,27 +10,76 @@ jobs:
     runs-on: ubuntu-latest
     permissions:
       contents: write
+      pull-requests: write
     steps:
       - uses: actions/checkout@v5
         with:
-          # fetch last 2 commits required for auto force push back
+          ref: main
           fetch-depth: 2
 
+      - name: Extract Neovim section from README.md
+        run: |
+          awk '
+            /^<details id="neovim-plugin">/ { capture=1; next }
+            capture && /^<\/details>/        { capture=0; exit }
+            capture && /^<summary>$/         { next }
+            capture && /^<\/summary>$/       { next }
+            capture && /<h2>.*<\/h2>/ {
+              gsub(/<\/?h2>/, "")
+              sub(/^[[:space:]]+/, "")
+              print "# " $0
+              print ""
+              print "The best file search picker for Neovim. Frecency-ranked, typo-resistant, git-award, very fast."
+              print ""
+              next
+            }
+            capture { print }
+          ' README.md > .panvimdoc-input.md
+          test -s .panvimdoc-input.md
+
       - name: panvimdoc
         uses: kdheepak/panvimdoc@main
         with:
           vimdoc: fff.nvim
+          pandoc: .panvimdoc-input.md
           version: "Neovim >= 0.10.0"
           demojify: true
           treesitter: true
 
-      - name: Get last commit message
-        id: last-commit
-        run: |
-          echo "message=$(git log -1 --pretty=%s)" >> $GITHUB_OUTPUT
-          echo "author=$(git log -1 --pretty=\"%an <%ae>\")" >> $GITHUB_OUTPUT
+      - name: Cleanup intermediate file
+        run: rm -f .panvimdoc-input.md
 
-      - uses: stefanzweifel/git-auto-commit-action@v6
+      # panvimdoc stamps "Last change: <today>" every run, so a daily cron always
+      # produces a one-line diff. Skip the PR unless a non-date line changed.
+      - name: Detect real doc changes
+        id: docdiff
+        run: |
+          if git diff --quiet -I 'Last change:' -- doc/fff.nvim.txt; then
+            echo "changed=false" >> "$GITHUB_OUTPUT"
+          else
+            echo "changed=true" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Create pull request
+        id: cpr
+        if: steps.docdiff.outputs.changed == 'true'
+        uses: peter-evans/create-pull-request@v7
         with:
-          commit_author: ${{ steps.last-commit.outputs.author }}
-          commit_message: "chore: Update docs for - ${{ steps.last-commit.outputs.message }}"
+          branch: bot/regenerate-vimdoc
+          token: ${{ secrets.GUSTAV_PAT }}
+          delete-branch: true
+          title: "chore: regenerate Neovim vimdoc"
+          commit-message: |
+            chore: regenerate Neovim vimdoc
+
+            Co-authored-by: Dmitriy Kovalenko <dmtr.kovalenko@outlook.com>
+          author: "gustav-fff <66k7bxj9m6@privaterelay.appleid.com>"
+          committer: "gustav-fff <66k7bxj9m6@privaterelay.appleid.com>"
+          body: Automated vimdoc regeneration from README.md, scribed by Gustav.
+          add-paths: doc/fff.nvim.txt
+
+      - name: Enable auto-merge
+        if: steps.cpr.outputs.pull-request-number
+        env:
+          GH_TOKEN: ${{ secrets.GUSTAV_PAT }}
+        run: gh pr merge --auto --squash "${{ steps.cpr.outputs.pull-request-number }}"

.github/workflows/python.yml

@@ -0,0 +1,70 @@
+name: Python CI
+
+on:
+  push:
+    branches: [main]
+    paths-ignore:
+      - '**.md'
+      - 'doc/**'
+  pull_request:
+    branches: [main]
+    paths-ignore:
+      - '**.md'
+      - 'doc/**'
+
+env:
+  CARGO_TERM_COLOR: always
+  MACOSX_DEPLOYMENT_TARGET: "13.0"
+  CARGO_PROFILE_RELEASE_LTO: thin
+  CARGO_PROFILE_RELEASE_CODEGEN_UNITS: 16
+  SCCACHE_GHA_ENABLED: "true"
+  RUSTC_WRAPPER: sccache
+  # fff-search alone exceeds the 600s default on windows, and the server sees
+  # no new requests while it compiles, so it would idle out mid-unit.
+  SCCACHE_IDLE_TIMEOUT: "0"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' && !startsWith(github.ref, 'refs/tags/') }}
+
+jobs:
+  test:
+    name: Python bindings (${{ matrix.os }})
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          persist-credentials: false
+
+      - name: Install Rust
+        uses: actions-rust-lang/setup-rust-toolchain@v1.15.4
+        with:
+          cache: true
+          cache-on-failure: true
+          cache-key: "v1-rust-python"
+
+      - name: Setup sccache
+        uses: mozilla-actions/sccache-action@v0.0.11
+        with:
+          version: v0.17.0
+
+      - name: Setup uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          version: "0.11.14"
+          enable-cache: true
+
+      - name: Build and test Python bindings
+        working-directory: packages/fff-python
+        shell: bash
+        run: |
+          # --no-install-project: uv would otherwise build the wheel via
+          # maturin's PEP 517 backend, then `maturin develop` builds it again.
+          uv sync --all-extras --no-install-project
+          uv run maturin develop --release
+          uv run pytest -v

.github/workflows/release.yaml

@@ -1,69 +1,87 @@
-name: Prebuild
+name: Build & Publish
 
 on:
   push:
-    branches: [main, fix/download-version]
+    branches: [main, fix/npm-pulish]
     tags:
       - "v*"
   pull_request:
+  workflow_dispatch:
+    inputs:
+      publish_pypi:
+        description: "Manually build and publish Python wheels to PyPI"
+        required: false
+        default: false
+        type: boolean
 
 env:
   FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' && !startsWith(github.ref, 'refs/tags/') }}
+
 jobs:
   build-nvim:
     name: Build Neovim ${{ matrix.target }}
     runs-on: ${{ matrix.os }}
+    env:
+      SCCACHE_GHA_ENABLED: "true"
+      RUSTC_WRAPPER: sccache
+      SCCACHE_IDLE_TIMEOUT: "0"
     permissions:
       contents: read
+      id-token: write
     strategy:
       matrix:
         include:
           ## Linux builds (using cargo-zigbuild)
-          # Glibc 2.17 (RHEL 7, CentOS 7 compatible)
+          # Glibc 2.31 (Ubuntu 20.04, Debian 11, RHEL 9).
+          # Rust 1.91+ requires glibc >= 2.31 for std::sys::random::getrandom,
+          # copy_file_range, and statx; earlier targets (2.17) no longer link.
           - os: ubuntu-latest
             target: x86_64-unknown-linux-gnu
-            zigbuild_target: x86_64-unknown-linux-gnu.2.17
-            artifact_name: target/x86_64-unknown-linux-gnu/release/libfff_nvim.so
+            zigbuild_target: x86_64-unknown-linux-gnu.2.31
+            artifact_name: target/x86_64-unknown-linux-gnu/ci/libfff_nvim.so
             ext: so
           - os: ubuntu-latest
             target: aarch64-unknown-linux-gnu
-            zigbuild_target: aarch64-unknown-linux-gnu.2.17
-            artifact_name: target/aarch64-unknown-linux-gnu/release/libfff_nvim.so
+            zigbuild_target: aarch64-unknown-linux-gnu.2.31
+            artifact_name: target/aarch64-unknown-linux-gnu/ci/libfff_nvim.so
             ext: so
           # Musl (statically linked)
           - os: ubuntu-latest
             target: x86_64-unknown-linux-musl
-            artifact_name: target/x86_64-unknown-linux-musl/release/libfff_nvim.so
+            artifact_name: target/x86_64-unknown-linux-musl/ci/libfff_nvim.so
             ext: so
           - os: ubuntu-latest
             target: aarch64-unknown-linux-musl
-            artifact_name: target/aarch64-unknown-linux-musl/release/libfff_nvim.so
+            artifact_name: target/aarch64-unknown-linux-musl/ci/libfff_nvim.so
             ext: so
 
           ## Android (Termux)
           - os: ubuntu-latest
             target: aarch64-linux-android
-            artifact_name: target/aarch64-linux-android/release/libfff_nvim.so
+            artifact_name: target/aarch64-linux-android/ci/libfff_nvim.so
             ext: so
 
           ## macOS builds
           - os: macos-latest
             target: x86_64-apple-darwin
-            artifact_name: target/x86_64-apple-darwin/release/libfff_nvim.dylib
+            artifact_name: target/x86_64-apple-darwin/ci/libfff_nvim.dylib
             ext: dylib
           - os: macos-latest
             target: aarch64-apple-darwin
-            artifact_name: target/aarch64-apple-darwin/release/libfff_nvim.dylib
+            artifact_name: target/aarch64-apple-darwin/ci/libfff_nvim.dylib
             ext: dylib
 
           - os: windows-latest
             target: x86_64-pc-windows-msvc
-            artifact_name: target/x86_64-pc-windows-msvc/release/fff_nvim.dll
+            artifact_name: target/x86_64-pc-windows-msvc/ci/fff_nvim.dll
             ext: dll
           - os: windows-latest
             target: aarch64-pc-windows-msvc
-            artifact_name: target/aarch64-pc-windows-msvc/release/fff_nvim.dll
+            artifact_name: target/aarch64-pc-windows-msvc/ci/fff_nvim.dll
             ext: dll
 
     steps:
@@ -74,10 +92,22 @@ jobs:
       - name: Install Rust
         run: rustup target add ${{ matrix.target }}
 
+      # Cache the per-target build dir and cargo-zigbuild binary. Keyed by
+      # target so matrix legs don't collide. See issue on slow release CI.
+      - name: Rust cache
+        uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4
+        with:
+          key: nvim-${{ matrix.target }}
+
+      - name: Setup sccache
+        uses: mozilla-actions/sccache-action@v0.0.11
+        with:
+          version: v0.17.0
+
       - name: Install Zig
         uses: mlugg/setup-zig@v2
         with:
-          version: 0.15.2
+          version: 0.16.0
 
       - name: Install cargo-zigbuild
         if: contains(matrix.os, 'ubuntu')
@@ -86,7 +116,7 @@ jobs:
       - name: Build for Linux
         if: contains(matrix.os, 'ubuntu') && !contains(matrix.target, 'android')
         run: |
-          cargo zigbuild --release --target ${{ matrix.zigbuild_target || matrix.target }} -p fff-nvim --features zlob
+          cargo zigbuild --profile ci --target ${{ matrix.zigbuild_target || matrix.target }} -p fff-nvim --no-default-features --features zlob
           mv "${{ matrix.artifact_name }}" "${{ matrix.target }}.${{ matrix.ext }}"
 
       - name: Build for Android (Termux)
@@ -100,13 +130,13 @@ jobs:
           export AR_aarch64_linux_android="$NDK_BIN/llvm-ar"
           export CARGO_TARGET_AARCH64_LINUX_ANDROID_LINKER="$NDK_BIN/aarch64-linux-android24-clang"
 
-          cargo build --release --target ${{ matrix.target }} -p fff-nvim --features zlob
+          cargo build --profile ci --target ${{ matrix.target }} -p fff-nvim --no-default-features --features zlob
           mv "${{ matrix.artifact_name }}" "${{ matrix.target }}.${{ matrix.ext }}"
 
       - name: Build for macOS
         if: contains(matrix.os, 'macos')
         run: |
-          MACOSX_DEPLOYMENT_TARGET="13" cargo build --release --target ${{ matrix.target }} -p fff-nvim --features zlob
+          MACOSX_DEPLOYMENT_TARGET="13" cargo build --profile ci --target ${{ matrix.target }} -p fff-nvim --no-default-features --features zlob
           mv "${{ matrix.artifact_name }}" "${{ matrix.target }}.${{ matrix.ext }}"
 
       - name: Ad-hoc sign macOS binary
@@ -117,7 +147,7 @@ jobs:
         if: contains(matrix.os, 'windows')
         shell: bash
         run: |
-          cargo build --release --target ${{ matrix.target }} -p fff-nvim --features zlob
+          cargo build --profile ci --target ${{ matrix.target }} -p fff-nvim --no-default-features --features zlob
           mv "${{ matrix.artifact_name }}" "${{ matrix.target }}.${{ matrix.ext }}"
 
       - name: Upload artifacts
@@ -129,6 +159,10 @@ jobs:
   build-c:
     name: Build C FFI ${{ matrix.target }}
     runs-on: ${{ matrix.os }}
+    env:
+      SCCACHE_GHA_ENABLED: "true"
+      RUSTC_WRAPPER: sccache
+      SCCACHE_IDLE_TIMEOUT: "0"
     permissions:
       contents: read
     strategy:
@@ -137,27 +171,27 @@ jobs:
           ## Linux builds
           - os: ubuntu-latest
             target: x86_64-unknown-linux-gnu
-            zigbuild_target: x86_64-unknown-linux-gnu.2.17
-            artifact_name: target/x86_64-unknown-linux-gnu/release/libfff_c.so
+            zigbuild_target: x86_64-unknown-linux-gnu.2.31
+            artifact_name: target/x86_64-unknown-linux-gnu/ci/libfff_c.so
             npm_package: fff-bin-linux-x64-gnu
             lib_filename: libfff_c.so
             ext: so
           - os: ubuntu-latest
             target: aarch64-unknown-linux-gnu
-            zigbuild_target: aarch64-unknown-linux-gnu.2.17
-            artifact_name: target/aarch64-unknown-linux-gnu/release/libfff_c.so
+            zigbuild_target: aarch64-unknown-linux-gnu.2.31
+            artifact_name: target/aarch64-unknown-linux-gnu/ci/libfff_c.so
             npm_package: fff-bin-linux-arm64-gnu
             lib_filename: libfff_c.so
             ext: so
           - os: ubuntu-latest
             target: x86_64-unknown-linux-musl
-            artifact_name: target/x86_64-unknown-linux-musl/release/libfff_c.so
+            artifact_name: target/x86_64-unknown-linux-musl/ci/libfff_c.so
             npm_package: fff-bin-linux-x64-musl
             lib_filename: libfff_c.so
             ext: so
           - os: ubuntu-latest
             target: aarch64-unknown-linux-musl
-            artifact_name: target/aarch64-unknown-linux-musl/release/libfff_c.so
+            artifact_name: target/aarch64-unknown-linux-musl/ci/libfff_c.so
             npm_package: fff-bin-linux-arm64-musl
             lib_filename: libfff_c.so
             ext: so
@@ -165,20 +199,21 @@ jobs:
           ## Android (Termux)
           - os: ubuntu-latest
             target: aarch64-linux-android
-            artifact_name: target/aarch64-linux-android/release/libfff_c.so
+            artifact_name: target/aarch64-linux-android/ci/libfff_c.so
+            npm_package: fff-bin-android-arm64
             lib_filename: libfff_c.so
             ext: so
 
           ## macOS builds
           - os: macos-latest
             target: x86_64-apple-darwin
-            artifact_name: target/x86_64-apple-darwin/release/libfff_c.dylib
+            artifact_name: target/x86_64-apple-darwin/ci/libfff_c.dylib
             npm_package: fff-bin-darwin-x64
             lib_filename: libfff_c.dylib
             ext: dylib
           - os: macos-latest
             target: aarch64-apple-darwin
-            artifact_name: target/aarch64-apple-darwin/release/libfff_c.dylib
+            artifact_name: target/aarch64-apple-darwin/ci/libfff_c.dylib
             npm_package: fff-bin-darwin-arm64
             lib_filename: libfff_c.dylib
             ext: dylib
@@ -186,13 +221,13 @@ jobs:
           ## Windows builds
           - os: windows-latest
             target: x86_64-pc-windows-msvc
-            artifact_name: target/x86_64-pc-windows-msvc/release/fff_c.dll
+            artifact_name: target/x86_64-pc-windows-msvc/ci/fff_c.dll
             npm_package: fff-bin-win32-x64
             lib_filename: fff_c.dll
             ext: dll
           - os: windows-latest
             target: aarch64-pc-windows-msvc
-            artifact_name: target/aarch64-pc-windows-msvc/release/fff_c.dll
+            artifact_name: target/aarch64-pc-windows-msvc/ci/fff_c.dll
             npm_package: fff-bin-win32-arm64
             lib_filename: fff_c.dll
             ext: dll
@@ -205,10 +240,20 @@ jobs:
       - name: Install Rust
         run: rustup target add ${{ matrix.target }}
 
+      - name: Rust cache
+        uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4
+        with:
+          key: c-${{ matrix.target }}
+
+      - name: Setup sccache
+        uses: mozilla-actions/sccache-action@v0.0.11
+        with:
+          version: v0.17.0
+
       - name: Install Zig
         uses: mlugg/setup-zig@v2
         with:
-          version: 0.15.2
+          version: 0.16.0
 
       - name: Install cargo-zigbuild
         if: contains(matrix.os, 'ubuntu')
@@ -217,7 +262,7 @@ jobs:
       - name: Build for Linux
         if: contains(matrix.os, 'ubuntu') && !contains(matrix.target, 'android')
         run: |
-          cargo zigbuild --release --target ${{ matrix.zigbuild_target || matrix.target }} -p fff-c --features zlob
+          cargo zigbuild --profile ci --target ${{ matrix.zigbuild_target || matrix.target }} -p fff-c --no-default-features --features zlob
           mv "${{ matrix.artifact_name }}" "c-lib-${{ matrix.target }}.${{ matrix.ext }}"
 
       - name: Build for Android (Termux)
@@ -230,13 +275,13 @@ jobs:
           export AR_aarch64_linux_android="$NDK_BIN/llvm-ar"
           export CARGO_TARGET_AARCH64_LINUX_ANDROID_LINKER="$NDK_BIN/aarch64-linux-android24-clang"
 
-          cargo build --release --target ${{ matrix.target }} -p fff-c --features zlob
+          cargo build --profile ci --target ${{ matrix.target }} -p fff-c --no-default-features --features zlob
           mv "${{ matrix.artifact_name }}" "c-lib-${{ matrix.target }}.${{ matrix.ext }}"
 
       - name: Build for macOS
         if: contains(matrix.os, 'macos')
         run: |
-          MACOSX_DEPLOYMENT_TARGET="13" cargo build --release --target ${{ matrix.target }} -p fff-c --features zlob
+          MACOSX_DEPLOYMENT_TARGET="13" cargo build --profile ci --target ${{ matrix.target }} -p fff-c --no-default-features --features zlob
           mv "${{ matrix.artifact_name }}" "c-lib-${{ matrix.target }}.${{ matrix.ext }}"
 
       - name: Ad-hoc sign macOS binary
@@ -247,11 +292,10 @@ jobs:
         if: contains(matrix.os, 'windows')
         shell: bash
         run: |
-          cargo build --release --target ${{ matrix.target }} -p fff-c --features zlob
+          cargo build --profile ci --target ${{ matrix.target }} -p fff-c --no-default-features --features zlob
           mv "${{ matrix.artifact_name }}" "c-lib-${{ matrix.target }}.${{ matrix.ext }}"
 
       - name: Prepare npm package
-        if: "!contains(matrix.target, 'android')"
         shell: bash
         run: |
           # Copy the built binary into the platform npm package directory
@@ -264,7 +308,6 @@ jobs:
           path: c-lib-${{ matrix.target }}.*
 
       - name: Upload npm package artifact
-        if: "!contains(matrix.target, 'android')"
         uses: actions/upload-artifact@v4
         with:
           name: npm-${{ matrix.npm_package }}
@@ -273,6 +316,10 @@ jobs:
   build-mcp:
     name: Build MCP ${{ matrix.target }}
     runs-on: ${{ matrix.os }}
+    env:
+      SCCACHE_GHA_ENABLED: "true"
+      RUSTC_WRAPPER: sccache
+      SCCACHE_IDLE_TIMEOUT: "0"
     permissions:
       contents: read
     strategy:
@@ -281,34 +328,34 @@ jobs:
           ## Linux builds (using cargo-zigbuild)
           - os: ubuntu-latest
             target: x86_64-unknown-linux-gnu
-            zigbuild_target: x86_64-unknown-linux-gnu.2.17
-            artifact_name: target/x86_64-unknown-linux-gnu/release/fff-mcp
+            zigbuild_target: x86_64-unknown-linux-gnu.2.31
+            artifact_name: target/x86_64-unknown-linux-gnu/ci/fff-mcp
           - os: ubuntu-latest
             target: aarch64-unknown-linux-gnu
-            zigbuild_target: aarch64-unknown-linux-gnu.2.17
-            artifact_name: target/aarch64-unknown-linux-gnu/release/fff-mcp
+            zigbuild_target: aarch64-unknown-linux-gnu.2.31
+            artifact_name: target/aarch64-unknown-linux-gnu/ci/fff-mcp
           - os: ubuntu-latest
             target: x86_64-unknown-linux-musl
-            artifact_name: target/x86_64-unknown-linux-musl/release/fff-mcp
+            artifact_name: target/x86_64-unknown-linux-musl/ci/fff-mcp
           - os: ubuntu-latest
             target: aarch64-unknown-linux-musl
-            artifact_name: target/aarch64-unknown-linux-musl/release/fff-mcp
+            artifact_name: target/aarch64-unknown-linux-musl/ci/fff-mcp
 
           ## macOS builds
           - os: macos-latest
             target: x86_64-apple-darwin
-            artifact_name: target/x86_64-apple-darwin/release/fff-mcp
+            artifact_name: target/x86_64-apple-darwin/ci/fff-mcp
           - os: macos-latest
             target: aarch64-apple-darwin
-            artifact_name: target/aarch64-apple-darwin/release/fff-mcp
+            artifact_name: target/aarch64-apple-darwin/ci/fff-mcp
 
           ## Windows builds
           - os: windows-latest
             target: x86_64-pc-windows-msvc
-            artifact_name: target/x86_64-pc-windows-msvc/release/fff-mcp.exe
+            artifact_name: target/x86_64-pc-windows-msvc/ci/fff-mcp.exe
           - os: windows-latest
             target: aarch64-pc-windows-msvc
-            artifact_name: target/aarch64-pc-windows-msvc/release/fff-mcp.exe
+            artifact_name: target/aarch64-pc-windows-msvc/ci/fff-mcp.exe
 
     steps:
       - uses: actions/checkout@v5
@@ -318,10 +365,20 @@ jobs:
       - name: Install Rust
         run: rustup target add ${{ matrix.target }}
 
+      - name: Rust cache
+        uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4
+        with:
+          key: mcp-${{ matrix.target }}
+
+      - name: Setup sccache
+        uses: mozilla-actions/sccache-action@v0.0.11
+        with:
+          version: v0.17.0
+
       - name: Install Zig
         uses: mlugg/setup-zig@v2
         with:
-          version: 0.15.2
+          version: 0.16.0
 
       - name: Install cargo-zigbuild
         if: contains(matrix.os, 'ubuntu')
@@ -330,13 +387,13 @@ jobs:
       - name: Build for Linux
         if: contains(matrix.os, 'ubuntu')
         run: |
-          cargo zigbuild --release --target ${{ matrix.zigbuild_target || matrix.target }} -p fff-mcp --features zlob
+          cargo zigbuild --profile ci --target ${{ matrix.zigbuild_target || matrix.target }} -p fff-mcp --no-default-features --features zlob
           cp "${{ matrix.artifact_name }}" "fff-mcp-${{ matrix.target }}"
 
       - name: Build for macOS
         if: contains(matrix.os, 'macos')
         run: |
-          MACOSX_DEPLOYMENT_TARGET="13" cargo build --release --target ${{ matrix.target }} -p fff-mcp --features zlob
+          MACOSX_DEPLOYMENT_TARGET="13" cargo build --profile ci --target ${{ matrix.target }} -p fff-mcp --no-default-features --features zlob
           cp "${{ matrix.artifact_name }}" "fff-mcp-${{ matrix.target }}"
 
       - name: Ad-hoc sign macOS binary
@@ -347,7 +404,7 @@ jobs:
         if: contains(matrix.os, 'windows')
         shell: bash
         run: |
-          cargo build --release --target ${{ matrix.target }} -p fff-mcp --features zlob
+          cargo build --profile ci --target ${{ matrix.target }} -p fff-mcp --no-default-features --features zlob
           cp "${{ matrix.artifact_name }}" "fff-mcp-${{ matrix.target }}.exe"
 
       - name: Upload artifact
@@ -356,16 +413,110 @@ jobs:
           name: mcp-${{ matrix.target }}
           path: fff-mcp-${{ matrix.target }}*
 
+  build-python:
+    name: Build Python wheels ${{ matrix.target }} (${{ matrix.os }})
+    # Wheels are release artifacts; PR validation uses the develop build in
+    # python.yml, so skip the cross-compile matrix on pull requests.
+    if: github.event_name != 'pull_request'
+    runs-on: ${{ matrix.os }}
+    permissions:
+      contents: read
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: ubuntu-latest
+            target: x86_64
+            container: "off"
+          - os: ubuntu-latest
+            target: aarch64
+            container: "off"
+          - os: macos-latest
+            target: x86_64
+          - os: macos-latest
+            target: aarch64
+          - os: windows-latest
+            target: x86_64
+
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          persist-credentials: false
+
+      - name: Install Rust
+        uses: dtolnay/rust-toolchain@stable
+
+      - name: Install Zig
+        uses: mlugg/setup-zig@v2
+        with:
+          version: 0.16.0
+
+      - name: Install cargo-zigbuild
+        if: contains(matrix.os, 'ubuntu')
+        run: cargo install cargo-zigbuild
+
+      - name: Install aarch64 cross compiler
+        if: matrix.target == 'aarch64' && contains(matrix.os, 'ubuntu')
+        run: |
+          sudo apt-get update -qq
+          sudo apt-get install -y gcc-aarch64-linux-gnu g++-aarch64-linux-gnu
+
+      - name: Build wheels
+        uses: PyO3/maturin-action@v1
+        env:
+          CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER: aarch64-linux-gnu-gcc
+          CC_aarch64_unknown_linux_gnu: aarch64-linux-gnu-gcc
+          CXX_aarch64_unknown_linux_gnu: aarch64-linux-gnu-g++
+          AR_aarch64_unknown_linux_gnu: aarch64-linux-gnu-ar
+        with:
+          target: ${{ matrix.target }}
+          args: --release --out dist --no-default-features --features zlob
+          sccache: "true"
+          working-directory: packages/fff-python
+          container: ${{ matrix.container || '' }}
+
+      - name: Upload wheels
+        uses: actions/upload-artifact@v4
+        with:
+          name: python-wheels-${{ matrix.os }}-${{ matrix.target }}
+          path: packages/fff-python/dist/
+
+  build-python-sdist:
+    name: Build Python sdist
+    if: github.event_name != 'pull_request'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          persist-credentials: false
+
+      - name: Build sdist
+        uses: PyO3/maturin-action@v1
+        with:
+          command: sdist
+          args: --out dist
+          working-directory: packages/fff-python
+
+      - name: Upload sdist
+        uses: actions/upload-artifact@v4
+        with:
+          name: python-sdist
+          path: packages/fff-python/dist/
+
   release:
     name: Release
-    needs: [build-nvim, build-c, build-mcp]
+    needs: [build-nvim, build-c, build-mcp, build-python, build-python-sdist]
     runs-on: ubuntu-latest
-    # do not create releases on the forks (no permissions)
-    if: github.event_name != 'pull_request' || github.event.pull_request.user.login == 'dmtrKovalenko'
+    if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v'))
     permissions:
       contents: write
     steps:
+      # we have to make sure that pushing a commit on this workflow triggers the CI for nightly neovim
       - uses: actions/checkout@v5
+        with:
+          token: ${{ secrets.GUSTAV_PAT || github.token }}
 
       - name: Install Lua
         uses: leafo/gh-actions-lua@v12
@@ -418,6 +569,20 @@ jobs:
             rmdir "$dir" 2>/dev/null || true
           done
 
+      - name: Move Python wheels to release directory
+        working-directory: ./binaries
+        run: |
+          mkdir -p python
+          for dir in python-wheels-*/ python-sdist/; do
+            [ -d "$dir" ] || continue
+            for file in "$dir"*; do
+              if [ -f "$file" ]; then
+                mv "$file" "python/$(basename "$file")"
+              fi
+            done
+            rmdir "$dir" 2>/dev/null || true
+          done
+
       - name: Remove npm package artifacts from release binaries
         working-directory: ./binaries
         run: |
@@ -427,7 +592,7 @@ jobs:
         working-directory: ./binaries
         run: |
           ls -la
-          for file in *; do
+          for file in * python/*; do
             if [ -f "$file" ] && [[ ! "$file" == *.sha256 ]]; then
               sha256sum "$file" > "${file}.sha256"
             fi
@@ -437,19 +602,32 @@ jobs:
         id: version
         run: lua scripts/determine-version.lua
 
+      # Nightlies publish to a permanent per-sha tag (release_tag == version) so
+      # pinned/stale installs always fetch the binary built for their own commit.
+      # The rolling `nightly` tag is also moved to HEAD for "give me latest" tooling.
+      - name: Move rolling nightly tag to current commit
+        if: steps.version.outputs.is_release != 'true'
+        run: |
+          git tag -f nightly "${{ github.sha }}"
+          git push -f origin refs/tags/nightly
+
       - name: Upload Release Assets
         uses: softprops/action-gh-release@v2
         with:
           name: "${{ steps.version.outputs.version }}"
-          tag_name: "${{ steps.version.outputs.is_release == 'true' && format('v{0}', steps.version.outputs.version) || steps.version.outputs.version }}"
+          tag_name: "${{ steps.version.outputs.release_tag }}"
           token: ${{ github.token }}
-          files: ./binaries/*
+          files: |
+            ./binaries/*
+            ./binaries/python/*
           draft: false
           prerelease: ${{ steps.version.outputs.is_release != 'true' }}
           generate_release_notes: ${{ steps.version.outputs.is_release == 'true' }}
           body: |
             ${{ steps.version.outputs.is_release == 'true' && format('Release {0}', steps.version.outputs.version) || format('Nightly release from commit: {0}', github.sha) }}
 
+            npm packages, rust crates and python wheels are available under this version ${{ steps.version.outputs.version }}
+
             ## Neovim Plugin
             - `{target}.so` / `.dylib` / `.dll` - Lua module for Neovim
 
@@ -459,21 +637,71 @@ jobs:
             ## MCP Server
             - `fff-mcp-{target}` - MCP server binary
 
-            Install with:
+            ## Python Package
+            - `python/*.whl` / `python/*.tar.gz` - Python wheels and sdist
+            - Install from PyPI: `pip install fff-search` (when published)
+
+            Update mcp via:
             ```sh
-            curl -fsSL https://raw.githubusercontent.com/dmtrKovalenko/fff.nvim/main/install-mcp.sh | bash
+            curl -fsSL https://raw.githubusercontent.com/dmtrKovalenko/fff/main/install-mcp.sh | bash
             ```
 
+      - name: Bump Homebrew formula (uses local checksums)
+        if: steps.version.outputs.is_release == 'true' && github.repository_owner == 'dmtrKovalenko'
+        run: make bump-homebrew-formula VERSION="${{ steps.version.outputs.version }}" BINARIES_DIR=./binaries
+
+      - name: Pin SHAs in install-mcp.sh (uses local checksums)
+        if: steps.version.outputs.is_release == 'true' && github.repository_owner == 'dmtrKovalenko'
+        run: make bump-install-mcp-sh VERSION="${{ steps.version.outputs.version }}" BINARIES_DIR=./binaries
+
+      - name: Commit formula + installer bump to main
+        if: steps.version.outputs.is_release == 'true' && github.repository_owner == 'dmtrKovalenko'
+        uses: stefanzweifel/git-auto-commit-action@v5
+        with:
+          # Workflow runs on detached HEAD at the v* tag — explicit target needed.
+          branch: main
+          commit_message: "chore: bump fff-mcp release artifacts to v${{ steps.version.outputs.version }}"
+          file_pattern: "Formula/fff-mcp.rb install-mcp.sh"
+          commit_user_name: github-actions[bot]
+          commit_user_email: 41898282+github-actions[bot]@users.noreply.github.com
+
+  pypi-publish:
+    name: Publish Python wheels to PyPI
+    needs: [build-python, build-python-sdist]
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v'))
+    environment:
+      name: pypi
+      url: https://pypi.org/p/fff-search
+    permissions:
+      contents: read
+      id-token: write
+    steps:
+      - name: Download Python wheels and sdist
+        uses: actions/download-artifact@v4
+        with:
+          pattern: python-*
+          path: dist
+          merge-multiple: true
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          packages-dir: dist/
+          skip-existing: true
+
   crates-publish:
     name: Publish Rust crates
     needs: [build-nvim, build-c, build-mcp]
     runs-on: ubuntu-latest
-    if: >-
-      (github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/fix/download-version' || startsWith(github.ref, 'refs/tags/v')))
-      || (github.event_name == 'pull_request' && github.head_ref == 'main')
-
+    if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v'))
+    permissions:
+      contents: read
+      id-token: write
     steps:
       - uses: actions/checkout@v5
+      - uses: rust-lang/crates-io-auth-action@v1
+        id: auth
 
       - name: Install Lua
         uses: leafo/gh-actions-lua@v12
@@ -482,7 +710,7 @@ jobs:
         uses: dtolnay/rust-toolchain@stable
 
       - name: Install cargo-edit
-        run: cargo install cargo-edit
+        run: cargo install cargo-edit --force --locked
 
       - name: Determine version
         id: version
@@ -490,18 +718,17 @@ jobs:
 
       - name: Publish crates
         env:
-          CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
+          CARGO_REGISTRY_TOKEN: ${{ steps.auth.outputs.token }}
         run: make publish-crates V="${{ steps.version.outputs.version }}"
 
   npm-publish:
     name: Publish npm packages
     needs: [build-c]
     runs-on: ubuntu-latest
-    if: >-
-      (github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/fix/download-version' || startsWith(github.ref, 'refs/tags/v')))
-      || (github.event_name == 'pull_request' && github.head_ref == 'main')
+    if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/fix/npm-pulish' || startsWith(github.ref, 'refs/tags/v'))
     permissions:
       contents: read
+      id-token: write
     steps:
       - uses: actions/checkout@v5
 
@@ -514,6 +741,8 @@ jobs:
           node-version: "25"
           registry-url: "https://registry.npmjs.org"
 
+      - uses: oven-sh/setup-bun@v2
+
       - name: Determine version
         id: version
         run: lua scripts/determine-version.lua
@@ -525,8 +754,6 @@ jobs:
           path: ./npm-packages
 
       - name: Publish platform packages
-        env:
-          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
         run: |
           VERSION="${{ steps.version.outputs.version }}"
           TAG="${{ steps.version.outputs.npm_tag }}"
@@ -539,14 +766,22 @@ jobs:
               make set-npm-version PKG="$pkg_dir" VERSION="$VERSION"
 
               cd "$pkg_dir"
-              npm publish --tag "$TAG" --access public || echo "Failed to publish ${pkg_name} (may already exist)"
+              npm publish --tag "$TAG" --access public --provenance
               cd -
             fi
           done
 
+      # Every install must happen before the first set-npm-version: the bump
+      # injects platform optionalDependencies that only resolve once published,
+      # and npm ci validates the whole workspace against the lockfile.
+      - name: Install and build JS packages
+        working-directory: packages
+        run: |
+          npm ci
+          (cd fff-bun && npm run build)
+          (cd fff-node && npm run build)
+
       - name: Publish bun package
-        env:
-          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
         run: |
           VERSION="${{ steps.version.outputs.version }}"
           TAG="${{ steps.version.outputs.npm_tag }}"
@@ -555,11 +790,9 @@ jobs:
           make set-npm-version PKG=packages/fff-bun VERSION="$VERSION"
 
           cd packages/fff-bun
-          npm publish --tag "$TAG" --access public || echo "Failed to publish @ff-labs/fff-bun (may already exist)"
+          npm publish --tag "$TAG" --access public --provenance
 
       - name: Publish Node.js package
-        env:
-          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
         run: |
           VERSION="${{ steps.version.outputs.version }}"
           TAG="${{ steps.version.outputs.npm_tag }}"
@@ -568,6 +801,15 @@ jobs:
           make set-npm-version PKG=packages/fff-node VERSION="$VERSION"
 
           cd packages/fff-node
-          npm install
-          npm run build
-          npm publish --tag "$TAG" --access public || echo "Failed to publish @ff-labs/fff-node (may already exist)"
+          npm publish --tag "$TAG" --access public --provenance
+
+      - name: Publish pi-fff package
+        run: |
+          VERSION="${{ steps.version.outputs.version }}"
+          TAG="${{ steps.version.outputs.npm_tag }}"
+
+          echo "Publishing @ff-labs/pi-fff@${VERSION} with tag ${TAG}..."
+          make set-npm-version PKG=packages/pi-fff VERSION="$VERSION"
+
+          cd packages/pi-fff
+          npm publish --tag "$TAG" --access public --provenance

.github/workflows/rust.yml

@@ -3,22 +3,41 @@ name: Rust CI
 on:
   push:
     branches: [main]
+    paths-ignore:
+      - '**.md'
+      - 'doc/**'
   pull_request:
     branches: [main]
+    paths-ignore:
+      - '**.md'
+      - 'doc/**'
 
 env:
   CARGO_TERM_COLOR: always
   # Ensure consistent macOS deployment target across all compiled objects
   # (Rust, cc-compiled C code, and Zig-compiled zlob) to avoid linker warnings
   MACOSX_DEPLOYMENT_TARGET: "13"
+  # RUSTC_WRAPPER is set per job, since cargo fmt runs without sccache.
+  SCCACHE_GHA_ENABLED: "true"
+  # fff-search alone exceeds the 600s default on windows, and the server sees
+  # no new requests while it compiles, so it would idle out mid-unit.
+  SCCACHE_IDLE_TIMEOUT: "0"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' && !startsWith(github.ref, 'refs/tags/') }}
 
 jobs:
   test:
     name: Test
     runs-on: ${{ matrix.os }}
     strategy:
+      fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-latest]
+        os: [ubuntu-latest, macos-latest, windows-latest]
+    # Guard against deadlocks in the shared-picker / watcher teardown
+    # path: a stuck test would otherwise consume a full 6h CI slot.
+    timeout-minutes: 15
     steps:
       - uses: actions/checkout@v5
       
@@ -26,7 +45,7 @@ jobs:
       - name: Install Zig
         uses: goto-bus-stop/setup-zig@v2
         with:
-          version: 0.15.2
+          version: 0.16.0
 
       - name: Install Rust
         uses: actions-rust-lang/setup-rust-toolchain@v1.15.4
@@ -36,8 +55,107 @@ jobs:
           cache-key: "v1-rust"
           components: rustfmt, clippy
 
+      - name: Setup sccache
+        uses: mozilla-actions/sccache-action@v0.0.11
+        with:
+          version: v0.17.0
+
       - name: Run tests
-        run: cargo test --features zlob --workspace --exclude fff-nvim
+        env:
+          RUSTC_WRAPPER: sccache
+        # fff-python requires full python o3 machinery which is very slow
+        run: cargo test --no-default-features --features zlob --workspace --exclude fff-nvim --exclude fff-python
+
+  stress-test:
+    name: Fuzz Tests
+    runs-on: ${{ matrix.os }}
+    strategy:
+      # Keep going after one OS fails so we can see whether a bug
+      # reproduces everywhere or is platform-specific.
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+    # Long-running; don't let a stuck watcher thread burn a full CI
+    # timeout. Two scenarios should finish well under this limit.
+    timeout-minutes: 20
+    env:
+      FFF_STRESS_CASES: "5"
+      FFF_STRESS_MIN_OPS: "30"
+      FFF_STRESS_MAX_OPS: "60"
+      RUSTC_WRAPPER: sccache
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Install Zig
+        uses: goto-bus-stop/setup-zig@v2
+        with:
+          version: 0.16.0
+
+      - name: Install Rust
+        uses: actions-rust-lang/setup-rust-toolchain@v1.15.4
+        with:
+          cache: true
+          cache-on-failure: true
+          cache-key: "v1-rust-stress-${{ matrix.os }}"
+          components: rustfmt, clippy
+
+      - name: Setup sccache
+        uses: mozilla-actions/sccache-action@v0.0.11
+        with:
+          version: v0.17.0
+
+      - name: Stress test seeded
+        shell: bash
+        run: make test-stress-seeded
+
+      - name: Stress test random
+        shell: bash
+        run: make test-stress-random
+
+      - name: Stress test regressions
+        shell: bash
+        run: make test-stress-regressions
+
+      - name: Upload proptest regressions on failure
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: proptest-regressions-${{ matrix.os }}
+          path: crates/fff-core/tests/fuzz_git_watcher_stress.proptest-regressions
+          if-no-files-found: ignore
+
+  build-i686:
+    name: Build i686-unknown-linux-gnu
+    runs-on: ubuntu-latest
+    # Verifies that fff-search compiles on 32-bit x86, where std::arch::x86_64
+    # is unavailable. SIMD paths are disabled on this target; only the scalar
+    # fallback should build. See issue #656.
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Install cross toolchain
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y gcc-multilib g++-multilib
+
+      - name: Install Rust (i686 target)
+        uses: actions-rust-lang/setup-rust-toolchain@v1.15.4
+        with:
+          target: i686-unknown-linux-gnu
+          cache: true
+          cache-on-failure: true
+          cache-key: "v1-rust-i686"
+
+      - name: Setup sccache
+        uses: mozilla-actions/sccache-action@v0.0.11
+        with:
+          version: v0.17.0
+
+      - name: Build fff-search for i686
+        env:
+          RUSTC_WRAPPER: sccache
+        run: cargo build -p fff-search --target i686-unknown-linux-gnu
 
   fmt:
     name: cargo fmt
@@ -62,13 +180,20 @@ jobs:
       - name: Install Zig
         uses: goto-bus-stop/setup-zig@v2
         with:
-          version: 0.15.2
+          version: 0.16.0
 
       - name: Install Rust
         uses: dtolnay/rust-toolchain@master
         with:
           toolchain: stable
           components: clippy
-          
+
+      - name: Setup sccache
+        uses: mozilla-actions/sccache-action@v0.0.11
+        with:
+          version: v0.17.0
+
       - name: Run clippy
-        run: cargo clippy -- -D warnings
+        env:
+          RUSTC_WRAPPER: sccache
+        run: cargo clippy --no-default-features --features zlob -- -D warnings

.github/workflows/spelling.yaml

@@ -12,6 +12,10 @@ on:
 env:
   CLICOLOR: 1
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' && !startsWith(github.ref, 'refs/tags/') }}
+
 jobs:
   spelling:
     name: Spell Check with Typos

.github/workflows/stylua.yaml

@@ -20,6 +20,10 @@ on:
 env:
   CLICOLOR: 1
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' && !startsWith(github.ref, 'refs/tags/') }}
+
 jobs:
   stylua:
     name: Check lua files using Stylua

.gitignore

@@ -1,4 +1,5 @@
 doc/tags
+big-repo
 target/
 .archive.lua
 _*.lua
@@ -10,10 +11,12 @@ result
 .repro/
 .wrangler/
 *.so
-big-repo/
+*.dylib
 # all the perf like utility files
 *.data
 node_modules/
+crates/fff-notify-debouncer-full/
+packages/fff-bun/glob-bench-bin
 
 dist/
 scripts/benchmark-results/
@@ -22,3 +25,41 @@ scripts/benchmark-results/
 *.dylib
 *.so
 *.dll
+*.pdb
+
+# Instruments traces
+*.trace/
+
+# Test logs
+fff-test.log
+
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.pyd
+*.egg-info/
+*.egg
+.eggs/
+build/
+*.whl
+# Virtual environments
+.venv/
+venv/
+env/
+ENV/
+# uv
+# Testing / linting
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+.coverage
+htmlcov/
+.tox/
+.nox/
+# IDEs
+.idea/
+.vscode/
+*.swp
+*.swo
+*~

.luacheckrc

@@ -1,4 +1,4 @@
--- luacheck configuration for fff.nvim
+-- luacheck configuration for fff
 -- https://luacheck.readthedocs.io/en/stable/config.html
 
 -- Neovim globals

.luarc.json

@@ -5,14 +5,14 @@
   },
   "workspace": {
     "library": [
-      "/opt/homebrew/share/nvim/runtime/lua/vim/_meta",
-      "/opt/homebrew/share/nvim/runtime/lua/vim/shared.lua",
+      "$VIMRUNTIME/lua",
       "${3rd}/luv/library",
       "${3rd}/busted/library"
     ],
     "checkThirdParty": false
   },
   "diagnostics": {
+    "globals": ["vim"],
     "severity": {
       "undefined-global": "Error",
       "undefined-field": "Warning",

.mcp.json

@@ -2,7 +2,7 @@
   "mcpServers": {
     "fff": {
       "type": "stdio",
-      "command": "/Users/neogoose/dev/fff.nvim/target/release/fff-mcp",
+      "command": "./target/release/fff-mcp",
       "args": []
     }
   }

AGENTS.md

@@ -0,0 +1,118 @@
+# To Clankers
+
+This repository contains **FFF.nvim (Fast File Finder)**, a high-performance file picker for Neovim inspired by blink.cmp's fuzzy matching technology. It's NOT a completion plugin, but rather a standalone file finder with advanced fuzzy search and frecency scoring. The project aims to be the drop-in replacement for telescope, fzf-lua, snacks.picker and similar plugins, focusing on speed, accuracy search and usability features.
+
+## Development Commands
+
+Always prefer Makefile commands listed to the cargo/bun/node if possible.
+
+### Building
+
+- `make build` - build everything
+
+### Testing and Development Tools
+
+This project does not have a traditional test suite. Testing is done through:
+
+- Create e2e local test file for Neovim: Load any Lua test file with `nvim -l <test_file>`
+- Write inline rust unit tests for any functionality that is standalone and scoped within a single function
+
+### Code Quality
+
+- `make lint` - Rust linting and code analysis
+- `make format` - Format all code 
+- `make test` - Run unit tests (limited coverage, primarily integration testing)
+
+When doing code make sure to REDUCE SIZE OF COMMENTS. This is very important. Every comment should be concise 1-2 liner maximum 4 lines if describes really extensive and unnatural concept.
+
+### Important coding rules
+
+- Do not add doc comments to the private structs and functions.
+- Do not make public structs if something can be private
+
+
+## Style guide
+
+- NO MODULES COMMENTS
+- NO TOP FILE COMMENTS
+- NO COMMENT LONGER THAN 2 LINES UNLESS ASKED EXPLICITLY
+- UTILITY FUNCTIONS GO INTO THE END OF FILE
+
+## Architecture
+
+Everything that is performance critical happens in rust world, everything that is neovim specific happens in the lua code.
+
+There are 3 main components:
+
+- Rust binary with the global file picker state containing index of all files
+- Background thread with the file system watcher that updates the index in real time
+- Lua UI layer that renders the picker, handles user input, and calls the rust functions via FFI
+
+There are 2 databases:
+
+- Frecency database (LMDB) that tracks file access patterns for scoring
+- Query history database used to track the user's previous search queries
+
+### Key Files
+
+- `lua/fff.lua` - Entry point, delegates to main.lua
+- `lua/fff/main.lua` - Public API (find_files, search, change_directory)
+- `lua/fff/core.lua` - Initialization, autocmds, global state management
+- `lua/fff/picker_ui.lua` - UI rendering, layout calculation, keymaps
+- `lua/fff/file_picker/preview.lua` - File preview with syntax highlighting
+- `lua/fff/file_picker/image.lua` - Image preview (snacks.nvim integration)
+- `lua/fff/conf.lua` - Default config
+- `lua/fff/rust/init.lua` - Loads compiled Rust shared library
+
+**Rust Side:**
+
+- `lua/fff/rust/lib.rs` - FFI bindings, global state (FILE_PICKER, FRECENCY)
+- `lua/fff/rust/file_picker.rs` - Core FilePicker struct, indexing, background watcher
+- `lua/fff/rust/frecency.rs` - Frecency database (LMDB) and scoring
+- `lua/fff/rust/query_tracker.rs` - Search query history tracking
+- `lua/fff/rust/score.rs` - Fuzzy match scoring with frizbee integration
+- `lua/fff/rust/git.rs` - Git status caching and repository detection
+- `lua/fff/rust/background_watcher.rs` - File system watcher thread
+
+### Scoring Algorithm
+
+Located at the score.rs file
+
+### Build System
+
+- `Cargo.toml` - Rust dependencies and build configuration (package name: `fff_nvim`)
+- `rust-toolchain.toml` - Specifies Rust nightly toolchain with required components
+- `Cross.toml` - Cross-compilation settings using Zig for Linux targets
+- **CI/CD Workflows**:
+  - `.github/workflows/rust.yml` - Rust testing, formatting, and clippy checks
+  - `.github/workflows/release.yaml` - Automated multi-platform builds
+  - `.github/workflows/stylua.yaml` - Lua code formatting validation
+  - `.github/workflows/nix.yml` - Nix build validation
+- **Cross-compilation Support**: Uses `cross` tool with Zig backend for efficient cross-compilation
+
+## Development Notes
+
+### Working with Rust Code
+
+- Prefer struct methods over functions
+- If there is more than 2 impls in the file - create new file
+- Smaller concise comments over giant comment blocks
+- Do not add doc comments to the private functions/structs
+- Be very careful around locking and better double check with the human if something is going to require potentially long lock on a mutex/rwlock
+
+### Working with lua code
+
+- Document the types of public functions in every module
+- Use `vim.validate()` for validating user inputs in public functions
+- Try to reuse as much of existing functions as possible
+- When working on new features for the UI **IT IS EXTREMELY IMPORTANT** to keep the core functionality of navigating between files, selecting, and seeing the preview working as is. NEVER break anything from the core UI functionality, only add new features on top of the current UI.
+- When making a large chunk of code make lua test that opens neovim at `~/dev/lightsource` and opens the picker to test the ui functionality across the actual code.
+- When adding a new highlights or any new shortcuts and configurable UI options add them to the neovim config. AND IMPORTANT: update the README.md with the new configuration options.
+
+### UI rendering
+
+When working on the UI changeds IT IS EXTREMELY important for you to test it for both prompt_position="bottom" and prompt_position="top" as the rendering logic is different for both of them in both rust and lua world. When the prompt is positioed in the bottom everything should work the same way as the top but would be reversed in order. (though navigation is same for both)
+
+## Top level API that can not introduce breaking changes under any circumstance
+
+Top level rust, lua, C, and bun APIs can not be changed under any circumstance

CLAUDE.md

@@ -0,0 +1 @@
+AGENTS.md

Cargo.lock

@@ -91,9 +91,9 @@ dependencies = [
 
 [[package]]
 name = "anyhow"
-version = "1.0.102"
+version = "1.0.103"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c"
+checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3"
 
 [[package]]
 name = "arrayref"
@@ -139,17 +139,11 @@ dependencies = [
  "serde",
 ]
 
-[[package]]
-name = "bindet"
-version = "0.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5afee99ef5f7527f3944f2bf4f5d443749fa47d43eb1d4f83a36e839be7900a3"
-
 [[package]]
 name = "bindgen"
-version = "0.70.1"
+version = "0.72.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f49d8fed880d473ea71efb9bf597651e77201bdd4893efe54c9e5d65ae04ce6f"
+checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895"
 dependencies = [
  "bitflags 2.11.0",
  "cexpr",
@@ -160,7 +154,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "regex",
- "rustc-hash 1.1.0",
+ "rustc-hash",
  "shlex",
  "syn",
 ]
@@ -446,9 +440,9 @@ dependencies = [
 
 [[package]]
 name = "crossbeam-epoch"
-version = "0.9.18"
+version = "0.9.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
+checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f"
 dependencies = [
  "crossbeam-utils",
 ]
@@ -474,6 +468,16 @@ version = "0.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5"
 
+[[package]]
+name = "ctor"
+version = "0.2.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "32a2785755761f3ddc1492979ce1e48d2c00d09311c39e4466429188f3dd6501"
+dependencies = [
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "ctrlc"
 version = "3.5.2"
@@ -485,6 +489,12 @@ dependencies = [
  "windows-sys 0.61.2",
 ]
 
+[[package]]
+name = "cty"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b365fabc795046672053e29c954733ec3b05e4be654ab130fe8f1f94d7051f35"
+
 [[package]]
 name = "darling"
 version = "0.23.0"
@@ -623,29 +633,25 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
 
 [[package]]
 name = "fff-c"
-version = "0.4.2"
+version = "0.10.5"
 dependencies = [
  "fff-query-parser",
  "fff-search",
  "git2",
- "mimalloc",
- "serde",
  "serde_json",
- "tracing",
 ]
 
 [[package]]
 name = "fff-grep"
-version = "0.4.2"
+version = "0.10.5"
 dependencies = [
  "bstr",
- "grep-matcher",
  "memchr",
 ]
 
 [[package]]
 name = "fff-mcp"
-version = "0.4.2"
+version = "0.10.5"
 dependencies = [
  "clap",
  "fff-query-parser",
@@ -656,46 +662,60 @@ dependencies = [
  "schemars",
  "serde",
  "serde_json",
+ "tempfile",
  "tokio",
  "tracing",
+ "windows-sys 0.60.2",
+]
+
+[[package]]
+name = "fff-notify-debouncer-full"
+version = "0.9.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "29a4ebea7b8a2840cd59358bbf396f6f04313ce8eae84ac79703ce80298b8731"
+dependencies = [
+ "file-id",
+ "log",
+ "notify",
+ "notify-types",
+ "rustc-hash",
+ "walkdir",
 ]
 
 [[package]]
 name = "fff-nvim"
-version = "0.4.2"
+version = "0.10.5"
 dependencies = [
  "ahash",
- "blake3",
  "chrono",
  "criterion",
  "ctrlc",
- "dirs",
  "fff-query-parser",
  "fff-search",
  "git2",
- "glidesort",
- "heed",
  "ignore",
  "mimalloc",
  "mlua",
- "neo_frizbee",
- "notify",
- "notify-debouncer-full 0.6.0",
  "once_cell",
- "pathdiff",
- "rand",
- "rayon",
- "serde",
- "smallvec",
- "smartstring",
- "tempfile",
- "thiserror 2.0.18",
+ "rand 0.8.5",
  "tracing",
+ "tracing-subscriber",
+ "zlob",
+]
+
+[[package]]
+name = "fff-python"
+version = "0.10.5"
+dependencies = [
+ "fff-query-parser",
+ "fff-search",
+ "git2",
+ "pyo3",
 ]
 
 [[package]]
 name = "fff-query-parser"
-version = "0.4.2"
+version = "0.10.5"
 dependencies = [
  "criterion",
  "zlob",
@@ -703,42 +723,42 @@ dependencies = [
 
 [[package]]
 name = "fff-search"
-version = "0.4.2"
+version = "0.10.5"
 dependencies = [
  "ahash",
  "aho-corasick",
- "bindet",
  "blake3",
- "chrono",
  "criterion",
+ "ctor",
  "dirs",
  "dunce",
  "fff-grep",
+ "fff-notify-debouncer-full",
  "fff-query-parser",
  "git2",
  "glidesort",
  "globset",
- "grep-matcher",
  "heed",
  "ignore",
+ "libc",
+ "libmimalloc-sys",
  "memchr",
  "memmap2",
+ "mimalloc",
  "neo_frizbee",
  "notify",
- "notify-debouncer-full 0.7.0",
- "once_cell",
  "parking_lot",
  "pathdiff",
- "rand",
+ "proptest",
+ "rand 0.8.5",
  "rayon",
  "regex",
+ "regex-syntax",
  "serde",
- "serde_json",
+ "signal-hook-registry",
  "smallvec",
- "smartstring",
  "tempfile",
  "thiserror 2.0.18",
- "toml",
  "tracing",
  "tracing-appender",
  "tracing-subscriber",
@@ -760,6 +780,12 @@ version = "0.1.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
 
+[[package]]
+name = "fnv"
+version = "1.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
+
 [[package]]
 name = "foldhash"
 version = "0.1.5"
@@ -775,15 +801,6 @@ dependencies = [
  "percent-encoding",
 ]
 
-[[package]]
-name = "fsevent-sys"
-version = "4.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "76ee7a02da4d231650c7cea31349b889be2f45ddb3ef3032d2ec8185f6313fd2"
-dependencies = [
- "libc",
-]
-
 [[package]]
 name = "futures"
 version = "0.3.32"
@@ -910,15 +927,14 @@ dependencies = [
 
 [[package]]
 name = "git2"
-version = "0.20.4"
+version = "0.21.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7b88256088d75a56f8ecfa070513a775dd9107f6530ef14919dac831af9cfe2b"
+checksum = "ddddbf932745a6be37109b6112d3ee09696106f848449069d3a57bba937ab82e"
 dependencies = [
  "bitflags 2.11.0",
  "libc",
  "libgit2-sys",
  "log",
- "url",
 ]
 
 [[package]]
@@ -946,15 +962,6 @@ dependencies = [
  "regex-syntax",
 ]
 
-[[package]]
-name = "grep-matcher"
-version = "0.1.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "36d7b71093325ab22d780b40d7df3066ae4aebb518ba719d38c697a8228a8023"
-dependencies = [
- "memchr",
-]
-
 [[package]]
 name = "half"
 version = "2.7.1"
@@ -1197,6 +1204,15 @@ dependencies = [
  "serde_core",
 ]
 
+[[package]]
+name = "indoc"
+version = "2.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706"
+dependencies = [
+ "rustversion",
+]
+
 [[package]]
 name = "inotify"
 version = "0.11.1"
@@ -1252,15 +1268,6 @@ dependencies = [
  "either",
 ]
 
-[[package]]
-name = "itertools"
-version = "0.14.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285"
-dependencies = [
- "either",
-]
-
 [[package]]
 name = "itoa"
 version = "1.0.17"
@@ -1327,9 +1334,9 @@ checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d"
 
 [[package]]
 name = "libgit2-sys"
-version = "0.18.3+1.9.2"
+version = "0.18.7+1.9.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c9b3acc4b91781bb0b3386669d325163746af5f6e4f73e6d2d630e09a35f3487"
+checksum = "23c7391e4b9f4ffab1a624223cc1d7385ff9a678f490768add717de7ea2f4d89"
 dependencies = [
  "cc",
  "libc",
@@ -1354,6 +1361,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "667f4fec20f29dfc6bc7357c582d91796c169ad7e2fce709468aefeb2c099870"
 dependencies = [
  "cc",
+ "cty",
  "libc",
 ]
 
@@ -1433,13 +1441,22 @@ checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
 
 [[package]]
 name = "memmap2"
-version = "0.9.10"
+version = "0.9.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3"
+checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0"
 dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "memoffset"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a"
+dependencies = [
+ "autocfg",
+]
+
 [[package]]
 name = "mimalloc"
 version = "0.1.48"
@@ -1480,7 +1497,7 @@ dependencies = [
  "mlua_derive",
  "num-traits",
  "parking_lot",
- "rustc-hash 2.1.1",
+ "rustc-hash",
  "rustversion",
 ]
 
@@ -1509,13 +1526,9 @@ dependencies = [
 
 [[package]]
 name = "neo_frizbee"
-version = "0.8.2"
+version = "0.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8c3404d265de0390885288ecf22fcdc4fb77519fe58168bac3d26ce65033b18b"
-dependencies = [
- "itertools 0.14.0",
- "raw-cpuid",
-]
+checksum = "7a2f6120a8da26bea3587731072111062c5d8c51ca3a3a75a716bd8b735d5882"
 
 [[package]]
 name = "nix"
@@ -1541,46 +1554,22 @@ dependencies = [
 
 [[package]]
 name = "notify"
-version = "8.2.0"
+version = "9.0.0-rc.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4d3d07927151ff8575b7087f245456e549fea62edf0ec4e565a5ee50c8402bc3"
+checksum = "b44b771d4dd781ef14c84078693e67495da6b47f609f72e8a4da8420a861240e"
 dependencies = [
  "bitflags 2.11.0",
- "fsevent-sys",
  "inotify",
  "kqueue",
  "libc",
  "log",
  "mio",
  "notify-types",
+ "objc2-core-foundation",
+ "objc2-core-services",
  "walkdir",
- "windows-sys 0.60.2",
-]
-
-[[package]]
-name = "notify-debouncer-full"
-version = "0.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "375bd3a138be7bfeff3480e4a623df4cbfb55b79df617c055cd810ba466fa078"
-dependencies = [
- "file-id",
- "log",
- "notify",
- "notify-types",
- "walkdir",
-]
-
-[[package]]
-name = "notify-debouncer-full"
-version = "0.7.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c02b49179cfebc9932238d04d6079912d26de0379328872846118a0fa0dbb302"
-dependencies = [
- "file-id",
- "log",
- "notify",
- "notify-types",
- "walkdir",
+ "windows-sys 0.61.2",
+ "xxhash-rust",
 ]
 
 [[package]]
@@ -1625,6 +1614,25 @@ dependencies = [
  "objc2-encode",
 ]
 
+[[package]]
+name = "objc2-core-foundation"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536"
+dependencies = [
+ "bitflags 2.11.0",
+]
+
+[[package]]
+name = "objc2-core-services"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "583300ad934cba24ff5292aee751ecc070f7ca6b39a574cc21b7b5e588e06a0b"
+dependencies = [
+ "libc",
+ "objc2-core-foundation",
+]
+
 [[package]]
 name = "objc2-encode"
 version = "4.1.0"
@@ -1723,7 +1731,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d"
 dependencies = [
  "phf_shared",
- "rand",
+ "rand 0.8.5",
 ]
 
 [[package]]
@@ -1788,6 +1796,12 @@ dependencies = [
  "plotters-backend",
 ]
 
+[[package]]
+name = "portable-atomic"
+version = "1.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49"
+
 [[package]]
 name = "potential_utf"
 version = "0.1.4"
@@ -1831,6 +1845,92 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "proptest"
+version = "1.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744"
+dependencies = [
+ "bitflags 2.11.0",
+ "num-traits",
+ "rand 0.9.4",
+ "rand_chacha 0.9.0",
+ "rand_xorshift",
+ "regex-syntax",
+ "rusty-fork",
+ "tempfile",
+ "unarray",
+]
+
+[[package]]
+name = "pyo3"
+version = "0.24.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e5203598f366b11a02b13aa20cab591229ff0a89fd121a308a5df751d5fc9219"
+dependencies = [
+ "cfg-if",
+ "indoc",
+ "libc",
+ "memoffset",
+ "once_cell",
+ "portable-atomic",
+ "pyo3-build-config",
+ "pyo3-ffi",
+ "pyo3-macros",
+ "unindent",
+]
+
+[[package]]
+name = "pyo3-build-config"
+version = "0.24.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "99636d423fa2ca130fa5acde3059308006d46f98caac629418e53f7ebb1e9999"
+dependencies = [
+ "once_cell",
+ "target-lexicon",
+]
+
+[[package]]
+name = "pyo3-ffi"
+version = "0.24.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "78f9cf92ba9c409279bc3305b5409d90db2d2c22392d443a87df3a1adad59e33"
+dependencies = [
+ "libc",
+ "pyo3-build-config",
+]
+
+[[package]]
+name = "pyo3-macros"
+version = "0.24.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b999cb1a6ce21f9a6b147dcf1be9ffedf02e0043aec74dc390f3007047cecd9"
+dependencies = [
+ "proc-macro2",
+ "pyo3-macros-backend",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "pyo3-macros-backend"
+version = "0.24.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "822ece1c7e1012745607d5cf0bcb2874769f0f7cb34c4cde03b9358eb9ef911a"
+dependencies = [
+ "heck",
+ "proc-macro2",
+ "pyo3-build-config",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "quick-error"
+version = "1.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0"
+
 [[package]]
 name = "quote"
 version = "1.0.45"
@@ -1859,8 +1959,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
 dependencies = [
  "libc",
- "rand_chacha",
- "rand_core",
+ "rand_chacha 0.3.1",
+ "rand_core 0.6.4",
+]
+
+[[package]]
+name = "rand"
+version = "0.9.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea"
+dependencies = [
+ "rand_chacha 0.9.0",
+ "rand_core 0.9.5",
 ]
 
 [[package]]
@@ -1870,7 +1980,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
 dependencies = [
  "ppv-lite86",
- "rand_core",
+ "rand_core 0.6.4",
+]
+
+[[package]]
+name = "rand_chacha"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"
+dependencies = [
+ "ppv-lite86",
+ "rand_core 0.9.5",
 ]
 
 [[package]]
@@ -1883,12 +2003,21 @@ dependencies = [
 ]
 
 [[package]]
-name = "raw-cpuid"
-version = "11.6.0"
+name = "rand_core"
+version = "0.9.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186"
+checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c"
 dependencies = [
- "bitflags 2.11.0",
+ "getrandom 0.3.4",
+]
+
+[[package]]
+name = "rand_xorshift"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a"
+dependencies = [
+ "rand_core 0.9.5",
 ]
 
 [[package]]
@@ -1982,9 +2111,9 @@ checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a"
 
 [[package]]
 name = "rmcp"
-version = "1.2.0"
+version = "1.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ba6b9d2f0efe2258b23767f1f9e0054cfbcac9c2d6f81a031214143096d7864f"
+checksum = "0810a9f717d9828f475fe1f629f4c305c8464b7f496c3a854b58d29e65f4058e"
 dependencies = [
  "async-trait",
  "base64",
@@ -2004,9 +2133,9 @@ dependencies = [
 
 [[package]]
 name = "rmcp-macros"
-version = "1.2.0"
+version = "1.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ab9d95d7ed26ad8306352b0d5f05b593222b272790564589790d210aa15caa9e"
+checksum = "6aefac48c364756e97f04c0401ba3231e8607882c7c1d92da0437dc16307904d"
 dependencies = [
  "darling",
  "proc-macro2",
@@ -2017,15 +2146,9 @@ dependencies = [
 
 [[package]]
 name = "rustc-hash"
-version = "1.1.0"
+version = "2.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
-
-[[package]]
-name = "rustc-hash"
-version = "2.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d"
+checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe"
 
 [[package]]
 name = "rustix"
@@ -2046,6 +2169,17 @@ version = "1.0.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d"
 
+[[package]]
+name = "rusty-fork"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cc6bf79ff24e648f6da1f8d1f011e9cac26491b619e6b9280f2b47f1774e6ee2"
+dependencies = [
+ "fnv",
+ "quick-error",
+ "tempfile",
+]
+
 [[package]]
 name = "same-file"
 version = "1.0.6"
@@ -2147,15 +2281,6 @@ dependencies = [
  "zmij",
 ]
 
-[[package]]
-name = "serde_spanned"
-version = "0.6.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3"
-dependencies = [
- "serde",
-]
-
 [[package]]
 name = "sharded-slab"
 version = "0.1.7"
@@ -2199,18 +2324,6 @@ version = "1.15.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03"
 
-[[package]]
-name = "smartstring"
-version = "1.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3fb72c633efbaa2dd666986505016c32c3044395ceaf881518399d2f4127ee29"
-dependencies = [
- "autocfg",
- "serde",
- "static_assertions",
- "version_check",
-]
-
 [[package]]
 name = "socket2"
 version = "0.6.3"
@@ -2227,12 +2340,6 @@ version = "1.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596"
 
-[[package]]
-name = "static_assertions"
-version = "1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
-
 [[package]]
 name = "strsim"
 version = "0.11.1"
@@ -2270,6 +2377,12 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "target-lexicon"
+version = "0.13.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
+
 [[package]]
 name = "tempfile"
 version = "3.27.0"
@@ -2424,47 +2537,6 @@ dependencies = [
  "tokio",
 ]
 
-[[package]]
-name = "toml"
-version = "0.8.23"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362"
-dependencies = [
- "serde",
- "serde_spanned",
- "toml_datetime",
- "toml_edit",
-]
-
-[[package]]
-name = "toml_datetime"
-version = "0.6.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c"
-dependencies = [
- "serde",
-]
-
-[[package]]
-name = "toml_edit"
-version = "0.22.27"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a"
-dependencies = [
- "indexmap",
- "serde",
- "serde_spanned",
- "toml_datetime",
- "toml_write",
- "winnow",
-]
-
-[[package]]
-name = "toml_write"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801"
-
 [[package]]
 name = "tracing"
 version = "0.1.44"
@@ -2538,6 +2610,12 @@ dependencies = [
  "tracing-log",
 ]
 
+[[package]]
+name = "unarray"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94"
+
 [[package]]
 name = "unicode-ident"
 version = "1.0.24"
@@ -2550,6 +2628,12 @@ version = "0.2.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
 
+[[package]]
+name = "unindent"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3"
+
 [[package]]
 name = "url"
 version = "2.5.8"
@@ -2954,15 +3038,6 @@ version = "0.53.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650"
 
-[[package]]
-name = "winnow"
-version = "0.7.15"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945"
-dependencies = [
- "memchr",
-]
-
 [[package]]
 name = "wit-bindgen"
 version = "0.51.0"
@@ -3057,6 +3132,12 @@ version = "0.6.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9"
 
+[[package]]
+name = "xxhash-rust"
+version = "0.8.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3"
+
 [[package]]
 name = "yoke"
 version = "0.8.1"
@@ -3156,9 +3237,9 @@ dependencies = [
 
 [[package]]
 name = "zlob"
-version = "1.3.0"
+version = "1.6.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "07934fb13f0f4e14281bab6b0e984cfc03891d2cf5b0a8bd0ab14fd240e21106"
+checksum = "f0f010d9b10495cc0037cb47cedc143c39c7fa2d59f752dca19036fd0234a210"
 dependencies = [
  "bindgen",
  "bitflags 2.11.0",

Cargo.toml

@@ -4,45 +4,50 @@ members = [
   "crates/fff-core",
   "crates/fff-mcp",
   "crates/fff-nvim",
+  "crates/fff-python",
   "crates/fff-query-parser",
   "crates/fff-grep",
 ]
+
 resolver = "2"
 
+[workspace.lints.clippy]
+module_inception = "allow"
+
 [workspace.dependencies]
-fff-grep = { version = "0.4.0", path = "crates/fff-grep" }
-fff-query-parser = { version = "0.4.0", path = "crates/fff-query-parser", default-features = false }
+fff-grep = { version = "0.10.5", path = "crates/fff-grep" }
+fff-query-parser = { version = "0.10.5", path = "crates/fff-query-parser", default-features = false }
 
 # Shared dependencies
 ahash = "0.8"
-bindet = "0.3"
 blake3 = "1.8.2"
 chrono = { version = "0.4", features = ["serde"] }
 ctrlc = "3.4.2"
 dirs = "5.0"
 dunce = "1.0"
 # git2 - base config without TLS (each crate adds platform-specific TLS)
-git2 = { version = "0.20.2", default-features = false, features = [
+git2 = { version = "0.21.0", default-features = false, features = [
   "vendored-libgit2",
 ] }
 glidesort = "0.1"
 globset = "0.4"
-grep-matcher = "0.1.8"
 heed = "0.22.0"
 ignore = "0.4.22"
 memmap2 = "0.9"
 mimalloc = "0.1.47"
-zlob = "1.3.0"
+signal-hook-registry = "1.4"
+zlob = { version = "=1.6.3" }
 
 mlua = { version = "0.11.1", features = ["module", "luajit"] }
-neo_frizbee = "0.8.2"
-notify = "8.1.0"
-notify-debouncer-full = "0.7"
+neo_frizbee = { version = "0.11.0", features = ["match_end_col"] }
+notify = { version = "9.0.0-rc.3" }
+notify-debouncer-full = { package = "fff-notify-debouncer-full", version = "0.9.4" }
 once_cell = "1.20.2"
 parking_lot = "0.12"
 pathdiff = "0.2.1"
 rayon = "1.8.0"
 regex = "1.11"
+regex-syntax = "0.8"
 smallvec = { version = "1.13", features = ["const_generics", "union"] }
 thiserror = "2.0.10"
 tracing = "0.1"
@@ -51,7 +56,21 @@ tracing = "0.1"
 opt-level = 3
 lto = "fat"
 codegen-units = 1
-strip = true
+strip = "debuginfo"
+
+[profile.ci]
+inherits = "release"
+# we use lto=fat locally for better SIMD for the march=native but
+# on CI when we cross compiling we should not exclude any cpu flags checking
+lto = "thin"
 
 [profile.bench]
 inherits = "release"
+
+# For Instruments / xctrace: release-level optimization but keep debuginfo
+# and symbols so sampled frames resolve to real Rust names.
+[profile.prof]
+inherits = "release"
+debug = "full"
+strip = false
+lto = "thin"

Formula/fff-mcp.rb

@@ -0,0 +1,60 @@
+# Originally authored by @jellydn (https://github.com/jellydn/homebrew-tap).
+# Maintained in-repo; auto-bumped by .github/workflows/release.yaml on stable releases.
+class FffMcp < Formula
+  desc "Fast file search toolkit for AI agents (MCP server)"
+  homepage "https://github.com/dmtrKovalenko/fff"
+  license "MIT"
+  version "0.10.5"
+
+  LIVECHECK_REPO = "dmtrKovalenko/fff".freeze
+  RELEASE_BASE = "https://github.com/dmtrKovalenko/fff/releases/download".freeze
+
+  on_macos do
+    on_arm do
+      url "#{RELEASE_BASE}/v#{version}/fff-mcp-aarch64-apple-darwin"
+      sha256 "2f1a75c2401e5df7f5a10827f9ce9fe8401f290ff99c3f3f1423c7d7b156d849"
+    end
+
+    on_intel do
+      url "#{RELEASE_BASE}/v#{version}/fff-mcp-x86_64-apple-darwin"
+      sha256 "c81e22a7fb657731a7441be877c622a110eb7e7dbc94e3b57d79c9cea91f38db"
+    end
+  end
+
+  on_linux do
+    on_arm do
+      url "#{RELEASE_BASE}/v#{version}/fff-mcp-aarch64-unknown-linux-gnu"
+      sha256 "48f5af0d9e6ef4506c5cccd3f4571da4f0acc9b9c0967832167e49fe651f8ee5"
+    end
+
+    on_intel do
+      url "#{RELEASE_BASE}/v#{version}/fff-mcp-x86_64-unknown-linux-gnu"
+      sha256 "0408cbac06303b3bf2041181089d8b4974b527883242bdcf8ba599b41c003973"
+    end
+  end
+
+  livecheck do
+    url "https://github.com/#{LIVECHECK_REPO}/releases/latest"
+    strategy :github_latest
+  end
+
+  def install
+    if OS.mac?
+      if Hardware::CPU.arm?
+        bin.install "fff-mcp-aarch64-apple-darwin" => "fff-mcp"
+      elsif Hardware::CPU.intel?
+        bin.install "fff-mcp-x86_64-apple-darwin" => "fff-mcp"
+      end
+    elsif OS.linux?
+      if Hardware::CPU.arm?
+        bin.install "fff-mcp-aarch64-unknown-linux-gnu" => "fff-mcp"
+      elsif Hardware::CPU.intel?
+        bin.install "fff-mcp-x86_64-unknown-linux-gnu" => "fff-mcp"
+      end
+    end
+  end
+
+  test do
+    system bin/"fff-mcp", "--healthcheck"
+  end
+end

Makefile

@@ -1,68 +1,280 @@
 PLENARY_DIR ?= ../plenary.nvim
+MINI_DIR ?= ../mini.nvim
 
-.PHONY: build test test-rust test-lua test-version test-bun test-node prepare-bun prepare-node set-npm-version header
+PREFIX ?= /usr/local
+LIBDIR ?= $(PREFIX)/lib
+INCLUDEDIR ?= $(PREFIX)/include
+
+# Compile-time cfg that gates the watcher + git-status fuzz stress test.
+STRESS_RUSTFLAGS := --cfg stress
+FFF_STRESS_DEFAULT_SEED ?= 0xDEADBEEFCAFEBABE
+
+SHELL := bash
+# Order matters: `-c` must be last so bash treats the recipe as the script
+# string rather than the literal `-o` / `pipefail` tokens.
+.SHELLFLAGS := -o pipefail -euc
+
+.PHONY: build build-c-lib install uninstall test test-rust test-rescan test-rescan-known-defects rescan-probe test-c-smoke test-c-api test-lua test-lua-snap test-version test-bun test-node prepare-bun prepare-bun-packaged prepare-node set-npm-version header test-stress test-stress-seeded test-stress-random test-stress-regressions test-stress-repos test-node-stress sync-js-api sync-js-api-check bump-homebrew-formula bump-install-mcp-sh test-bun-compile
+
+all: format test lint
+
+SYNC_API_SRC := packages/shared/fff-api.ts
+SYNC_API_TARGETS := packages/fff-node/src/fff-api.ts packages/fff-bun/src/fff-api.ts
+SYNC_API_BANNER := // ----------------------------------------------------------------------------\n// GENERATED FILE - DO NOT EDIT.\n// Copied from: ${SYNC_API_SRC}\n// Run make sync-js-api from the repo root to regenerate.\n// ----------------------------------------------------------------------------\n\n
+
+sync-js-api:
+	@for target in $(SYNC_API_TARGETS); do \
+		printf '$(SYNC_API_BANNER)' > "$$target"; \
+		cat $(SYNC_API_SRC) >> "$$target"; \
+		echo "synced: $$target"; \
+	done
+
+sync-js-api-check:
+	@status=0; \
+	for target in $(SYNC_API_TARGETS); do \
+		tmp=$$(mktemp); \
+		printf '$(SYNC_API_BANNER)' > "$$tmp"; \
+		cat $(SYNC_API_SRC) >> "$$tmp"; \
+		if ! cmp -s "$$tmp" "$$target"; then \
+			echo "out of date: $$target (run make sync-js-api)"; status=1; \
+		fi; \
+		rm -f "$$tmp"; \
+	done; \
+	exit $$status
 
 build:
-	cargo build --release --features zlob
+	cargo build --release --no-default-features --features zlob
+
+# Only the crates the e2e suites load (nvim lua tests + C/bun/node FFI tests),
+# skipping fff-python (pyo3) and fff-mcp (tokio/rmcp) which e2e never touches.
+build-e2e:
+	cargo build --release -p fff-nvim -p fff-c --no-default-features --features zlob
+
+build-c-lib:
+	cargo build --release -p fff-c --no-default-features --features zlob
 
 header:
 	cbindgen --config crates/fff-c/cbindgen.toml --crate fff-c --output crates/fff-c/include/fff.h
 
+# Install the C library and header under $(PREFIX) (default /usr/local).
+# Override PREFIX for user-local installs, e.g. `make install PREFIX=$$HOME/.local`.
+# DESTDIR is honoured for packagers.
+install: build-c-lib
+	install -d $(DESTDIR)$(LIBDIR)
+	install -d $(DESTDIR)$(INCLUDEDIR)
+	install -m 0644 crates/fff-c/include/fff.h $(DESTDIR)$(INCLUDEDIR)/fff.h
+	@if [ -f target/release/libfff_c.dylib ]; then \
+		install -m 0755 target/release/libfff_c.dylib $(DESTDIR)$(LIBDIR)/libfff_c.dylib; \
+		echo "Installed $(DESTDIR)$(LIBDIR)/libfff_c.dylib"; \
+	fi
+	@if [ -f target/release/libfff_c.so ]; then \
+		install -m 0755 target/release/libfff_c.so $(DESTDIR)$(LIBDIR)/libfff_c.so; \
+		echo "Installed $(DESTDIR)$(LIBDIR)/libfff_c.so"; \
+	fi
+	@if [ -f target/release/fff_c.dll ]; then \
+		install -m 0755 target/release/fff_c.dll $(DESTDIR)$(LIBDIR)/fff_c.dll; \
+		echo "Installed $(DESTDIR)$(LIBDIR)/fff_c.dll"; \
+	fi
+	@echo "Installed header $(DESTDIR)$(INCLUDEDIR)/fff.h"
+
+uninstall:
+	rm -f $(DESTDIR)$(LIBDIR)/libfff_c.dylib
+	rm -f $(DESTDIR)$(LIBDIR)/libfff_c.so
+	rm -f $(DESTDIR)$(LIBDIR)/fff_c.dll
+	rm -f $(DESTDIR)$(INCLUDEDIR)/fff.h
+	@echo "Removed fff-c from $(DESTDIR)$(PREFIX)"
+
 test-setup:
 	@if [ ! -d "$(PLENARY_DIR)" ]; then \
 		echo "Cloning plenary.nvim..."; \
 		git clone --depth 1 https://github.com/nvim-lua/plenary.nvim $(PLENARY_DIR); \
 	fi
+	@if [ ! -d "$(MINI_DIR)" ]; then \
+		echo "Cloning mini.nvim..."; \
+		git clone --depth 1 https://github.com/echasnovski/mini.nvim $(MINI_DIR); \
+	fi
 
 test-rust:
-	cargo test --workspace --features zlob --exclude fff-nvim
+	cargo test --workspace --no-default-features --features zlob --exclude fff-nvim --exclude fff-python
 
-test-lua: test-setup build
+# Watcher rescan harness: asserts that editing, build output, git activity and
+# preview reads all stay on the incremental path instead of re-walking the tree.
+test-rescan:
+	cargo test -p fff-search --no-default-features --features zlob \
+		--lib --test rescan_regression -- rescan
+
+# Live probe for watcher rescan requests and their causes.
+# Usage: make rescan-probe DIR=~/some/repo [SECONDS=120]
+rescan-probe:
+	cargo run --release -p fff-nvim --bin rescan_probe \
+		--no-default-features --features zlob,rescan-stats -- \
+		$(or $(DIR),.) $(if $(SECONDS),--seconds $(SECONDS),)
+
+# The same harness, restricted to cases that currently fail on purpose. Each
+# `#[ignore]` reason names the code that causes the unnecessary rescan.
+test-rescan-known-defects:
+	cargo test --no-fail-fast -p fff-search --no-default-features --features zlob \
+		--lib --test rescan_regression -- --ignored --nocapture
+
+CC ?= cc
+CFLAGS ?= -O0 -g -Wall -Wextra -std=c99
+TARGET_DIR ?= target/release
+SMOKE_BIN := $(TARGET_DIR)/fff_c_smoke
+SMOKE_SRC := crates/fff-c/tests/smoke.c
+SMOKE_INCLUDE := crates/fff-c/include
+
+test-c-smoke: build-e2e
+	$(CC) $(CFLAGS) -I $(SMOKE_INCLUDE) -L $(TARGET_DIR) \
+		-Wl,-rpath,@loader_path/../target/release \
+		-Wl,-rpath,$$(pwd)/$(TARGET_DIR) \
+		$(SMOKE_SRC) -lfff_c -o $(SMOKE_BIN)
+	$(SMOKE_BIN) .
+
+# Alias kept for the `external-tests.yml` workflow naming.
+test-c-api: test-c-smoke
+
+# neovim instance swallows internal crashes and doesn't rise the the error exiting silently
+# so check the stdout in case the sigsegv coming out of fff was printed (actual regression).
+# Output is streamed live via `tee`; pipefail (set above) propagates nvim's exit.
+test-lua: test-setup build-e2e
+	@logfile=$$(mktemp); \
+	trap 'rm -f "$$logfile"' EXIT; \
 	nvim --headless -u tests/minimal_init.lua \
-		-c "PlenaryBustedFile tests/fff_core_spec.lua" 2>&1
+		-c "PlenaryBustedDirectory tests/ {minimal_init = 'tests/minimal_init.lua'}" 2>&1 \
+		| tee "$$logfile"; \
+	if grep -qE "SIG(SEGV|ABRT|BUS|FPE|ILL)" "$$logfile"; then \
+		echo ""; \
+		echo "FAIL: native crash detected during lua tests"; \
+		exit 1; \
+	fi
+
+test-lua-snap: test-setup build-e2e
+	@logfile=$$(mktemp); \
+	trap 'rm -f "$$logfile"' EXIT; \
+	nvim --headless -u tests/minimal_init.lua \
+		-c "lua local ok,err=pcall(require('mini.test').run_file,'tests/picker_ui_snap.lua'); if not ok then io.stderr:write('mini.test failed to load: '..tostring(err)..'\\n'); vim.cmd('cquit 2') end" 2>&1 \
+		| tee "$$logfile"; \
+	if grep -qE "SIG(SEGV|ABRT|BUS|FPE|ILL)" "$$logfile"; then \
+		echo ""; \
+		echo "FAIL: native crash detected during snapshot tests"; \
+		exit 1; \
+	fi
 
 test-version: test-setup
 	nvim --headless -u tests/minimal_init.lua \
 		-c "PlenaryBustedFile tests/version_spec.lua" 2>&1
 
-prepare-bun: build
+prepare-bun: build-e2e sync-js-api
 	mkdir -p packages/fff-bun/bin
-	cp target/release/libfff_c.dylib packages/fff-bun/bin/ 2>/dev/null; \
-	cp target/release/libfff_c.so packages/fff-bun/bin/ 2>/dev/null; \
-	cp target/release/fff_c.dll packages/fff-bun/bin/ 2>/dev/null; \
-	true
+	cp target/release/libfff_c.dylib packages/fff-bun/bin/ 2>/dev/null || true; \
+	cp target/release/libfff_c.so packages/fff-bun/bin/ 2>/dev/null || true; \
+	cp target/release/fff_c.dll packages/fff-bun/bin/ 2>/dev/null || true
 
-prepare-node: build
+prepare-node: build-e2e sync-js-api
 	mkdir -p packages/fff-node/bin
-	cp target/release/libfff_c.dylib packages/fff-node/bin/ 2>/dev/null; \
-	cp target/release/libfff_c.so packages/fff-node/bin/ 2>/dev/null; \
-	cp target/release/fff_c.dll packages/fff-node/bin/ 2>/dev/null; \
-	true
+	cp target/release/libfff_c.dylib packages/fff-node/bin/ 2>/dev/null || true; \
+	cp target/release/libfff_c.so packages/fff-node/bin/ 2>/dev/null || true; \
+	cp target/release/fff_c.dll packages/fff-node/bin/ 2>/dev/null || true
 
 test-bun: prepare-bun
-	cd packages/fff-bun && bun test src/
+	cd packages/fff-bun && bun test test/
+	cd packages/pi-fff && bun test test/
+
+# Same as prepare-bun but puts the compiled binary into the actual npm package location
+prepare-bun-packaged: prepare-bun
+	@machine=$$(uname -m); \
+	case "$$machine" in \
+	  x86_64|amd64) arch=x64 ;; \
+	  aarch64|arm64) arch=arm64 ;; \
+	  *) echo "unsupported arch: $$machine" >&2; exit 1 ;; \
+	esac; \
+	case "$$(uname -s)" in \
+	  Darwin) lib=libfff_c.dylib; pkg=fff-bin-darwin-$$arch ;; \
+	  Linux) lib=libfff_c.so; \
+	    if ldd --version 2>&1 | grep -qi musl; then libc=musl; else libc=gnu; fi; \
+	    pkg=fff-bin-linux-$$arch-$$libc ;; \
+	  MINGW*|MSYS*|CYGWIN*|Windows_NT) lib=fff_c.dll; pkg=fff-bin-win32-$$arch ;; \
+	  *) echo "unsupported OS: $$(uname -s)" >&2; exit 1 ;; \
+	esac; \
+	src=target/release/$$lib; \
+	[ -f "$$src" ] || { echo "missing built library: $$src" >&2; exit 1; }; \
+	dest=packages/fff-bun/node_modules/@ff-labs/$$pkg; \
+	rm -rf "$$dest"; mkdir -p "$$dest"; \
+	cp "$$src" "$$dest/$$lib"; \
+	printf '{ "name": "@ff-labs/%s", "version": "0.0.0", "main": "%s" }\n' "$$pkg" "$$lib" > "$$dest/package.json"
+
+# Compile a bun example to a standalone executable and run it. Verifies the
+# native libfff_c is embedded + loaded from a `bun build --compile` binary.
+# The staged bin package is removed before running so success proves the lib
+# was embedded, not resolved from disk.
+test-bun-compile: prepare-bun-packaged
+	cd packages/fff-bun && \
+		if [ "$$(uname -s)" = "Linux" ]; then \
+		  if ldd --version 2>&1 | grep -qi musl; then DEFINE='--define FFF_LIBC="musl"'; \
+		  else DEFINE='--define FFF_LIBC="gnu"'; fi; \
+		else DEFINE=""; fi; \
+		bun build --compile $$DEFINE ./examples/glob-bench.ts --outfile ./glob-bench-bin && \
+		EXE=./glob-bench-bin; [ -f "$$EXE.exe" ] && EXE="$$EXE.exe"; \
+		rm -rf bin node_modules/@ff-labs; \
+		"$$EXE" . '**/*.ts' 1 | tee /tmp/fff-compile-e2e.log && \
+		grep -q 'fff.glob' /tmp/fff-compile-e2e.log
+	rm -f packages/fff-bun/glob-bench-bin packages/fff-bun/glob-bench-bin.exe
 
 test-node: prepare-node
-	cd packages/fff-node && npm run build && node test/e2e.mjs
+	cd packages/fff-node && npm run build && node test/e2e.mjs && node test/watch.mjs
 
-test: test-rust test-lua test-version test-bun test-node
+test-js: test-bun test-node
+
+# Bug pinning stress test script over fff-node for issue #515
+# Just keep it untouched because it's good enough + some stress for SDK
+FFF_STRESS_ITERS ?= 50
+test-node-stress: prepare-node
+	cd packages/fff-node && npm run build && \
+		FFF_STRESS_ITERS=$(FFF_STRESS_ITERS) node test/stress-515.mjs
+
+test: test-rust test-lua test-lua-snap test-version test-bun test-node test-node-stress
+
+test-stress-seeded:
+	FFF_STRESS_SEED="$${FFF_STRESS_SEED:-$(FFF_STRESS_DEFAULT_SEED)}" \
+	RUSTFLAGS="$(STRESS_RUSTFLAGS)" \
+	cargo test --release \
+		-p fff-search \
+		--test fuzz_git_watcher_stress \
+		--no-default-features --features zlob \
+		-- --nocapture stress_seeded
+
+test-stress-random:
+	RUSTFLAGS="$(STRESS_RUSTFLAGS)" \
+	cargo test --release \
+		-p fff-search \
+		--test fuzz_git_watcher_stress \
+		--no-default-features --features zlob \
+		-- --nocapture stress_random
+
+test-stress-regressions:
+	RUSTFLAGS="$(STRESS_RUSTFLAGS)" \
+	cargo test --release \
+		-p fff-search \
+		--test fuzz_git_watcher_stress \
+		--no-default-features --features zlob \
+		-- --nocapture stress_regression stress_merge_conflict_convergence
+
+test-stress-repos:
+	RUSTFLAGS="$(STRESS_RUSTFLAGS)" \
+	cargo test --release \
+		-p fff-search \
+		--test fuzz_real_repos \
+		--no-default-features --features zlob \
+		-- --nocapture
+
+test-stress: test-stress-seeded test-stress-random test-stress-regressions test-stress-repos
 
 # Update version in a package.json, including optionalDependencies.
 # Usage: make set-npm-version PKG=packages/fff-bun VERSION=1.0.0-nightly.abc1234
 set-npm-version:
 	@test -n "$(PKG)" || (echo "PKG is required" && exit 1)
 	@test -n "$(VERSION)" || (echo "VERSION is required" && exit 1)
-	node -e " \
-		const fs = require('fs'); \
-		const pkg = JSON.parse(fs.readFileSync('$(PKG)/package.json', 'utf8')); \
-		pkg.version = '$(VERSION)'; \
-		if (pkg.optionalDependencies) { \
-			for (const dep of Object.keys(pkg.optionalDependencies)) { \
-				pkg.optionalDependencies[dep] = '$(VERSION)'; \
-			} \
-		} \
-		fs.writeFileSync('$(PKG)/package.json', JSON.stringify(pkg, null, 2) + '\n'); \
-	"
+	node scripts/set-npm-version.mjs "$(PKG)" "$(VERSION)"
 	@echo "Set $(PKG) to $(VERSION)"
 
 format-rust:
@@ -70,26 +282,80 @@ format-rust:
 format-lua:
 	stylua .
 format-ts:
-	bun format
+	cd packages && bun format
 
 format: format-rust format-lua format-ts
 
 lint-rust:
-	cargo clippy --workspace --features zlob -- -D warnings
+	cargo clippy --workspace --no-default-features --features zlob -- -D warnings
 lint-lua:
 	 ~/.luarocks/bin/luacheck .
 lint-ts:
-	bun lint
+	cd packages && bun lint
 
 lint: lint-rust lint-lua lint-ts
 
 check: format lint
 
+FFF_RELEASE_REPO ?= dmtrKovalenko/fff
+FFF_FORMULA_PATH ?= Formula/fff-mcp.rb
+FFF_INSTALL_SCRIPT_PATH ?= install-mcp.sh
+
+# Read the sha256 for $1 (filename, no .sha256 suffix). Reads from
+# BINARIES_DIR/$1.sha256 when set; otherwise curls the GitHub release.
+define fff_fetch_sha
+	if [ -n "$$BINARIES_DIR" ]; then \
+		awk '{print $$1}' "$$BINARIES_DIR/$$1.sha256" \
+			|| { echo "Missing checksum file: $$BINARIES_DIR/$$1.sha256" >&2; exit 1; }; \
+	else \
+		curl -fsSL "https://github.com/$(FFF_RELEASE_REPO)/releases/download/v$(VERSION)/$$1.sha256" \
+			| awk '{print $$1}'; \
+	fi
+endef
+
+bump-homebrew-formula:
+	@test -n "$(VERSION)" || (echo "VERSION is required. Usage: make bump-homebrew-formula VERSION=0.9.1 [BINARIES_DIR=./binaries]" && exit 1)
+	@export BINARIES_DIR="$(BINARIES_DIR)"; \
+	fetch_sha() { $(fff_fetch_sha); }; \
+	sha_darwin_arm="$$(fetch_sha fff-mcp-aarch64-apple-darwin)"; \
+	sha_darwin_intel="$$(fetch_sha fff-mcp-x86_64-apple-darwin)"; \
+	sha_linux_arm="$$(fetch_sha fff-mcp-aarch64-unknown-linux-gnu)"; \
+	sha_linux_intel="$$(fetch_sha fff-mcp-x86_64-unknown-linux-gnu)"; \
+	sed -i.bak \
+		-e 's/^  version "[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*"$$/  version "$(VERSION)"/' \
+		-e '/fff-mcp-aarch64-apple-darwin"$$/{n;s/sha256 "[a-f0-9]*"/sha256 "'"$$sha_darwin_arm"'"/;}' \
+		-e '/fff-mcp-x86_64-apple-darwin"$$/{n;s/sha256 "[a-f0-9]*"/sha256 "'"$$sha_darwin_intel"'"/;}' \
+		-e '/fff-mcp-aarch64-unknown-linux-gnu"$$/{n;s/sha256 "[a-f0-9]*"/sha256 "'"$$sha_linux_arm"'"/;}' \
+		-e '/fff-mcp-x86_64-unknown-linux-gnu"$$/{n;s/sha256 "[a-f0-9]*"/sha256 "'"$$sha_linux_intel"'"/;}' \
+		"$(FFF_FORMULA_PATH)" && rm -f "$(FFF_FORMULA_PATH).bak"; \
+	echo "Bumped $(FFF_FORMULA_PATH) to v$(VERSION)"
+
+bump-install-mcp-sh:
+	@test -n "$(VERSION)" || (echo "VERSION is required. Usage: make bump-install-mcp-sh VERSION=0.9.1 [BINARIES_DIR=./binaries]" && exit 1)
+	@export BINARIES_DIR="$(BINARIES_DIR)"; \
+	fetch_sha() { $(fff_fetch_sha); }; \
+	sha_linux_intel="$$(fetch_sha fff-mcp-x86_64-unknown-linux-musl)"; \
+	sha_linux_arm="$$(fetch_sha fff-mcp-aarch64-unknown-linux-musl)"; \
+	sha_darwin_intel="$$(fetch_sha fff-mcp-x86_64-apple-darwin)"; \
+	sha_darwin_arm="$$(fetch_sha fff-mcp-aarch64-apple-darwin)"; \
+	sha_win_intel="$$(fetch_sha fff-mcp-x86_64-pc-windows-msvc.exe)"; \
+	sha_win_arm="$$(fetch_sha fff-mcp-aarch64-pc-windows-msvc.exe)"; \
+	sed -i.bak \
+		-e 's|^PINNED_RELEASE_TAG=".*"|PINNED_RELEASE_TAG="v$(VERSION)"|' \
+		-e 's|^SHA256_X86_64_UNKNOWN_LINUX_MUSL=".*"|SHA256_X86_64_UNKNOWN_LINUX_MUSL="'"$$sha_linux_intel"'"|' \
+		-e 's|^SHA256_AARCH64_UNKNOWN_LINUX_MUSL=".*"|SHA256_AARCH64_UNKNOWN_LINUX_MUSL="'"$$sha_linux_arm"'"|' \
+		-e 's|^SHA256_X86_64_APPLE_DARWIN=".*"|SHA256_X86_64_APPLE_DARWIN="'"$$sha_darwin_intel"'"|' \
+		-e 's|^SHA256_AARCH64_APPLE_DARWIN=".*"|SHA256_AARCH64_APPLE_DARWIN="'"$$sha_darwin_arm"'"|' \
+		-e 's|^SHA256_X86_64_PC_WINDOWS_MSVC=".*"|SHA256_X86_64_PC_WINDOWS_MSVC="'"$$sha_win_intel"'"|' \
+		-e 's|^SHA256_AARCH64_PC_WINDOWS_MSVC=".*"|SHA256_AARCH64_PC_WINDOWS_MSVC="'"$$sha_win_arm"'"|' \
+		"$(FFF_INSTALL_SCRIPT_PATH)" && rm -f "$(FFF_INSTALL_SCRIPT_PATH).bak"; \
+	echo "Bumped $(FFF_INSTALL_SCRIPT_PATH) tag + checksums to v$(VERSION)"
+
 CRATES_TO_PUBLISH= fff-grep fff-query-parser fff-search
 
 publish-crates:
 	@test -n "$(V)" || (echo "V is required. Usage: make publish-crates V=0.2.0" && exit 1)
-	cargo install cargo-edit
+	cargo install cargo-edit --force --locked
 	cargo set-version $(V) || exit 1;
 	@for crate in $(CRATES_TO_PUBLISH); do \
 		cargo publish -p $$crate --allow-dirty $$(if [ -n "$$CI" ]; then echo "--no-verify"; fi) || exit 1; \

_typos.toml

@@ -4,6 +4,14 @@ extend-exclude = ["/CHANGELOG.md", "data/filetypes/base.lua"]
 [default.extend-words]
 noice = "noice"
 fo = "fo"
+ba = "ba"
+ue = "ue"
+# file extensions that look like typos
+thm = "thm"
+# some typos we use for tests
+comparsion = "comparsion"
+modfiers = "modfiers"
+shcema = "shcema"
 
 [default]
 extend-ignore-re = [

biome.json

@@ -1,35 +0,0 @@
-{
-  "$schema": "https://biomejs.dev/schemas/2.4.4/schema.json",
-  "files": {
-    "includes": ["packages/**/*.ts", "!packages/*/dist"],
-    "ignoreUnknown": true
-  },
-  "formatter": {
-    "enabled": true,
-    "indentStyle": "space",
-    "indentWidth": 2,
-    "lineWidth": 90
-  },
-  "javascript": {
-    "formatter": {
-      "quoteStyle": "double",
-      "trailingCommas": "all",
-      "semicolons": "always"
-    }
-  },
-  "linter": {
-    "enabled": true,
-    "rules": {
-      "recommended": true,
-      "style": {
-        "noNonNullAssertion": "off"
-      },
-      "suspicious": {
-        "noExplicitAny": "off"
-      },
-      "complexity": {
-        "noForEach": "off"
-      }
-    }
-  }
-}

bun.lock

@@ -1,148 +0,0 @@
-{
-  "lockfileVersion": 1,
-  "configVersion": 1,
-  "workspaces": {
-    "": {
-      "devDependencies": {
-        "@biomejs/biome": "^2.4.4",
-      },
-    },
-    "packages/fff-bun": {
-      "name": "@ff-labs/fff-bun",
-      "version": "0.1.37",
-      "bin": {
-        "fff": "./scripts/cli.ts",
-        "fff-demo": "./examples/search.ts",
-        "fff-grep": "./examples/grep.ts",
-      },
-      "devDependencies": {
-        "@types/bun": "^1.3.8",
-        "typescript": "^5.0.0",
-      },
-      "optionalDependencies": {
-        "@ff-labs/fff-bun-darwin-arm64": "0.0.0",
-        "@ff-labs/fff-bun-darwin-x64": "0.0.0",
-        "@ff-labs/fff-bun-linux-arm64-gnu": "0.0.0",
-        "@ff-labs/fff-bun-linux-arm64-musl": "0.0.0",
-        "@ff-labs/fff-bun-linux-x64-gnu": "0.0.0",
-        "@ff-labs/fff-bun-linux-x64-musl": "0.0.0",
-        "@ff-labs/fff-bun-win32-arm64": "0.0.0",
-        "@ff-labs/fff-bun-win32-x64": "0.0.0",
-      },
-      "peerDependencies": {
-        "bun": ">=1.0.0",
-      },
-    },
-    "packages/fff-node": {
-      "name": "@ff-labs/fff-node",
-      "version": "0.1.37",
-      "bin": {
-        "fff-node": "./dist/scripts/cli.js",
-      },
-      "dependencies": {
-        "ffi-rs": "^1.0.0",
-      },
-      "devDependencies": {
-        "@types/node": "^22.0.0",
-        "typescript": "^5.0.0",
-      },
-      "optionalDependencies": {
-        "@ff-labs/fff-bun-darwin-arm64": "0.0.0",
-        "@ff-labs/fff-bun-darwin-x64": "0.0.0",
-        "@ff-labs/fff-bun-linux-arm64-gnu": "0.0.0",
-        "@ff-labs/fff-bun-linux-arm64-musl": "0.0.0",
-        "@ff-labs/fff-bun-linux-x64-gnu": "0.0.0",
-        "@ff-labs/fff-bun-linux-x64-musl": "0.0.0",
-        "@ff-labs/fff-bun-win32-arm64": "0.0.0",
-        "@ff-labs/fff-bun-win32-x64": "0.0.0",
-      },
-    },
-  },
-  "packages": {
-    "@biomejs/biome": ["@biomejs/biome@2.4.4", "", { "optionalDependencies": { "@biomejs/cli-darwin-arm64": "2.4.4", "@biomejs/cli-darwin-x64": "2.4.4", "@biomejs/cli-linux-arm64": "2.4.4", "@biomejs/cli-linux-arm64-musl": "2.4.4", "@biomejs/cli-linux-x64": "2.4.4", "@biomejs/cli-linux-x64-musl": "2.4.4", "@biomejs/cli-win32-arm64": "2.4.4", "@biomejs/cli-win32-x64": "2.4.4" }, "bin": { "biome": "bin/biome" } }, "sha512-tigwWS5KfJf0cABVd52NVaXyAVv4qpUXOWJ1rxFL8xF1RVoeS2q/LK+FHgYoKMclJCuRoCWAPy1IXaN9/mS61Q=="],
-
-    "@biomejs/cli-darwin-arm64": ["@biomejs/cli-darwin-arm64@2.4.4", "", { "os": "darwin", "cpu": "arm64" }, "sha512-jZ+Xc6qvD6tTH5jM6eKX44dcbyNqJHssfl2nnwT6vma6B1sj7ZLTGIk6N5QwVBs5xGN52r3trk5fgd3sQ9We9A=="],
-
-    "@biomejs/cli-darwin-x64": ["@biomejs/cli-darwin-x64@2.4.4", "", { "os": "darwin", "cpu": "x64" }, "sha512-Dh1a/+W+SUCXhEdL7TiX3ArPTFCQKJTI1mGncZNWfO+6suk+gYA4lNyJcBB+pwvF49uw0pEbUS49BgYOY4hzUg=="],
-
-    "@biomejs/cli-linux-arm64": ["@biomejs/cli-linux-arm64@2.4.4", "", { "os": "linux", "cpu": "arm64" }, "sha512-V/NFfbWhsUU6w+m5WYbBenlEAz8eYnSqRMDMAW3K+3v0tYVkNyZn8VU0XPxk/lOqNXLSCCrV7FmV/u3SjCBShg=="],
-
-    "@biomejs/cli-linux-arm64-musl": ["@biomejs/cli-linux-arm64-musl@2.4.4", "", { "os": "linux", "cpu": "arm64" }, "sha512-+sPAXq3bxmFwhVFJnSwkSF5Rw2ZAJMH3MF6C9IveAEOdSpgajPhoQhbbAK12SehN9j2QrHpk4J/cHsa/HqWaYQ=="],
-
-    "@biomejs/cli-linux-x64": ["@biomejs/cli-linux-x64@2.4.4", "", { "os": "linux", "cpu": "x64" }, "sha512-R4+ZCDtG9kHArasyBO+UBD6jr/FcFCTH8QkNTOCu0pRJzCWyWC4EtZa2AmUZB5h3e0jD7bRV2KvrENcf8rndBg=="],
-
-    "@biomejs/cli-linux-x64-musl": ["@biomejs/cli-linux-x64-musl@2.4.4", "", { "os": "linux", "cpu": "x64" }, "sha512-gGvFTGpOIQDb5CQ2VC0n9Z2UEqlP46c4aNgHmAMytYieTGEcfqhfCFnhs6xjt0S3igE6q5GLuIXtdQt3Izok+g=="],
-
-    "@biomejs/cli-win32-arm64": ["@biomejs/cli-win32-arm64@2.4.4", "", { "os": "win32", "cpu": "arm64" }, "sha512-trzCqM7x+Gn832zZHgr28JoYagQNX4CZkUZhMUac2YxvvyDRLJDrb5m9IA7CaZLlX6lTQmADVfLEKP1et1Ma4Q=="],
-
-    "@biomejs/cli-win32-x64": ["@biomejs/cli-win32-x64@2.4.4", "", { "os": "win32", "cpu": "x64" }, "sha512-gnOHKVPFAAPrpoPt2t+Q6FZ7RPry/FDV3GcpU53P3PtLNnQjBmKyN2Vh/JtqXet+H4pme8CC76rScwdjDcT1/A=="],
-
-    "@ff-labs/fff-bun": ["@ff-labs/fff-bun@workspace:packages/fff-bun"],
-
-    "@ff-labs/fff-node": ["@ff-labs/fff-node@workspace:packages/fff-node"],
-
-    "@oven/bun-darwin-aarch64": ["@oven/bun-darwin-aarch64@1.3.10", "", { "os": "darwin", "cpu": "arm64" }, "sha512-PXgg5gqcS/rHwa1hF0JdM1y5TiyejVrMHoBmWY/DjtfYZoFTXie1RCFOkoG0b5diOOmUcuYarMpH7CSNTqwj+w=="],
-
-    "@oven/bun-darwin-x64": ["@oven/bun-darwin-x64@1.3.10", "", { "os": "darwin", "cpu": "x64" }, "sha512-Nhssuh7GBpP5PiDSOl3+qnoIG7PJo+ec2oomDevnl9pRY6x6aD2gRt0JE+uf+A8Om2D6gjeHCxjEdrw5ZHE8mA=="],
-
-    "@oven/bun-darwin-x64-baseline": ["@oven/bun-darwin-x64-baseline@1.3.10", "", { "os": "darwin", "cpu": "x64" }, "sha512-w1gaTlqU0IJCmJ1X+PGHkdNU1n8Gemx5YKkjhkJIguvFINXEBB5U1KG82QsT65Tk4KyNMfbLTlmy4giAvUoKfA=="],
-
-    "@oven/bun-linux-aarch64": ["@oven/bun-linux-aarch64@1.3.10", "", { "os": "linux", "cpu": "arm64" }, "sha512-OUgPHfL6+PM2Q+tFZjcaycN3D7gdQdYlWnwMI31DXZKY1r4HINWk9aEz9t/rNaHg65edwNrt7dsv9TF7xK8xIA=="],
-
-    "@oven/bun-linux-aarch64-musl": ["@oven/bun-linux-aarch64-musl@1.3.10", "", { "os": "linux", "cpu": "arm64" }, "sha512-Ui5pAgM7JE9MzHokF0VglRMkbak3lTisY4Mf1AZutPACXWgKJC5aGrgnHBfkl7QS6fEeYb0juy1q4eRznRHOsw=="],
-
-    "@oven/bun-linux-x64": ["@oven/bun-linux-x64@1.3.10", "", { "os": "linux", "cpu": "x64" }, "sha512-bzUgYj/PIZziB/ZesIP9HUyfvh6Vlf3od+TrbTTyVEuCSMKzDPQVW/yEbRp0tcHO3alwiEXwJDrWrHAguXlgiQ=="],
-
-    "@oven/bun-linux-x64-baseline": ["@oven/bun-linux-x64-baseline@1.3.10", "", { "os": "linux", "cpu": "x64" }, "sha512-oqvMDYpX6dGJO03HgO5bXuccEsH3qbdO3MaAiAlO4CfkBPLUXz3N0DDElg5hz0L6ktdDVKbQVE5lfe+LAUISQg=="],
-
-    "@oven/bun-linux-x64-musl": ["@oven/bun-linux-x64-musl@1.3.10", "", { "os": "linux", "cpu": "x64" }, "sha512-poVXvOShekbexHq45b4MH/mRjQKwACAC8lHp3Tz/hEDuz0/20oncqScnmKwzhBPEpqJvydXficXfBYuSim8opw=="],
-
-    "@oven/bun-linux-x64-musl-baseline": ["@oven/bun-linux-x64-musl-baseline@1.3.10", "", { "os": "linux", "cpu": "x64" }, "sha512-/hOZ6S1VsTX6vtbhWVL9aAnOrdpuO54mAGUWpTdMz7dFG5UBZ/VUEiK0pBkq9A1rlBk0GeD/6Y4NBFl8Ha7cRA=="],
-
-    "@oven/bun-windows-aarch64": ["@oven/bun-windows-aarch64@1.3.10", "", { "os": "win32", "cpu": "arm64" }, "sha512-GXbz2swvN2DLw2dXZFeedMxSJtI64xQ9xp9Eg7Hjejg6mS2E4dP1xoQ2yAo2aZPi/2OBPAVaGzppI2q20XumHA=="],
-
-    "@oven/bun-windows-x64": ["@oven/bun-windows-x64@1.3.10", "", { "os": "win32", "cpu": "x64" }, "sha512-qaS1In3yfC/Z/IGQriVmF8GWwKuNqiw7feTSJWaQhH5IbL6ENR+4wGNPniZSJFaM/SKUO0e/YCRdoVBvgU4C1g=="],
-
-    "@oven/bun-windows-x64-baseline": ["@oven/bun-windows-x64-baseline@1.3.10", "", { "os": "win32", "cpu": "x64" }, "sha512-gh3UAHbUdDUG6fhLc1Csa4IGdtghue6U8oAIXWnUqawp6lwb3gOCRvp25IUnLF5vUHtgfMxuEUYV7YA2WxVutw=="],
-
-    "@types/bun": ["@types/bun@1.3.9", "", { "dependencies": { "bun-types": "1.3.9" } }, "sha512-KQ571yULOdWJiMH+RIWIOZ7B2RXQGpL1YQrBtLIV3FqDcCu6FsbFUBwhdKUlCKUpS3PJDsHlJ1QKlpxoVR+xtw=="],
-
-    "@types/node": ["@types/node@22.19.15", "", { "dependencies": { "undici-types": "~6.21.0" } }, "sha512-F0R/h2+dsy5wJAUe3tAU6oqa2qbWY5TpNfL/RGmo1y38hiyO1w3x2jPtt76wmuaJI4DQnOBu21cNXQ2STIUUWg=="],
-
-    "@yuuang/ffi-rs-android-arm64": ["@yuuang/ffi-rs-android-arm64@1.3.1", "", { "os": "android", "cpu": "arm64" }, "sha512-V4nmlXdOYZEa7GOxSExVG95SLp8FE0iTq2yKeN54UlfNMr3Sik+1Ff57LcCv7qYcn4TBqnBAt5rT3FAM6T6caQ=="],
-
-    "@yuuang/ffi-rs-darwin-arm64": ["@yuuang/ffi-rs-darwin-arm64@1.3.1", "", { "os": "darwin", "cpu": "arm64" }, "sha512-YlnTMIyzfW3mAULC5ZA774nzQfFlYXM0rrfq/8ZzWt+IMbYk55a++jrI+6JeKV+1EqlDS3TFBEFtjdBNG94KzQ=="],
-
-    "@yuuang/ffi-rs-darwin-x64": ["@yuuang/ffi-rs-darwin-x64@1.3.1", "", { "os": "darwin", "cpu": "x64" }, "sha512-sI3LpQQ34SX4nyOHc5yxA7FSqs9qPEUMqW/y/wWo9cuyPpaHMFsi/BeOVYsnC0syp3FrY7gzn6RnD6PlXCktXg=="],
-
-    "@yuuang/ffi-rs-linux-arm-gnueabihf": ["@yuuang/ffi-rs-linux-arm-gnueabihf@1.3.1", "", { "os": "linux", "cpu": "arm" }, "sha512-1WkcGkJTlwh4ZA59htKI+RXhiL3oKiYwLv7PO8LUf6FuADK73s5GcXp67iakKu243uYu+qGYr4RHco4ySddYhQ=="],
-
-    "@yuuang/ffi-rs-linux-arm64-gnu": ["@yuuang/ffi-rs-linux-arm64-gnu@1.3.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-J2PwqviycZxaEVA0Bwv38LqGDGSB9A1DPN4iYginYJZSvTvKW8kh7Tis0HbZrX1YDKnY8hi3lt0N0tCTNPDH5Q=="],
-
-    "@yuuang/ffi-rs-linux-arm64-musl": ["@yuuang/ffi-rs-linux-arm64-musl@1.3.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-Hn1W1hBPssTaqikU1Bqp1XUdDdOgbnYVIOtR++LVx66hhrtjf/xrIUQOhTm+NmOFDG16JUKXe1skfM4gpaqYwg=="],
-
-    "@yuuang/ffi-rs-linux-x64-gnu": ["@yuuang/ffi-rs-linux-x64-gnu@1.3.1", "", { "os": "linux", "cpu": "x64" }, "sha512-kW6e+oCYZPvpH2ppPsffA18e1aLowtmWTRjVlyHtY04g/nQDepQvDUkkcvInh9fW5jLna7PjHvktW1tVgYIj2A=="],
-
-    "@yuuang/ffi-rs-linux-x64-musl": ["@yuuang/ffi-rs-linux-x64-musl@1.3.1", "", { "os": "linux", "cpu": "x64" }, "sha512-HTwblAzruUS16nQPrez3ozvEHm1Xxh8J8w7rZYrpmAcNl1hzyOT8z/hY70M9Rt9fOqQ4Ovgor9qVy/U3ZJo0ZA=="],
-
-    "@yuuang/ffi-rs-win32-arm64-msvc": ["@yuuang/ffi-rs-win32-arm64-msvc@1.3.1", "", { "os": "win32", "cpu": "arm64" }, "sha512-WeZkGl2BP1U4tRhEQH+FXLQS52N8obp74smK5AAGOfzPAT1pHkq6+dVkC1QCSIt7dHJs7SPtlnQw+5DkdZYlWA=="],
-
-    "@yuuang/ffi-rs-win32-ia32-msvc": ["@yuuang/ffi-rs-win32-ia32-msvc@1.3.1", "", { "os": "win32", "cpu": [ "x64", "ia32", ] }, "sha512-rNGgMeCH5mdeHiMiJgt7wWXovZ+FHEfXhU9p4zZBH4n8M1/QnEsRUwlapISPLpILSGpoYS6iBuq9/fUlZY8Mhg=="],
-
-    "@yuuang/ffi-rs-win32-x64-msvc": ["@yuuang/ffi-rs-win32-x64-msvc@1.3.1", "", { "os": "win32", "cpu": "x64" }, "sha512-dr2LcLD2CXo2a7BktlOpV68QhayqiI112KxIJC9tBgQO/Dkdg4CPsdqmvzzLhFo64iC5RLl2BT7M5lJImrfUWw=="],
-
-    "bun": ["bun@1.3.10", "", { "optionalDependencies": { "@oven/bun-darwin-aarch64": "1.3.10", "@oven/bun-darwin-x64": "1.3.10", "@oven/bun-darwin-x64-baseline": "1.3.10", "@oven/bun-linux-aarch64": "1.3.10", "@oven/bun-linux-aarch64-musl": "1.3.10", "@oven/bun-linux-x64": "1.3.10", "@oven/bun-linux-x64-baseline": "1.3.10", "@oven/bun-linux-x64-musl": "1.3.10", "@oven/bun-linux-x64-musl-baseline": "1.3.10", "@oven/bun-windows-aarch64": "1.3.10", "@oven/bun-windows-x64": "1.3.10", "@oven/bun-windows-x64-baseline": "1.3.10" }, "os": [ "linux", "win32", "darwin", ], "cpu": [ "x64", "arm64", ], "bin": { "bun": "bin/bun.exe", "bunx": "bin/bunx.exe" } }, "sha512-S/CXaXXIyA4CMjdMkYQ4T2YMqnAn4s0ysD3mlsY4bUiOCqGlv28zck4Wd4H4kpvbekx15S9mUeLQ7Uxd0tYTLA=="],
-
-    "bun-types": ["bun-types@1.3.9", "", { "dependencies": { "@types/node": "*" } }, "sha512-+UBWWOakIP4Tswh0Bt0QD0alpTY8cb5hvgiYeWCMet9YukHbzuruIEeXC2D7nMJPB12kbh8C7XJykSexEqGKJg=="],
-
-    "ffi-rs": ["ffi-rs@1.3.1", "", { "optionalDependencies": { "@yuuang/ffi-rs-android-arm64": "1.3.1", "@yuuang/ffi-rs-darwin-arm64": "1.3.1", "@yuuang/ffi-rs-darwin-x64": "1.3.1", "@yuuang/ffi-rs-linux-arm-gnueabihf": "1.3.1", "@yuuang/ffi-rs-linux-arm64-gnu": "1.3.1", "@yuuang/ffi-rs-linux-arm64-musl": "1.3.1", "@yuuang/ffi-rs-linux-x64-gnu": "1.3.1", "@yuuang/ffi-rs-linux-x64-musl": "1.3.1", "@yuuang/ffi-rs-win32-arm64-msvc": "1.3.1", "@yuuang/ffi-rs-win32-ia32-msvc": "1.3.1", "@yuuang/ffi-rs-win32-x64-msvc": "1.3.1" } }, "sha512-ZyNXL9fnclnZV+waQmWB9JrfbIEyxQa1OWtMrHOrAgcC04PgP5hBMG5TdhVN8N4uT/eul8zCFMVnJUukAFFlXA=="],
-
-    "typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="],
-
-    "undici-types": ["undici-types@6.21.0", "", {}, "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ=="],
-
-    "bun-types/@types/node": ["@types/node@25.3.3", "", { "dependencies": { "undici-types": "~7.18.0" } }, "sha512-DpzbrH7wIcBaJibpKo9nnSQL0MTRdnWttGyE5haGwK86xgMOkFLp7vEyfQPGLOJh5wNYiJ3V9PmUMDhV9u8kkQ=="],
-
-    "bun-types/@types/node/undici-types": ["undici-types@7.18.2", "", {}, "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w=="],
-  }
-}

crates/fff-c/Cargo.toml

@@ -1,23 +1,24 @@
 [package]
 name = "fff-c"
-version = "0.4.2"
+version = "0.10.5"
 edition = "2024"
 description = "Raw C api of FFF file finder"
 license = "MIT"
 
+[lints]
+workspace = true
+
 [lib]
 crate-type = ["cdylib"]
 
 [features]
-default = []
-zlob = ["fff/zlob"]
+default = ["ripgrep"] # use ripgrep base crates to avoid requiring zig for rust crate
+ripgrep = ["fff/ripgrep", "fff-query-parser/ripgrep"]
+zlob = ["fff/zlob", "fff-query-parser/zlob"]
 
 [dependencies]
-mimalloc.workspace = true
-tracing.workspace = true
 git2.workspace = true
 
-fff = { package = "fff-search", path = "../fff-core" , version = "0.4.0" }
-fff-query-parser = { path = "../fff-query-parser" , version = "0.4.2" }
-serde = { version = "1.0", features = ["derive"] }
+fff = { package = "fff-search", path = "../fff-core", version = "0.10.5", default-features = false }
+fff-query-parser = { path = "../fff-query-parser", version = "0.10.5", default-features = false }
 serde_json = "1.0"

crates/fff-c/cbindgen.toml

@@ -26,3 +26,8 @@ include = [
 
 [fn]
 sort_by = "None"
+# Translate `#[deprecated]` on extern "C" fns into a real C compiler
+# attribute so callers get a warning when they use a removed/legacy entry.
+# `{}` is substituted with the Rust deprecation note as a C string literal
+# (already quoted) — do not wrap in extra quotes.
+deprecated_with_note = "__attribute__((deprecated({})))"

crates/fff-c/src/accessors.rs

@@ -0,0 +1,896 @@
+//! Stable accessor functions for `fff-c` FFI struct fields: a named API so
+//! FFI callers (Emacs Lisp, Python `ctypes`, etc.) don't hardcode struct byte
+//! offsets that break silently on layout changes. For array iteration use
+//! `fff_search_result_get_item` / `fff_grep_result_get_match` in `lib.rs`.
+
+use std::ffi::c_char;
+use std::ptr;
+
+use crate::ffi_types::{
+    FffFileItem, FffGrepMatch, FffGrepResult, FffMatchRange, FffResult, FffSearchResult,
+};
+
+// ── FffResult ────────────────────────────────────────────────────────────────
+
+/// Returns whether the operation completed successfully. Returns `false` if `result` is null.
+///
+/// ## Safety
+/// `result` must be a valid `FffResult` pointer or null.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_result_get_success(result: *const FffResult) -> bool {
+    if result.is_null() {
+        return false;
+    }
+    unsafe { (*result).success }
+}
+
+/// Returns the operation error message, or null when there is no error or `result` is null.
+///
+/// Do not free the returned pointer. It remains valid until `fff_free_result` is called.
+///
+/// ## Safety
+/// `result` must be a valid `FffResult` pointer or null.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_result_get_error(result: *const FffResult) -> *const c_char {
+    if result.is_null() {
+        return ptr::null();
+    }
+    unsafe { (*result).error }
+}
+
+/// Returns the result payload handle, or null if `result` is null.
+///
+/// ## Safety
+/// `result` must be a valid `FffResult` pointer or null.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_result_get_handle(result: *const FffResult) -> *mut std::ffi::c_void {
+    if result.is_null() {
+        return ptr::null_mut();
+    }
+    unsafe { (*result).handle }
+}
+
+/// Returns the result integer payload. Returns `0` if `result` is null.
+///
+/// ## Safety
+/// `result` must be a valid `FffResult` pointer or null.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_result_get_int_value(result: *const FffResult) -> i64 {
+    if result.is_null() {
+        return 0;
+    }
+    unsafe { (*result).int_value }
+}
+
+// ── FffFileItem ──────────────────────────────────────────────────────────────
+
+/// Relative path of a file item (e.g. `"src/main.rs"`); null if `item` is null. Do not free.
+///
+/// ## Safety
+/// `item` must be a valid `FffFileItem` pointer or null.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_file_item_get_relative_path(
+    item: *const FffFileItem,
+) -> *const c_char {
+    if item.is_null() {
+        return ptr::null();
+    }
+    unsafe { (*item).relative_path }
+}
+
+/// File-name component of a file item (e.g. `"main.rs"`); null if `item` is null. Do not free.
+///
+/// ## Safety
+/// `item` must be a valid `FffFileItem` pointer or null.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_file_item_get_file_name(item: *const FffFileItem) -> *const c_char {
+    if item.is_null() {
+        return ptr::null();
+    }
+    unsafe { (*item).file_name }
+}
+
+/// Git status string of a file item (e.g. `"M "`, `"??"`); null if git is unavailable,
+/// the file is untracked, or `item` is null. Do not free.
+///
+/// ## Safety
+/// `item` must be a valid `FffFileItem` pointer or null.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_file_item_get_git_status(item: *const FffFileItem) -> *const c_char {
+    if item.is_null() {
+        return ptr::null();
+    }
+    unsafe { (*item).git_status }
+}
+
+/// File size in bytes; `0` if `item` is null.
+///
+/// ## Safety
+/// `item` must be a valid `FffFileItem` pointer or null.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_file_item_get_size(item: *const FffFileItem) -> u64 {
+    if item.is_null() {
+        return 0;
+    }
+    unsafe { (*item).size }
+}
+
+/// Last-modified time as seconds since the UNIX epoch; `0` if `item` is null.
+///
+/// ## Safety
+/// `item` must be a valid `FffFileItem` pointer or null.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_file_item_get_modified(item: *const FffFileItem) -> u64 {
+    if item.is_null() {
+        return 0;
+    }
+    unsafe { (*item).modified }
+}
+
+/// Combined frecency score; `0` if `item` is null.
+///
+/// ## Safety
+/// `item` must be a valid `FffFileItem` pointer or null.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_file_item_get_total_frecency_score(item: *const FffFileItem) -> i64 {
+    if item.is_null() {
+        return 0;
+    }
+    unsafe { (*item).total_frecency_score }
+}
+
+/// Access-based frecency score; `0` if `item` is null.
+///
+/// ## Safety
+/// `item` must be a valid `FffFileItem` pointer or null.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_file_item_get_access_frecency_score(item: *const FffFileItem) -> i64 {
+    if item.is_null() {
+        return 0;
+    }
+    unsafe { (*item).access_frecency_score }
+}
+
+/// Modification-based frecency score; `0` if `item` is null.
+///
+/// ## Safety
+/// `item` must be a valid `FffFileItem` pointer or null.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_file_item_get_modification_frecency_score(
+    item: *const FffFileItem,
+) -> i64 {
+    if item.is_null() {
+        return 0;
+    }
+    unsafe { (*item).modification_frecency_score }
+}
+
+/// `true` if the file was detected as binary; `false` if `item` is null.
+///
+/// ## Safety
+/// `item` must be a valid `FffFileItem` pointer or null.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_file_item_get_is_binary(item: *const FffFileItem) -> bool {
+    if item.is_null() {
+        return false;
+    }
+    unsafe { (*item).is_binary }
+}
+
+// ── FffGrepMatch ─────────────────────────────────────────────────────────────
+
+/// Relative path of the file containing this grep match; null if `m` is null. Do not free.
+///
+/// ## Safety
+/// `m` must be a valid `FffGrepMatch` pointer or null.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_grep_match_get_relative_path(m: *const FffGrepMatch) -> *const c_char {
+    if m.is_null() {
+        return ptr::null();
+    }
+    unsafe { (*m).relative_path }
+}
+
+/// File-name component of the file containing this grep match; null if `m` is null. Do not free.
+///
+/// ## Safety
+/// `m` must be a valid `FffGrepMatch` pointer or null.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_grep_match_get_file_name(m: *const FffGrepMatch) -> *const c_char {
+    if m.is_null() {
+        return ptr::null();
+    }
+    unsafe { (*m).file_name }
+}
+
+/// Git status string of the matched file (e.g. `"M "`, `"??"`); null if git is unavailable,
+/// the file is untracked, or `m` is null. Do not free.
+///
+/// ## Safety
+/// `m` must be a valid `FffGrepMatch` pointer or null.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_grep_match_get_git_status(m: *const FffGrepMatch) -> *const c_char {
+    if m.is_null() {
+        return ptr::null();
+    }
+    unsafe { (*m).git_status }
+}
+
+/// Full text content of the matched line; null if `m` is null. Do not free.
+///
+/// ## Safety
+/// `m` must be a valid `FffGrepMatch` pointer or null.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_grep_match_get_line_content(m: *const FffGrepMatch) -> *const c_char {
+    if m.is_null() {
+        return ptr::null();
+    }
+    unsafe { (*m).line_content }
+}
+
+/// 1-based line number of the match within its file; `0` if `m` is null.
+///
+/// ## Safety
+/// `m` must be a valid `FffGrepMatch` pointer or null.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_grep_match_get_line_number(m: *const FffGrepMatch) -> u64 {
+    if m.is_null() {
+        return 0;
+    }
+    unsafe { (*m).line_number }
+}
+
+/// 0-based column of the match start within its line; `0` if `m` is null.
+///
+/// ## Safety
+/// `m` must be a valid `FffGrepMatch` pointer or null.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_grep_match_get_col(m: *const FffGrepMatch) -> u32 {
+    if m.is_null() {
+        return 0;
+    }
+    unsafe { (*m).col }
+}
+
+/// Byte offset of the match start from the beginning of the file; `0` if `m` is null.
+///
+/// ## Safety
+/// `m` must be a valid `FffGrepMatch` pointer or null.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_grep_match_get_byte_offset(m: *const FffGrepMatch) -> u64 {
+    if m.is_null() {
+        return 0;
+    }
+    unsafe { (*m).byte_offset }
+}
+
+/// File size in bytes of the matched file; `0` if `m` is null.
+///
+/// ## Safety
+/// `m` must be a valid `FffGrepMatch` pointer or null.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_grep_match_get_size(m: *const FffGrepMatch) -> u64 {
+    if m.is_null() {
+        return 0;
+    }
+    unsafe { (*m).size }
+}
+
+/// Combined frecency score of the matched file; `0` if `m` is null.
+///
+/// ## Safety
+/// `m` must be a valid `FffGrepMatch` pointer or null.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_grep_match_get_total_frecency_score(m: *const FffGrepMatch) -> i64 {
+    if m.is_null() {
+        return 0;
+    }
+    unsafe { (*m).total_frecency_score }
+}
+
+/// Access-based frecency score of the matched file; `0` if `m` is null.
+///
+/// ## Safety
+/// `m` must be a valid `FffGrepMatch` pointer or null.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_grep_match_get_access_frecency_score(m: *const FffGrepMatch) -> i64 {
+    if m.is_null() {
+        return 0;
+    }
+    unsafe { (*m).access_frecency_score }
+}
+
+/// Modification-based frecency score of the matched file; `0` if `m` is null.
+///
+/// ## Safety
+/// `m` must be a valid `FffGrepMatch` pointer or null.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_grep_match_get_modification_frecency_score(
+    m: *const FffGrepMatch,
+) -> i64 {
+    if m.is_null() {
+        return 0;
+    }
+    unsafe { (*m).modification_frecency_score }
+}
+
+/// Last-modified time of the matched file as seconds since the UNIX epoch; `0` if `m` is null.
+///
+/// ## Safety
+/// `m` must be a valid `FffGrepMatch` pointer or null.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_grep_match_get_modified(m: *const FffGrepMatch) -> u64 {
+    if m.is_null() {
+        return 0;
+    }
+    unsafe { (*m).modified }
+}
+
+/// Number of highlight ranges in this match; `0` if `m` is null.
+/// Use with [`fff_grep_match_get_match_range`] to iterate the highlight spans.
+///
+/// ## Safety
+/// `m` must be a valid `FffGrepMatch` pointer or null.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_grep_match_get_match_ranges_count(m: *const FffGrepMatch) -> u32 {
+    if m.is_null() {
+        return 0;
+    }
+    unsafe { (*m).match_ranges_count }
+}
+
+/// Pointer to the `index`-th [`FffMatchRange`] highlight span; null if `m` is null,
+/// `index >= match_ranges_count`, or the ranges array is null. Valid until the owning `FffGrepResult` is freed; do not free.
+///
+/// ## Safety
+/// `m` must be a valid `FffGrepMatch` pointer or null.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_grep_match_get_match_range(
+    m: *const FffGrepMatch,
+    index: u32,
+) -> *const FffMatchRange {
+    if m.is_null() {
+        return ptr::null();
+    }
+    let m = unsafe { &*m };
+    if index >= m.match_ranges_count || m.match_ranges.is_null() {
+        return ptr::null();
+    }
+    unsafe { m.match_ranges.add(index as usize) }
+}
+
+/// Number of context lines captured before the match; `0` if `m` is null.
+/// Use with [`fff_grep_match_get_context_before`] to read each line.
+///
+/// ## Safety
+/// `m` must be a valid `FffGrepMatch` pointer or null.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_grep_match_get_context_before_count(m: *const FffGrepMatch) -> u32 {
+    if m.is_null() {
+        return 0;
+    }
+    unsafe { (*m).context_before_count }
+}
+
+/// The `index`-th context line before the match; null if `m` is null,
+/// `index >= context_before_count`, or the context array is null. Do not free.
+///
+/// ## Safety
+/// `m` must be a valid `FffGrepMatch` pointer or null.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_grep_match_get_context_before(
+    m: *const FffGrepMatch,
+    index: u32,
+) -> *const c_char {
+    if m.is_null() {
+        return ptr::null();
+    }
+    let m = unsafe { &*m };
+    if index >= m.context_before_count || m.context_before.is_null() {
+        return ptr::null();
+    }
+    unsafe { *m.context_before.add(index as usize) }
+}
+
+/// Number of context lines captured after the match; `0` if `m` is null.
+/// Use with [`fff_grep_match_get_context_after`] to read each line.
+///
+/// ## Safety
+/// `m` must be a valid `FffGrepMatch` pointer or null.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_grep_match_get_context_after_count(m: *const FffGrepMatch) -> u32 {
+    if m.is_null() {
+        return 0;
+    }
+    unsafe { (*m).context_after_count }
+}
+
+/// The `index`-th context line after the match; null if `m` is null,
+/// `index >= context_after_count`, or the context array is null. Do not free.
+///
+/// ## Safety
+/// `m` must be a valid `FffGrepMatch` pointer or null.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_grep_match_get_context_after(
+    m: *const FffGrepMatch,
+    index: u32,
+) -> *const c_char {
+    if m.is_null() {
+        return ptr::null();
+    }
+    let m = unsafe { &*m };
+    if index >= m.context_after_count || m.context_after.is_null() {
+        return ptr::null();
+    }
+    unsafe { *m.context_after.add(index as usize) }
+}
+
+/// Fuzzy match score; `0` if `m` is null or no fuzzy score is present.
+/// Always check [`fff_grep_match_get_has_fuzzy_score`] first; `0` is ambiguous without that flag.
+///
+/// ## Safety
+/// `m` must be a valid `FffGrepMatch` pointer or null.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_grep_match_get_fuzzy_score(m: *const FffGrepMatch) -> u16 {
+    if m.is_null() {
+        return 0;
+    }
+    unsafe { (*m).fuzzy_score }
+}
+
+/// `true` if this match carries a valid fuzzy score; `false` if `m` is null.
+///
+/// ## Safety
+/// `m` must be a valid `FffGrepMatch` pointer or null.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_grep_match_get_has_fuzzy_score(m: *const FffGrepMatch) -> bool {
+    if m.is_null() {
+        return false;
+    }
+    unsafe { (*m).has_fuzzy_score }
+}
+
+/// `true` if the match was identified as a symbol definition; `false` if `m` is null.
+///
+/// ## Safety
+/// `m` must be a valid `FffGrepMatch` pointer or null.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_grep_match_get_is_definition(m: *const FffGrepMatch) -> bool {
+    if m.is_null() {
+        return false;
+    }
+    unsafe { (*m).is_definition }
+}
+
+/// `true` if the matched file was detected as binary; `false` if `m` is null.
+///
+/// ## Safety
+/// `m` must be a valid `FffGrepMatch` pointer or null.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_grep_match_get_is_binary(m: *const FffGrepMatch) -> bool {
+    if m.is_null() {
+        return false;
+    }
+    unsafe { (*m).is_binary }
+}
+
+// ── FffSearchResult ──────────────────────────────────────────────────────────
+
+/// Number of items in the result; `0` if `r` is null.
+///
+/// ## Safety
+/// `r` must be a valid `FffSearchResult` pointer or null.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_search_result_get_count(r: *const FffSearchResult) -> u32 {
+    if r.is_null() {
+        return 0;
+    }
+    unsafe { (*r).count }
+}
+
+/// Total number of files that matched before truncation to the page size; `0` if `r` is null.
+///
+/// ## Safety
+/// `r` must be a valid `FffSearchResult` pointer or null.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_search_result_get_total_matched(r: *const FffSearchResult) -> u32 {
+    if r.is_null() {
+        return 0;
+    }
+    unsafe { (*r).total_matched }
+}
+
+/// Total number of indexed files considered during search; `0` if `r` is null.
+///
+/// ## Safety
+/// `r` must be a valid `FffSearchResult` pointer or null.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_search_result_get_total_files(r: *const FffSearchResult) -> u32 {
+    if r.is_null() {
+        return 0;
+    }
+    unsafe { (*r).total_files }
+}
+
+// ── FffGrepResult ─────────────────────────────────────────────────────────────
+
+/// Number of matches in the result; `0` if `r` is null.
+///
+/// ## Safety
+/// `r` must be a valid `FffGrepResult` pointer or null.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_grep_result_get_count(r: *const FffGrepResult) -> u32 {
+    if r.is_null() {
+        return 0;
+    }
+    unsafe { (*r).count }
+}
+
+/// Total number of matches found across all pages; `0` if `r` is null.
+///
+/// ## Safety
+/// `r` must be a valid `FffGrepResult` pointer or null.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_grep_result_get_total_matched(r: *const FffGrepResult) -> u32 {
+    if r.is_null() {
+        return 0;
+    }
+    unsafe { (*r).total_matched }
+}
+
+/// Number of files actually opened and searched in this call; `0` if `r` is null.
+///
+/// ## Safety
+/// `r` must be a valid `FffGrepResult` pointer or null.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_grep_result_get_total_files_searched(r: *const FffGrepResult) -> u32 {
+    if r.is_null() {
+        return 0;
+    }
+    unsafe { (*r).total_files_searched }
+}
+
+/// Total number of indexed files before any filtering; `0` if `r` is null.
+///
+/// ## Safety
+/// `r` must be a valid `FffGrepResult` pointer or null.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_grep_result_get_total_files(r: *const FffGrepResult) -> u32 {
+    if r.is_null() {
+        return 0;
+    }
+    unsafe { (*r).total_files }
+}
+
+/// Number of files eligible for search after path/type filtering; `0` if `r` is null.
+///
+/// ## Safety
+/// `r` must be a valid `FffGrepResult` pointer or null.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_grep_result_get_filtered_file_count(r: *const FffGrepResult) -> u32 {
+    if r.is_null() {
+        return 0;
+    }
+    unsafe { (*r).filtered_file_count }
+}
+
+/// File offset for the next page; `0` if all files have been searched or `r` is null.
+/// Pass as `file_offset` to a subsequent `fff_live_grep`/`fff_multi_grep` call to continue pagination.
+///
+/// ## Safety
+/// `r` must be a valid `FffGrepResult` pointer or null.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_grep_result_get_next_file_offset(r: *const FffGrepResult) -> u32 {
+    if r.is_null() {
+        return 0;
+    }
+    unsafe { (*r).next_file_offset }
+}
+
+/// Regex compilation error string if the engine fell back to literal matching;
+/// null if there was no error or `r` is null. Do not free.
+///
+/// ## Safety
+/// `r` must be a valid `FffGrepResult` pointer or null.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_grep_result_get_regex_fallback_error(
+    r: *const FffGrepResult,
+) -> *const c_char {
+    if r.is_null() {
+        return ptr::null();
+    }
+    unsafe { (*r).regex_fallback_error }
+}
+
+// ── Tests ─────────────────────────────────────────────────────────────────────
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::ffi::CString;
+    use std::ptr;
+
+    // ── helpers ──────────────────────────────────────────────────────────────
+
+    fn make_file_item(path: &str, name: &str) -> FffFileItem {
+        FffFileItem {
+            relative_path: CString::new(path).unwrap().into_raw(),
+            file_name: CString::new(name).unwrap().into_raw(),
+            git_status: ptr::null_mut(),
+            size: 1024,
+            modified: 1_700_000_000,
+            access_frecency_score: 10,
+            modification_frecency_score: 20,
+            total_frecency_score: 30,
+            is_binary: false,
+        }
+    }
+
+    unsafe fn free_file_item(item: &mut FffFileItem) {
+        unsafe {
+            if !item.relative_path.is_null() {
+                drop(CString::from_raw(item.relative_path));
+            }
+            if !item.file_name.is_null() {
+                drop(CString::from_raw(item.file_name));
+            }
+            if !item.git_status.is_null() {
+                drop(CString::from_raw(item.git_status));
+            }
+        }
+    }
+
+    fn make_grep_match(path: &str, line: &str) -> FffGrepMatch {
+        FffGrepMatch {
+            relative_path: CString::new(path).unwrap().into_raw(),
+            file_name: CString::new("file.rs").unwrap().into_raw(),
+            git_status: ptr::null_mut(),
+            line_content: CString::new(line).unwrap().into_raw(),
+            match_ranges: ptr::null_mut(),
+            context_before: ptr::null_mut(),
+            context_after: ptr::null_mut(),
+            size: 512,
+            modified: 1_600_000_000,
+            total_frecency_score: 5,
+            access_frecency_score: 6,
+            modification_frecency_score: 7,
+            line_number: 42,
+            byte_offset: 100,
+            col: 8,
+            match_ranges_count: 0,
+            context_before_count: 0,
+            context_after_count: 0,
+            fuzzy_score: 0,
+            has_fuzzy_score: false,
+            is_binary: false,
+            is_definition: true,
+        }
+    }
+
+    unsafe fn free_grep_match(m: &mut FffGrepMatch) {
+        unsafe {
+            if !m.relative_path.is_null() {
+                drop(CString::from_raw(m.relative_path));
+            }
+            if !m.file_name.is_null() {
+                drop(CString::from_raw(m.file_name));
+            }
+            if !m.line_content.is_null() {
+                drop(CString::from_raw(m.line_content));
+            }
+        }
+    }
+
+    fn make_search_result(count: u32, total: u32, files: u32) -> FffSearchResult {
+        FffSearchResult {
+            items: ptr::null_mut(),
+            scores: ptr::null_mut(),
+            count,
+            total_matched: total,
+            total_files: files,
+            location: crate::ffi_types::FffLocation {
+                tag: 0,
+                line: 0,
+                col: 0,
+                end_line: 0,
+                end_col: 0,
+            },
+        }
+    }
+
+    fn make_grep_result() -> FffGrepResult {
+        FffGrepResult {
+            items: ptr::null_mut(),
+            count: 3,
+            total_matched: 10,
+            total_files_searched: 50,
+            total_files: 200,
+            filtered_file_count: 80,
+            next_file_offset: 51,
+            regex_fallback_error: ptr::null_mut(),
+        }
+    }
+
+    // ── null-guard tests: every function returns its zero-value on NULL ───────
+
+    #[test]
+    fn null_file_item_returns_null_or_zero() {
+        let null: *const FffFileItem = ptr::null();
+        unsafe {
+            assert!(fff_file_item_get_relative_path(null).is_null());
+            assert!(fff_file_item_get_file_name(null).is_null());
+            assert!(fff_file_item_get_git_status(null).is_null());
+            assert_eq!(fff_file_item_get_size(null), 0);
+            assert_eq!(fff_file_item_get_modified(null), 0);
+            assert_eq!(fff_file_item_get_access_frecency_score(null), 0);
+            assert_eq!(fff_file_item_get_modification_frecency_score(null), 0);
+            assert_eq!(fff_file_item_get_total_frecency_score(null), 0);
+            assert!(!fff_file_item_get_is_binary(null));
+        }
+    }
+
+    #[test]
+    fn null_grep_match_returns_null_or_zero() {
+        let null: *const FffGrepMatch = ptr::null();
+        unsafe {
+            assert!(fff_grep_match_get_relative_path(null).is_null());
+            assert!(fff_grep_match_get_file_name(null).is_null());
+            assert!(fff_grep_match_get_git_status(null).is_null());
+            assert!(fff_grep_match_get_line_content(null).is_null());
+            assert_eq!(fff_grep_match_get_line_number(null), 0);
+            assert_eq!(fff_grep_match_get_byte_offset(null), 0);
+            assert_eq!(fff_grep_match_get_col(null), 0);
+            assert_eq!(fff_grep_match_get_size(null), 0);
+            assert_eq!(fff_grep_match_get_modified(null), 0);
+            assert_eq!(fff_grep_match_get_total_frecency_score(null), 0);
+            assert_eq!(fff_grep_match_get_access_frecency_score(null), 0);
+            assert_eq!(fff_grep_match_get_modification_frecency_score(null), 0);
+            assert_eq!(fff_grep_match_get_match_ranges_count(null), 0);
+            assert_eq!(fff_grep_match_get_context_before_count(null), 0);
+            assert_eq!(fff_grep_match_get_context_after_count(null), 0);
+            assert!(!fff_grep_match_get_has_fuzzy_score(null));
+            assert_eq!(fff_grep_match_get_fuzzy_score(null), 0);
+            assert!(!fff_grep_match_get_is_binary(null));
+            assert!(!fff_grep_match_get_is_definition(null));
+            assert!(fff_grep_match_get_context_before(null, 0).is_null());
+            assert!(fff_grep_match_get_context_after(null, 0).is_null());
+            assert!(fff_grep_match_get_match_range(null, 0).is_null());
+        }
+    }
+
+    #[test]
+    fn null_search_result_returns_zero() {
+        let null: *const FffSearchResult = ptr::null();
+        unsafe {
+            assert_eq!(fff_search_result_get_count(null), 0);
+            assert_eq!(fff_search_result_get_total_matched(null), 0);
+            assert_eq!(fff_search_result_get_total_files(null), 0);
+        }
+    }
+
+    #[test]
+    fn null_grep_result_returns_zero_or_null() {
+        let null: *const FffGrepResult = ptr::null();
+        unsafe {
+            assert_eq!(fff_grep_result_get_count(null), 0);
+            assert_eq!(fff_grep_result_get_total_matched(null), 0);
+            assert_eq!(fff_grep_result_get_total_files_searched(null), 0);
+            assert_eq!(fff_grep_result_get_total_files(null), 0);
+            assert_eq!(fff_grep_result_get_filtered_file_count(null), 0);
+            assert_eq!(fff_grep_result_get_next_file_offset(null), 0);
+            assert!(fff_grep_result_get_regex_fallback_error(null).is_null());
+        }
+    }
+
+    #[test]
+    fn null_result_returns_zero_or_null() {
+        let null: *const FffResult = ptr::null();
+        unsafe {
+            assert!(!fff_result_get_success(null));
+            assert!(fff_result_get_error(null).is_null());
+            assert!(fff_result_get_handle(null).is_null());
+            assert_eq!(fff_result_get_int_value(null), 0);
+        }
+    }
+
+    // ── data correctness tests ────────────────────────────────────────────────
+
+    #[test]
+    fn result_getters_return_correct_values() {
+        let error = CString::new("failed").unwrap();
+        let handle = 0x1234usize as *mut std::ffi::c_void;
+        let result = FffResult {
+            success: false,
+            error: error.as_ptr() as *mut std::ffi::c_char,
+            handle,
+            int_value: -7,
+        };
+        let p = &result as *const FffResult;
+        unsafe {
+            assert!(!fff_result_get_success(p));
+            assert_eq!(
+                std::ffi::CStr::from_ptr(fff_result_get_error(p)),
+                error.as_c_str()
+            );
+            assert_eq!(fff_result_get_handle(p), handle);
+            assert_eq!(fff_result_get_int_value(p), -7);
+        }
+    }
+
+    #[test]
+    fn file_item_getters_return_correct_values() {
+        let mut item = make_file_item("src/main.rs", "main.rs");
+        let p = &item as *const FffFileItem;
+        unsafe {
+            let path = std::ffi::CStr::from_ptr(fff_file_item_get_relative_path(p));
+            assert_eq!(path.to_str().unwrap(), "src/main.rs");
+
+            let name = std::ffi::CStr::from_ptr(fff_file_item_get_file_name(p));
+            assert_eq!(name.to_str().unwrap(), "main.rs");
+
+            assert!(fff_file_item_get_git_status(p).is_null());
+            assert_eq!(fff_file_item_get_size(p), 1024);
+            assert_eq!(fff_file_item_get_modified(p), 1_700_000_000);
+            assert_eq!(fff_file_item_get_access_frecency_score(p), 10);
+            assert_eq!(fff_file_item_get_modification_frecency_score(p), 20);
+            assert_eq!(fff_file_item_get_total_frecency_score(p), 30);
+            assert!(!fff_file_item_get_is_binary(p));
+
+            free_file_item(&mut item);
+        }
+    }
+
+    #[test]
+    fn grep_match_getters_return_correct_values() {
+        let mut m = make_grep_match("src/lib.rs", "fn hello()");
+        let p = &m as *const FffGrepMatch;
+        unsafe {
+            let path = std::ffi::CStr::from_ptr(fff_grep_match_get_relative_path(p));
+            assert_eq!(path.to_str().unwrap(), "src/lib.rs");
+
+            let line = std::ffi::CStr::from_ptr(fff_grep_match_get_line_content(p));
+            assert_eq!(line.to_str().unwrap(), "fn hello()");
+
+            assert_eq!(fff_grep_match_get_line_number(p), 42);
+            assert_eq!(fff_grep_match_get_byte_offset(p), 100);
+            assert_eq!(fff_grep_match_get_col(p), 8);
+            assert_eq!(fff_grep_match_get_size(p), 512);
+            assert_eq!(fff_grep_match_get_modified(p), 1_600_000_000);
+            assert_eq!(fff_grep_match_get_total_frecency_score(p), 5);
+            assert_eq!(fff_grep_match_get_access_frecency_score(p), 6);
+            assert_eq!(fff_grep_match_get_modification_frecency_score(p), 7);
+            assert_eq!(fff_grep_match_get_match_ranges_count(p), 0);
+            assert!(!fff_grep_match_get_has_fuzzy_score(p));
+            assert!(!fff_grep_match_get_is_binary(p));
+            assert!(fff_grep_match_get_is_definition(p));
+
+            free_grep_match(&mut m);
+        }
+    }
+
+    #[test]
+    fn search_result_getters_return_correct_values() {
+        let r = make_search_result(5, 20, 100);
+        let p = &r as *const FffSearchResult;
+        unsafe {
+            assert_eq!(fff_search_result_get_count(p), 5);
+            assert_eq!(fff_search_result_get_total_matched(p), 20);
+            assert_eq!(fff_search_result_get_total_files(p), 100);
+        }
+    }
+
+    #[test]
+    fn grep_result_getters_return_correct_values() {
+        let r = make_grep_result();
+        let p = &r as *const FffGrepResult;
+        unsafe {
+            assert_eq!(fff_grep_result_get_count(p), 3);
+            assert_eq!(fff_grep_result_get_total_matched(p), 10);
+            assert_eq!(fff_grep_result_get_total_files_searched(p), 50);
+            assert_eq!(fff_grep_result_get_total_files(p), 200);
+            assert_eq!(fff_grep_result_get_filtered_file_count(p), 80);
+            assert_eq!(fff_grep_result_get_next_file_offset(p), 51);
+            assert!(fff_grep_result_get_regex_fallback_error(p).is_null());
+        }
+    }
+}

crates/fff-c/src/ffi_types.rs

@@ -1,18 +1,88 @@
-//! FFI-compatible type definitions
-//!
-//! All result types use `#[repr(C)]` structs for direct memory access from any
-//! language with C FFI support. No JSON serialization is used for search or grep
-//! results — callers read struct fields directly.
+//! FFI-compatible type definitions: all result types are `#[repr(C)]` structs
+//! read directly from any language with C FFI — no JSON serialization.
 
 use std::ffi::{CString, c_char, c_void};
 use std::ptr;
 
+use fff::file_picker::FilePicker;
 use fff::git::format_git_status;
-use fff::{FileItem, GrepMatch, GrepResult, Location, Score, SearchResult};
+use fff::{
+    DirItem, DirSearchResult, FileItem, GrepMatch, GrepResult, Location, MixedItemRef,
+    MixedSearchResult, Score, SearchResult,
+};
 
-// ---------------------------------------------------------------------------
-// Helpers
-// ---------------------------------------------------------------------------
+/// Current used version of [`FffCreateOptions`].
+pub const FFF_CREATE_OPTIONS_VERSION: u32 = 2;
+
+/// Options for `fff_create_instance_with`.
+///
+/// Versioned struct: the layout is stable across releases, new fields are
+/// only appended.
+#[repr(C)]
+pub struct FffCreateOptions {
+    /// Set to [`FFF_CREATE_OPTIONS_VERSION`] when allocating; tells the
+    /// library which trailing fields are populated.
+    pub version: u32,
+    /// Directory to index (required, non-NULL).
+    pub base_path: *const c_char,
+    /// Frecency LMDB database path. NULL/empty to skip frecency tracking.
+    pub frecency_db_path: *const c_char,
+    /// Query history LMDB database path. NULL/empty to skip query tracking.
+    pub history_db_path: *const c_char,
+    /// Pre-populate mmap caches for top-frecency files after the initial scan.
+    pub enable_mmap_cache: bool,
+    /// Build content index after the initial scan for faster grep.
+    pub enable_content_indexing: bool,
+    /// Start a background file-system watcher for live updates.
+    pub watch: bool,
+    /// Enable AI-agent optimizations.
+    pub ai_mode: bool,
+    /// Tracing log file path. NULL/empty to skip log init.
+    pub log_file_path: *const c_char,
+    /// Log level: `"trace" | "debug" | "info" | "warn" | "error"`.
+    /// NULL/empty defaults to `"info"`. Ignored when `log_file_path` is unset.
+    pub log_level: *const c_char,
+    /// Content cache file-count cap. 0 = auto.
+    pub cache_budget_max_files: u64,
+    /// Content cache byte cap. 0 = auto.
+    pub cache_budget_max_bytes: u64,
+    /// Per-file byte cap inside the content cache. 0 = auto.
+    pub cache_budget_max_file_size: u64,
+    /// Allow indexing the filesystem root (`/`). Off by default: root is rarely
+    /// intended and floods the watcher with churn.
+    pub enable_fs_root_scanning: bool,
+    /// Allow indexing the user's home directory. Same trade-off as `enable_fs_root_scanning`.
+    pub enable_home_dir_scanning: bool,
+    // ----- v2 fields -----
+    /// Follow symlinks during scan and watcher walks. Off by default: without
+    /// external loop protection cyclic symlinks can wedge the watcher.
+    pub follow_symlinks: bool,
+    // ----- new version 3+ fields go here, ALWAYS appended -----
+}
+
+impl FffCreateOptions {
+    /// Default values for a v1 options struct.
+    pub fn defaults() -> Self {
+        Self {
+            version: FFF_CREATE_OPTIONS_VERSION,
+            base_path: ptr::null(),
+            frecency_db_path: ptr::null(),
+            history_db_path: ptr::null(),
+            enable_mmap_cache: true,
+            enable_content_indexing: true,
+            watch: true,
+            ai_mode: false,
+            log_file_path: ptr::null(),
+            log_level: ptr::null(),
+            cache_budget_max_files: 0,
+            cache_budget_max_bytes: 0,
+            cache_budget_max_file_size: 0,
+            enable_fs_root_scanning: false,
+            enable_home_dir_scanning: false,
+            follow_symlinks: false,
+        }
+    }
+}
 
 /// Allocate a heap CString from a `&str`, returning a raw pointer.
 fn cstring_new(s: &str) -> *mut c_char {
@@ -58,13 +128,10 @@ unsafe fn free_cstring_array(arr: *mut *mut c_char, count: u32) {
     }
 }
 
-/// A file item returned by `fff_search`.
-///
-/// All string fields are heap-allocated and owned by the parent `FffSearchResult`.
-/// Free the entire result with `fff_free_search_result`.
+/// A file item returned by `fff_search`. Strings are owned by the parent
+/// `FffSearchResult`; free everything with `fff_free_search_result`.
 #[repr(C)]
 pub struct FffFileItem {
-    pub path: *mut c_char,
     pub relative_path: *mut c_char,
     pub file_name: *mut c_char,
     pub git_status: *mut c_char,
@@ -76,19 +143,18 @@ pub struct FffFileItem {
     pub is_binary: bool,
 }
 
-impl From<&FileItem> for FffFileItem {
-    fn from(item: &FileItem) -> Self {
+impl FffFileItem {
+    pub fn from_item(item: &FileItem, picker: &FilePicker) -> Self {
         FffFileItem {
-            path: cstring_new(&item.path.to_string_lossy()),
-            relative_path: cstring_new(&item.relative_path),
-            file_name: cstring_new(&item.file_name),
+            relative_path: cstring_new(&item.relative_path(picker)),
+            file_name: cstring_new(&item.file_name(picker)),
             git_status: cstring_new(format_git_status(item.git_status)),
             size: item.size,
             modified: item.modified,
-            access_frecency_score: item.access_frecency_score,
-            modification_frecency_score: item.modification_frecency_score,
-            total_frecency_score: item.total_frecency_score,
-            is_binary: item.is_binary,
+            access_frecency_score: item.access_frecency_score as i64,
+            modification_frecency_score: item.modification_frecency_score as i64,
+            total_frecency_score: item.total_frecency_score() as i64,
+            is_binary: item.is_binary(),
         }
     }
 }
@@ -98,9 +164,6 @@ impl FffFileItem {
     /// All string pointers must have been allocated by `CString::into_raw`.
     pub unsafe fn free_strings(&mut self) {
         unsafe {
-            if !self.path.is_null() {
-                drop(CString::from_raw(self.path));
-            }
             if !self.relative_path.is_null() {
                 drop(CString::from_raw(self.relative_path));
             }
@@ -125,6 +188,7 @@ pub struct FffScore {
     pub distance_penalty: i32,
     pub current_file_penalty: i32,
     pub combo_match_boost: i32,
+    pub path_alignment_bonus: i32,
     pub exact_match: bool,
     pub match_type: *mut c_char,
 }
@@ -140,6 +204,7 @@ impl From<&Score> for FffScore {
             distance_penalty: score.distance_penalty,
             current_file_penalty: score.current_file_penalty,
             combo_match_boost: score.combo_match_boost,
+            path_alignment_bonus: score.path_alignment_bonus,
             exact_match: score.exact_match,
             match_type: cstring_new(score.match_type),
         }
@@ -158,13 +223,9 @@ impl FffScore {
     }
 }
 
-/// Location parsed from a query string (e.g. `"file.ts:42:10"`).
-///
-/// `tag` encodes the variant:
-///   0 = no location,
-///   1 = line only (`line` is set),
-///   2 = position (`line` + `col`),
-///   3 = range (`line`/`col` = start, `end_line`/`end_col` = end).
+/// Location parsed from a query string (e.g. `"file.ts:42:10"`). `tag`:
+/// 0 = none, 1 = line, 2 = position (`line` + `col`),
+/// 3 = range (`line`/`col` = start, `end_line`/`end_col` = end).
 #[repr(C)]
 pub struct FffLocation {
     pub tag: u8,
@@ -209,14 +270,12 @@ impl From<Option<&Location>> for FffLocation {
     }
 }
 
-/// Search result returned by `fff_search`.
-///
-/// The caller must free this with `fff_free_search_result`.
+/// Search result returned by `fff_search`; free with `fff_free_search_result`.
 #[repr(C)]
 pub struct FffSearchResult {
-    /// Pointer to a heap-allocated array of `FffFileItem` (length = `count`).
+    /// Heap array of `FffFileItem` (length = `count`).
     pub items: *mut FffFileItem,
-    /// Pointer to a heap-allocated array of `FffScore` (length = `count`).
+    /// Heap array of `FffScore` (length = `count`).
     pub scores: *mut FffScore,
     /// Number of items/scores in the arrays.
     pub count: u32,
@@ -230,8 +289,12 @@ pub struct FffSearchResult {
 
 impl FffSearchResult {
     /// Convert a core `SearchResult` into a heap-allocated `FffSearchResult`.
-    pub fn from_core(result: &SearchResult) -> *mut Self {
-        let items: Vec<FffFileItem> = result.items.iter().map(|i| FffFileItem::from(*i)).collect();
+    pub fn from_core(result: &SearchResult, picker: &FilePicker) -> *mut Self {
+        let items: Vec<FffFileItem> = result
+            .items
+            .iter()
+            .map(|i| FffFileItem::from_item(i, picker))
+            .collect();
         let scores: Vec<FffScore> = result.scores.iter().map(FffScore::from).collect();
         let count = items.len() as u32;
 
@@ -260,14 +323,11 @@ pub struct FffMatchRange {
     pub end: u32,
 }
 
-/// A single grep match with file and line information.
-///
-/// All string fields and arrays are heap-allocated. Free the parent
-/// `FffGrepResult` with `fff_free_grep_result` to release everything.
+/// A single grep match with file and line information. Strings and arrays are
+/// owned by the parent `FffGrepResult`; free everything with `fff_free_grep_result`.
 #[repr(C)]
 pub struct FffGrepMatch {
     // -- pointers (8 bytes each) --
-    pub path: *mut c_char,
     pub relative_path: *mut c_char,
     pub file_name: *mut c_char,
     pub git_status: *mut c_char,
@@ -297,7 +357,7 @@ pub struct FffGrepMatch {
 }
 
 impl FffGrepMatch {
-    fn from_core_with_file(m: &GrepMatch, file: &FileItem) -> Self {
+    fn from_core_with_file(m: &GrepMatch, file: &FileItem, picker: &FilePicker) -> Self {
         let ranges: Vec<FffMatchRange> = m
             .match_byte_offsets
             .iter()
@@ -312,9 +372,8 @@ impl FffGrepMatch {
         };
 
         FffGrepMatch {
-            path: cstring_new(&file.path.to_string_lossy()),
-            relative_path: cstring_new(&file.relative_path),
-            file_name: cstring_new(&file.file_name),
+            relative_path: cstring_new(&file.relative_path(picker)),
+            file_name: cstring_new(&file.file_name(picker)),
             git_status: cstring_new(format_git_status(file.git_status)),
             line_content: cstring_new(&m.line_content),
             match_ranges,
@@ -322,9 +381,9 @@ impl FffGrepMatch {
             context_after,
             size: file.size,
             modified: file.modified,
-            total_frecency_score: file.total_frecency_score,
-            access_frecency_score: file.access_frecency_score,
-            modification_frecency_score: file.modification_frecency_score,
+            total_frecency_score: file.total_frecency_score() as i64,
+            access_frecency_score: file.access_frecency_score as i64,
+            modification_frecency_score: file.modification_frecency_score as i64,
             line_number: m.line_number,
             byte_offset: m.byte_offset,
             col: m.col as u32,
@@ -333,7 +392,7 @@ impl FffGrepMatch {
             context_after_count,
             fuzzy_score,
             has_fuzzy_score,
-            is_binary: file.is_binary,
+            is_binary: file.is_binary(),
             is_definition: m.is_definition,
         }
     }
@@ -342,9 +401,6 @@ impl FffGrepMatch {
     /// All pointers must have been allocated by the corresponding `from_core`.
     pub unsafe fn free_fields(&mut self) {
         unsafe {
-            if !self.path.is_null() {
-                drop(CString::from_raw(self.path));
-            }
             if !self.relative_path.is_null() {
                 drop(CString::from_raw(self.relative_path));
             }
@@ -370,12 +426,11 @@ impl FffGrepMatch {
     }
 }
 
-/// Grep result returned by `fff_live_grep` and `fff_multi_grep`.
-///
-/// The caller must free this with `fff_free_grep_result`.
+/// Grep result returned by `fff_live_grep` and `fff_multi_grep`;
+/// free with `fff_free_grep_result`.
 #[repr(C)]
 pub struct FffGrepResult {
-    /// Pointer to a heap-allocated array of `FffGrepMatch` (length = `count`).
+    /// Heap array of `FffGrepMatch` (length = `count`).
     pub items: *mut FffGrepMatch,
     /// Number of matches in the `items` array.
     pub count: u32,
@@ -395,13 +450,13 @@ pub struct FffGrepResult {
 
 impl FffGrepResult {
     /// Convert a core `GrepResult` into a heap-allocated `FffGrepResult`.
-    pub fn from_core(result: &GrepResult) -> *mut Self {
+    pub fn from_core(result: &GrepResult, picker: &FilePicker) -> *mut Self {
         let items: Vec<FffGrepMatch> = result
             .matches
             .iter()
             .map(|m| {
                 let file = result.files[m.file_index];
-                FffGrepMatch::from_core_with_file(m, file)
+                FffGrepMatch::from_core_with_file(m, file, picker)
             })
             .collect();
         let (items_ptr, count) = vec_to_raw(items);
@@ -424,7 +479,9 @@ impl FffGrepResult {
 
 /// Result envelope returned by all `fff_*` functions.
 ///
-/// Heap-allocated — the caller must free it with `fff_free_result`.
+/// Heap-allocated. The caller must free it with `fff_free_result`. Calling `fff_free_result`
+/// **does not** deallocate the underlying `handle` pointer. It needs to be cleaned separately.
+/// see (`fff_destroy`, `fff_free_search_result`, `fff_free_grep_result`, `fff_free_string`, etc.).
 ///
 /// Depending on the function, the payload is delivered through different fields:
 ///
@@ -444,18 +501,13 @@ impl FffGrepResult {
 /// | `fff_restart_index`        | (none)        | success flag only             |
 ///
 /// On failure, `success` is false and `error` contains the message.
-///
-/// **Important:** `fff_free_result` frees `error` but does **not** free `handle`.
-/// The caller must free the handle with the appropriate function
-/// (`fff_destroy`, `fff_free_search_result`, `fff_free_grep_result`,
-///  `fff_free_string`, etc.).
 #[repr(C)]
 pub struct FffResult {
     /// Whether the operation succeeded.
     pub success: bool,
     /// Error message on failure. Null on success.
     pub error: *mut c_char,
-    /// Opaque pointer payload (instance handle, typed result struct, or string). May be null.
+    /// Opaque pointer payload. May be null.
     pub handle: *mut c_void,
     /// Integer payload for simple return values (bool as 0/1, counts, etc.).
     pub int_value: i64,
@@ -515,15 +567,243 @@ impl FffResult {
     }
 }
 
-// ---------------------------------------------------------------------------
-// Scan progress
-// ---------------------------------------------------------------------------
+/// A directory item returned by `fff_search_directories`. Strings are owned by
+/// the parent `FffDirSearchResult`; free everything with `fff_free_dir_search_result`.
+#[repr(C)]
+pub struct FffDirItem {
+    pub relative_path: *mut c_char,
+    pub dir_name: *mut c_char,
+    pub max_access_frecency: i32,
+}
+
+impl FffDirItem {
+    pub fn from_item(item: &DirItem, picker: &FilePicker) -> Self {
+        FffDirItem {
+            relative_path: cstring_new(&item.relative_path(picker)),
+            dir_name: cstring_new(&item.dir_name(picker)),
+            max_access_frecency: item.max_access_frecency(),
+        }
+    }
+
+    /// ## Safety
+    /// All string pointers must have been allocated by the rust side
+    pub unsafe fn free_strings(&mut self) {
+        unsafe {
+            if !self.relative_path.is_null() {
+                drop(CString::from_raw(self.relative_path));
+            }
+            if !self.dir_name.is_null() {
+                drop(CString::from_raw(self.dir_name));
+            }
+        }
+    }
+}
+
+/// Directory search result returned by `fff_search_directories`;
+/// free with `fff_free_dir_search_result`.
+#[repr(C)]
+pub struct FffDirSearchResult {
+    /// Heap array of `FffDirItem` (length = `count`).
+    pub items: *mut FffDirItem,
+    /// Heap array of `FffScore` (length = `count`).
+    pub scores: *mut FffScore,
+    /// Number of items/scores in the arrays.
+    pub count: u32,
+    /// Total number of directories that matched the query.
+    pub total_matched: u32,
+    /// Total number of indexed directories.
+    pub total_dirs: u32,
+}
+
+impl FffDirSearchResult {
+    /// Convert a core `DirSearchResult` into a heap-allocated `FffDirSearchResult`.
+    pub fn from_core(result: &DirSearchResult, picker: &FilePicker) -> *mut Self {
+        let items: Vec<FffDirItem> = result
+            .items
+            .iter()
+            .map(|i| FffDirItem::from_item(i, picker))
+            .collect();
+        let scores: Vec<FffScore> = result.scores.iter().map(FffScore::from).collect();
+        let count = items.len() as u32;
+
+        let (items_ptr, _) = vec_to_raw(items);
+        let (scores_ptr, _) = vec_to_raw(scores);
+
+        Box::into_raw(Box::new(FffDirSearchResult {
+            items: items_ptr,
+            scores: scores_ptr,
+            count,
+            total_matched: result.total_matched as u32,
+            total_dirs: result.total_dirs as u32,
+        }))
+    }
+}
+
+/// A single item in a mixed (files + directories) search result.
+/// `item_type`: 0 = file, 1 = directory. Strings are owned by the parent
+/// `FffMixedSearchResult`.
+#[repr(C)]
+pub struct FffMixedItem {
+    /// 0 = file, 1 = directory.
+    pub item_type: u8,
+    pub relative_path: *mut c_char,
+    /// Filename for files, last directory segment for directories.
+    pub display_name: *mut c_char,
+    pub git_status: *mut c_char,
+    pub size: u64,
+    pub modified: u64,
+    /// Access frecency for files; max among immediate children for directories.
+    pub access_frecency_score: i64,
+    /// Always 0 for directories
+    pub modification_frecency_score: i64,
+    /// Always 0 for directories
+    pub total_frecency_score: i64,
+    /// Always 0 for directories
+    pub is_binary: bool,
+}
+
+impl FffMixedItem {
+    pub fn from_mixed_ref(item: &MixedItemRef<'_>, picker: &FilePicker) -> Self {
+        match item {
+            MixedItemRef::File(file) => FffMixedItem {
+                item_type: 0,
+                relative_path: cstring_new(&file.relative_path(picker)),
+                display_name: cstring_new(&file.file_name(picker)),
+                git_status: cstring_new(format_git_status(file.git_status)),
+                size: file.size,
+                modified: file.modified,
+                access_frecency_score: file.access_frecency_score as i64,
+                modification_frecency_score: file.modification_frecency_score as i64,
+                total_frecency_score: file.total_frecency_score() as i64,
+                is_binary: file.is_binary(),
+            },
+            MixedItemRef::Dir(dir) => FffMixedItem {
+                item_type: 1,
+                relative_path: cstring_new(&dir.relative_path(picker)),
+                display_name: cstring_new(&dir.dir_name(picker)),
+                git_status: cstring_new(""),
+                size: 0,
+                modified: 0,
+                access_frecency_score: dir.max_access_frecency() as i64,
+                modification_frecency_score: 0,
+                total_frecency_score: dir.max_access_frecency() as i64,
+                is_binary: false,
+            },
+        }
+    }
+
+    /// ## Safety
+    /// All string pointers must have been allocated by rust side
+    pub unsafe fn free_strings(&mut self) {
+        unsafe {
+            if !self.relative_path.is_null() {
+                drop(CString::from_raw(self.relative_path));
+            }
+            if !self.display_name.is_null() {
+                drop(CString::from_raw(self.display_name));
+            }
+            if !self.git_status.is_null() {
+                drop(CString::from_raw(self.git_status));
+            }
+        }
+    }
+}
+
+/// Mixed search result returned by `fff_search_mixed`
+/// free with `fff_free_mixed_search_result`.
+#[repr(C)]
+pub struct FffMixedSearchResult {
+    /// Heap array of `FffMixedItem` (length = `count`).
+    pub items: *mut FffMixedItem,
+    /// Heap array of `FffScore` (length = `count`).
+    pub scores: *mut FffScore,
+    /// Number of items/scores in the arrays.
+    pub count: u32,
+    /// Total number of items (files + dirs) that matched the query.
+    pub total_matched: u32,
+    /// Total number of indexed files.
+    pub total_files: u32,
+    /// Total number of indexed directories.
+    pub total_dirs: u32,
+    /// Location parsed from the query string.
+    pub location: FffLocation,
+}
+
+impl FffMixedSearchResult {
+    /// Convert a core `MixedSearchResult` into a heap-allocated `FffMixedSearchResult`.
+    pub fn from_core(result: &MixedSearchResult, picker: &FilePicker) -> *mut Self {
+        let items: Vec<FffMixedItem> = result
+            .items
+            .iter()
+            .map(|i| FffMixedItem::from_mixed_ref(i, picker))
+            .collect();
+        let scores: Vec<FffScore> = result.scores.iter().map(FffScore::from).collect();
+        let count = items.len() as u32;
+
+        let (items_ptr, _) = vec_to_raw(items);
+        let (scores_ptr, _) = vec_to_raw(scores);
+
+        Box::into_raw(Box::new(FffMixedSearchResult {
+            items: items_ptr,
+            scores: scores_ptr,
+            count,
+            total_matched: result.total_matched as u32,
+            total_files: result.total_files as u32,
+            total_dirs: result.total_dirs as u32,
+            location: FffLocation::from(result.location.as_ref()),
+        }))
+    }
+}
 
 /// Scan progress returned by `fff_get_scan_progress`.
-///
 /// The caller must free this with `fff_free_scan_progress`.
 #[repr(C)]
 pub struct FffScanProgress {
     pub scanned_files_count: u64,
     pub is_scanning: bool,
+    pub is_watcher_ready: bool,
+    pub is_warmup_complete: bool,
+}
+
+impl From<fff::file_picker::ScanProgress> for FffScanProgress {
+    fn from(p: fff::file_picker::ScanProgress) -> Self {
+        Self {
+            scanned_files_count: p.scanned_files_count as u64,
+            is_scanning: p.is_scanning,
+            is_watcher_ready: p.is_watcher_ready,
+            is_warmup_complete: p.is_warmup_complete,
+        }
+    }
+}
+
+#[cfg(test)]
+mod options_layout_tests {
+    use super::FffCreateOptions;
+    use std::mem::{align_of, offset_of, size_of};
+
+    // THIS TEST HAVE TO BE NEVER UPDATED ONLY ADDED NEW FIELDS
+    // this is needed to ensure ABI backward compatibility
+    #[test]
+    #[cfg(target_pointer_width = "64")]
+    fn fff_create_options_layout_is_stable_64bit() {
+        assert_eq!(size_of::<FffCreateOptions>(), 88);
+        assert_eq!(align_of::<FffCreateOptions>(), 8);
+
+        assert_eq!(offset_of!(FffCreateOptions, version), 0);
+        assert_eq!(offset_of!(FffCreateOptions, base_path), 8);
+        assert_eq!(offset_of!(FffCreateOptions, frecency_db_path), 16);
+        assert_eq!(offset_of!(FffCreateOptions, history_db_path), 24);
+        assert_eq!(offset_of!(FffCreateOptions, enable_mmap_cache), 32);
+        assert_eq!(offset_of!(FffCreateOptions, enable_content_indexing), 33);
+        assert_eq!(offset_of!(FffCreateOptions, watch), 34);
+        assert_eq!(offset_of!(FffCreateOptions, ai_mode), 35);
+        assert_eq!(offset_of!(FffCreateOptions, log_file_path), 40);
+        assert_eq!(offset_of!(FffCreateOptions, log_level), 48);
+        assert_eq!(offset_of!(FffCreateOptions, cache_budget_max_files), 56);
+        assert_eq!(offset_of!(FffCreateOptions, cache_budget_max_bytes), 64);
+        assert_eq!(offset_of!(FffCreateOptions, cache_budget_max_file_size), 72);
+        assert_eq!(offset_of!(FffCreateOptions, enable_fs_root_scanning), 80);
+        assert_eq!(offset_of!(FffCreateOptions, enable_home_dir_scanning), 81);
+        assert_eq!(offset_of!(FffCreateOptions, follow_symlinks), 82);
+    }
 }

crates/fff-c/src/watch.rs

@@ -0,0 +1,351 @@
+use std::ffi::{CString, c_char, c_void};
+use std::ptr;
+use std::sync::Arc;
+use std::sync::Mutex;
+
+use fff::{WatchEvent, WatchId, WatchOptions};
+
+use crate::ffi_types::FffResult;
+use crate::instance_ref;
+
+/// Current version of [`FffWatchOptions`].
+pub const FFF_WATCH_OPTIONS_VERSION: u32 = 1;
+
+/// Options for `fff_watch`. Versioned: new fields are only appended.
+#[repr(C)]
+pub struct FffWatchOptions {
+    /// Set to [`FFF_WATCH_OPTIONS_VERSION`] when allocating.
+    pub version: u32,
+    /// Per-subscription excludes (parcel-watcher style): entries with wildcards
+    /// are base-relative globs, entries without are path prefixes. NULL when
+    /// `ignore_count` is 0.
+    pub ignore: *const *const c_char,
+    pub ignore_count: u32,
+    // ----- new version 2+ fields go here, ALWAYS appended -----
+}
+
+/// A single watch event. `kind`: 0 = created, 1 = modified, 2 = removed,
+/// 3 = rescan (events were lost; re-stat what you care about).
+#[repr(C)]
+pub struct FffWatchEvent {
+    /// Absolute path (heap C string owned by the parent batch).
+    pub path: *mut c_char,
+    pub kind: u8,
+}
+
+/// A batch of watch events. Free with `fff_free_watch_events`.
+#[repr(C)]
+pub struct FffWatchEventBatch {
+    pub events: *mut FffWatchEvent,
+    pub count: u32,
+}
+
+/// Instance-wide callback invoked with `(watch_id, batch)` for every `fff_watch`
+/// subscription. The callee owns and frees `batch` via `fff_free_watch_events`.
+pub type FffWatchCallback =
+    unsafe extern "C" fn(watch_id: u64, batch: *mut FffWatchEventBatch, user_data: *mut c_void);
+
+fn batch_into_raw(events: &[WatchEvent]) -> *mut FffWatchEventBatch {
+    let items: Vec<FffWatchEvent> = events
+        .iter()
+        .map(|ev| FffWatchEvent {
+            path: CString::new(ev.path.to_string_lossy().as_bytes())
+                .unwrap_or_default()
+                .into_raw(),
+            kind: ev.kind as u8,
+        })
+        .collect();
+
+    let count = items.len() as u32;
+    let events_ptr = if items.is_empty() {
+        ptr::null_mut()
+    } else {
+        let mut boxed = items.into_boxed_slice();
+        let p = boxed.as_mut_ptr();
+        std::mem::forget(boxed);
+        p
+    };
+
+    Box::into_raw(Box::new(FffWatchEventBatch {
+        events: events_ptr,
+        count,
+    }))
+}
+
+unsafe fn watch_options_from_ffi(
+    opts: *const FffWatchOptions,
+) -> Result<WatchOptions, *mut FffResult> {
+    if opts.is_null() {
+        return Ok(WatchOptions::default());
+    }
+    let opts = unsafe { &*opts };
+    if opts.version == 0 || opts.version > FFF_WATCH_OPTIONS_VERSION {
+        return Err(FffResult::err(&format!(
+            "Unsupported FffWatchOptions version {} (library understands up to {})",
+            opts.version, FFF_WATCH_OPTIONS_VERSION
+        )));
+    }
+
+    let mut ignore = Vec::with_capacity(opts.ignore_count as usize);
+    if opts.ignore_count > 0 {
+        if opts.ignore.is_null() {
+            return Err(FffResult::err("ignore_count > 0 but ignore is NULL"));
+        }
+        for i in 0..opts.ignore_count as usize {
+            let entry = unsafe { *opts.ignore.add(i) };
+            match unsafe { crate::cstr_to_str(entry) } {
+                Some(s) if !s.is_empty() => ignore.push(s.to_string()),
+                Some(_) => {}
+                None => return Err(FffResult::err("ignore entry is NULL or invalid UTF-8")),
+            }
+        }
+    }
+
+    Ok(WatchOptions { ignore })
+}
+
+// The caller guarantees user_data is safe on the callback thread.
+struct UserData(*mut c_void);
+unsafe impl Send for UserData {}
+unsafe impl Sync for UserData {}
+
+// Shared so a closure surviving an unwatch race never dangles.
+#[derive(Default)]
+pub(crate) struct WatchCallbackSlot(Mutex<Option<(FffWatchCallback, UserData)>>);
+
+impl WatchCallbackSlot {
+    fn get(&self) -> Option<(FffWatchCallback, *mut c_void)> {
+        self.0
+            .lock()
+            .ok()
+            .and_then(|guard| guard.as_ref().map(|(cb, ud)| (*cb, ud.0)))
+    }
+
+    fn set(&self, callback: FffWatchCallback, user_data: *mut c_void) {
+        if let Ok(mut guard) = self.0.lock() {
+            *guard = Some((callback, UserData(user_data)));
+        }
+    }
+
+    pub(crate) fn clear(&self) {
+        if let Ok(mut guard) = self.0.lock() {
+            *guard = None;
+        }
+    }
+}
+
+/// Register the instance-wide watch callback used by all `fff_watch`
+/// subscriptions; call before the first `fff_watch`, calling again replaces it.
+///
+/// ## Safety
+/// * `fff_handle` must be a valid instance pointer from `fff_create_instance`.
+/// * `callback` must remain callable until fff_unwatch called
+///   `fff_destroy(fff_handle)` returns.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_set_watch_callback(
+    fff_handle: *mut c_void,
+    callback: FffWatchCallback,
+    user_data: *mut c_void,
+) -> *mut FffResult {
+    let inst = match unsafe { instance_ref(fff_handle) } {
+        Ok(i) => i,
+        Err(e) => return e,
+    };
+    inst.watch_callback.set(callback, user_data);
+    FffResult::ok_empty()
+}
+
+/// Subscribe to filesystem changes, delivered through the instance callback
+/// registered by `fff_set_watch_callback`.
+///
+/// Returns the watch id, pass it to `fff_unwatch` to stop.
+///
+/// `pattern` if non `NULL` can be wildcard pattern, absolute, or relative path
+/// that will be used to filter the events triggering exact subscription.
+///
+/// ## Safety
+/// * `fff_handle` must be a valid instance pointer from `fff_create_instance`.
+/// * `pattern` must be NULL or valid null-terminated UTF-8.
+/// * `opts` must be NULL or a valid `FffWatchOptions` pointer.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_watch(
+    fff_handle: *mut c_void,
+    pattern: *const c_char,
+    opts: *const FffWatchOptions,
+) -> *mut FffResult {
+    let inst = match unsafe { instance_ref(fff_handle) } {
+        Ok(i) => i,
+        Err(e) => return e,
+    };
+    // NULL pattern = watch the entire indexed tree ("" in core).
+    let pattern_str = if pattern.is_null() {
+        ""
+    } else {
+        match unsafe { crate::cstr_to_str(pattern) } {
+            Some(s) => s,
+            None => return FffResult::err("Pattern is not valid UTF-8"),
+        }
+    };
+    let options = match unsafe { watch_options_from_ffi(opts) } {
+        Ok(o) => o,
+        Err(e) => return e,
+    };
+    if inst.watch_callback.get().is_none() {
+        return FffResult::err("No watch callback registered. Call fff_set_watch_callback first.");
+    }
+
+    let slot = Arc::clone(&inst.watch_callback);
+    let result = inst.picker.watch(pattern_str, options, move |id, events| {
+        if let Some((cb, user_data)) = slot.get() {
+            let batch = batch_into_raw(events);
+            unsafe { cb(id.0, batch, user_data) };
+        }
+    });
+
+    match result {
+        Ok(id) => FffResult::ok_int(id.0 as i64),
+        Err(e) => FffResult::err(&format!("Failed to subscribe: {}", e)),
+    }
+}
+
+/// [`fff_watch`] adapter with flattened options, for FFI libraries that cannot
+/// marshal pointer arrays inside structs (e.g. Node's `ffi-rs`).
+///
+/// ## Safety
+/// * `fff_handle` must be a valid instance pointer from `fff_create_instance`.
+/// * `pattern` must be NULL (watch everything) or valid null-terminated UTF-8.
+/// * `ignore` must be NULL or point to `ignore_count` valid C strings.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_watch_args(
+    fff_handle: *mut c_void,
+    pattern: *const c_char,
+    ignore: *const *const c_char,
+    ignore_count: u32,
+) -> *mut FffResult {
+    let opts = FffWatchOptions {
+        version: FFF_WATCH_OPTIONS_VERSION,
+        ignore,
+        ignore_count,
+    };
+    unsafe { fff_watch(fff_handle, pattern, &opts) }
+}
+
+/// Remove a watch subscription. `int_value` = 1 if the id existed, 0 otherwise.
+///
+/// ## Safety
+/// `fff_handle` must be a valid instance pointer from `fff_create_instance`.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_unwatch(fff_handle: *mut c_void, watch_id: u64) -> *mut FffResult {
+    let inst = match unsafe { instance_ref(fff_handle) } {
+        Ok(i) => i,
+        Err(e) => return e,
+    };
+    FffResult::ok_int(inst.picker.unwatch(WatchId(watch_id)) as i64)
+}
+
+/// Number of events in a batch, 0 if `batch` is null.
+///
+/// ## Safety
+/// `batch` must be a valid `FffWatchEventBatch` pointer or null.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_watch_events_count(batch: *const FffWatchEventBatch) -> u32 {
+    if batch.is_null() {
+        return 0;
+    }
+    unsafe { (*batch).count }
+}
+
+/// Absolute path of event `index`, will be null when out of bounds
+///
+/// ## Safety
+/// `batch` must be a valid `FffWatchEventBatch` pointer or null.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_watch_events_get_path(
+    batch: *const FffWatchEventBatch,
+    index: u32,
+) -> *const c_char {
+    match unsafe { watch_event_at(batch, index) } {
+        Some(ev) => ev.path,
+        None => ptr::null(),
+    }
+}
+
+/// Kind of event `index` (0 = created, 1 = modified, 2 = removed, 3 = rescan)
+/// 3 (rescan aka "re-stat something" kind) returned when OS based buffer
+/// has been overflown and some events might be loss. Paths will contain a list of
+/// directories that needs to be rescanned to ensure consistency.
+///
+/// ## Safety
+/// `batch` must be a valid `FffWatchEventBatch` pointer or null.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_watch_events_get_kind(
+    batch: *const FffWatchEventBatch,
+    index: u32,
+) -> u8 {
+    match unsafe { watch_event_at(batch, index) } {
+        Some(ev) => ev.kind,
+        None => 3,
+    }
+}
+
+unsafe fn watch_event_at<'a>(
+    batch: *const FffWatchEventBatch,
+    index: u32,
+) -> Option<&'a FffWatchEvent> {
+    if batch.is_null() {
+        return None;
+    }
+    let batch = unsafe { &*batch };
+    if batch.events.is_null() || index >= batch.count {
+        return None;
+    }
+    Some(unsafe { &*batch.events.add(index as usize) })
+}
+
+/// Free a watch event batch delivered to the instance callback.
+///
+/// ## Safety
+/// `batch` must be a pointer produced by this library, or null (no-op).
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn fff_free_watch_events(batch: *mut FffWatchEventBatch) {
+    if batch.is_null() {
+        return;
+    }
+    unsafe {
+        let batch = Box::from_raw(batch);
+        if !batch.events.is_null() {
+            let events =
+                Vec::from_raw_parts(batch.events, batch.count as usize, batch.count as usize);
+            for ev in events {
+                if !ev.path.is_null() {
+                    drop(CString::from_raw(ev.path));
+                }
+            }
+        }
+    }
+}
+
+// THESE TESTS MUST NEVER BE UPDATED, ONLY EXTENDED WITH NEW FIELDS —
+// bindings hardcode these offsets (ABI stability).
+#[cfg(test)]
+mod layout_tests {
+    use super::*;
+    use std::mem::{offset_of, size_of};
+
+    #[test]
+    #[cfg(target_pointer_width = "64")]
+    fn watch_ffi_layouts_are_stable_64bit() {
+        assert_eq!(size_of::<FffWatchOptions>(), 24);
+        assert_eq!(offset_of!(FffWatchOptions, version), 0);
+        assert_eq!(offset_of!(FffWatchOptions, ignore), 8);
+        assert_eq!(offset_of!(FffWatchOptions, ignore_count), 16);
+
+        assert_eq!(size_of::<FffWatchEvent>(), 16);
+        assert_eq!(offset_of!(FffWatchEvent, path), 0);
+        assert_eq!(offset_of!(FffWatchEvent, kind), 8);
+
+        assert_eq!(size_of::<FffWatchEventBatch>(), 16);
+        assert_eq!(offset_of!(FffWatchEventBatch, events), 0);
+        assert_eq!(offset_of!(FffWatchEventBatch, count), 8);
+    }
+}

crates/fff-c/tests/smoke.c

@@ -0,0 +1,275 @@
+/*
+ * Smoke test for libfff_c — the smallest possible end-to-end exercise of
+ * the public C API. We:
+ *
+ *   1. Create a picker with an `FffCreateOptions` populated via C99
+ *      designated initializers (the recommended idiom for direct C use).
+ *   2. Wait for the initial scan to complete.
+ *   3. Search for "smoke.c".
+ *   4. Fail unless this very file appears in the results.
+ *
+ * Build + run via `make test-c-smoke`. Override $(CC) to test other
+ * compilers.
+ */
+
+/* expose mkdtemp/usleep under -std=c99 on glibc; harmless on musl/darwin */
+#define _DEFAULT_SOURCE
+#define _BSD_SOURCE
+
+#include <fff.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+// simple mock function to make sure that both globbing patterns and dir based pattern work
+static int watch_glob_hits = 0;
+static int watch_dir_hits = 0;
+static int watch_all_hits = 0;
+static int watch_ignored_leaks = 0;
+static uint64_t watch_glob_id = 0;
+static uint64_t watch_dir_id = 0;
+static uint64_t watch_all_id = 0;
+
+static void on_watch_batch(uint64_t watch_id, struct FffWatchEventBatch *batch, void *user_data) {
+    (void)user_data;
+    /* route by id like real SDKs do; unknown ids are benign no-ops */
+    for (uint32_t i = 0; i < batch->count; i++) {
+        const char *path = batch->events[i].path;
+        if (!path) continue;
+        if (watch_id == watch_glob_id && strstr(path, "hello.txt")) {
+            watch_glob_hits++;
+        }
+        if (watch_id == watch_dir_id) {
+            if (strstr(path, "hello.txt")) watch_dir_hits++;
+            if (strstr(path, "noise.log")) watch_ignored_leaks++;
+        }
+        if (watch_id == watch_all_id && strstr(path, "hello.txt")) {
+            watch_all_hits++;
+        }
+    }
+
+    fff_free_watch_events(batch); // need to clean dynamic array of events
+}
+
+static int watch_smoke(void) {
+    char tmpl[] = "/tmp/fff-c-watch-XXXXXX";
+    char *dir = mkdtemp(tmpl);
+    if (!dir) {
+        fprintf(stderr, "watch_smoke: mkdtemp failed\n");
+        return 1;
+    }
+
+    struct FffResult *create_result = fff_create_instance_with(&(struct FffCreateOptions){
+        .version = FFF_CREATE_OPTIONS_VERSION,
+        .base_path = dir,
+        .enable_mmap_cache = false,
+        .enable_content_indexing = false,
+        .watch = true,
+    });
+    if (!create_result->success) {
+        fprintf(stderr, "watch_smoke: create failed: %s\n",
+                create_result->error ? create_result->error : "?");
+        fff_free_result(create_result);
+        return 1;
+    }
+    void *picker = create_result->handle;
+    fff_free_result(create_result);
+
+    struct FffResult *r = fff_wait_for_scan(picker, 10000);
+    fff_free_result(r);
+    r = fff_wait_for_watcher(picker, 10000);
+    fff_free_result(r);
+    usleep(300 * 1000); /* let the FSEvents stream settle */
+
+    /* instance-wide callback, then two subscriptions routed by id */
+    r = fff_set_watch_callback(picker, on_watch_batch, NULL);
+    if (!r->success) {
+        fprintf(stderr, "watch_smoke: fff_set_watch_callback failed: %s\n",
+                r->error ? r->error : "?");
+        fff_free_result(r);
+        fff_destroy(picker);
+        return 1;
+    }
+    fff_free_result(r);
+
+    r = fff_watch(picker, "**/*.txt", NULL);
+    if (!r->success) {
+        fprintf(stderr, "watch_smoke: fff_watch failed: %s\n", r->error ? r->error : "?");
+        fff_free_result(r);
+        fff_destroy(picker);
+        return 1;
+    }
+    watch_glob_id = (uint64_t)r->int_value;
+    fff_free_result(r);
+
+    /* whole-tree dir subscription with an ignore glob */
+    const char *ignores[] = {"*.log"};
+    r = fff_watch(picker, dir,
+                  &(struct FffWatchOptions){.version = FFF_WATCH_OPTIONS_VERSION,
+                                            .ignore = ignores,
+                                            .ignore_count = 1});
+    if (!r->success) {
+        fprintf(stderr, "watch_smoke: dir fff_watch failed: %s\n", r->error ? r->error : "?");
+        fff_free_result(r);
+        fff_destroy(picker);
+        return 1;
+    }
+    watch_dir_id = (uint64_t)r->int_value;
+    fff_free_result(r);
+
+    /* NULL pattern subscribes to the entire indexed tree */
+    r = fff_watch(picker, NULL, NULL);
+    if (!r->success) {
+        fprintf(stderr, "watch_smoke: NULL-pattern fff_watch failed: %s\n",
+                r->error ? r->error : "?");
+        fff_free_result(r);
+        fff_destroy(picker);
+        return 1;
+    }
+    watch_all_id = (uint64_t)r->int_value;
+    fff_free_result(r);
+
+    char file_path[512];
+    snprintf(file_path, sizeof(file_path), "%s/hello.txt", dir);
+    FILE *f = fopen(file_path, "w");
+    if (!f) {
+        fprintf(stderr, "watch_smoke: fopen failed\n");
+        fff_destroy(picker);
+        return 1;
+    }
+    fputs("hello watch\n", f);
+    fclose(f);
+
+    /* must be filtered out by the dir subscription's ignore glob */
+    char log_path[512];
+    snprintf(log_path, sizeof(log_path), "%s/noise.log", dir);
+    FILE *lf = fopen(log_path, "w");
+    if (lf) {
+        fputs("noise\n", lf);
+        fclose(lf);
+    }
+
+    for (int attempt = 0;
+         attempt < 100 && (watch_glob_hits == 0 || watch_dir_hits == 0 || watch_all_hits == 0);
+         attempt++) {
+        usleep(100 * 1000);
+    }
+
+    r = fff_unwatch(picker, watch_glob_id);
+    fff_free_result(r);
+    r = fff_unwatch(picker, watch_dir_id);
+    fff_free_result(r);
+    r = fff_unwatch(picker, watch_all_id);
+    fff_free_result(r);
+    /* unwatch of an unknown id reports 0, not an error */
+    r = fff_unwatch(picker, watch_dir_id);
+    int unwatch_idempotent = r->success && r->int_value == 0;
+    fff_free_result(r);
+
+    /* fff_destroy is the quiescence barrier: after it returns the callback
+     * will never run again and could be freed (ours is static). */
+    fff_destroy(picker);
+
+    if (watch_glob_hits == 0) {
+        fprintf(stderr, "watch_smoke FAIL: glob subscriber saw no events\n");
+        return 1;
+    }
+    if (watch_dir_hits == 0) {
+        fprintf(stderr, "watch_smoke FAIL: dir subscriber saw no events\n");
+        return 1;
+    }
+    if (watch_all_hits == 0) {
+        fprintf(stderr, "watch_smoke FAIL: NULL-pattern subscriber saw no events\n");
+        return 1;
+    }
+    if (watch_ignored_leaks > 0) {
+        fprintf(stderr, "watch_smoke FAIL: ignore glob leaked %d events\n", watch_ignored_leaks);
+        return 1;
+    }
+    if (!unwatch_idempotent) {
+        fprintf(stderr, "watch_smoke FAIL: repeated unwatch was not a no-op\n");
+        return 1;
+    }
+
+    fprintf(stderr, "watch_smoke PASS (glob=%d dir=%d all=%d)\n", watch_glob_hits, watch_dir_hits,
+            watch_all_hits);
+    return 0;
+}
+
+int main(int argc, char **argv) {
+    const char *base_path = argc > 1 ? argv[1] : ".";
+
+    // make sure that FFF C api is designed more for FFI rather than for direct C usage (I'm sorry)
+    struct FffResult *create_result = fff_create_instance_with(&(struct FffCreateOptions){
+        .version = FFF_CREATE_OPTIONS_VERSION,
+        .base_path = base_path,
+        .enable_mmap_cache = false,
+        .enable_content_indexing = false,
+        .watch = false,
+    });
+
+    if (!create_result->success) {
+        fprintf(stderr, "fff couldn't create instance: %s\n",
+                create_result->error ? create_result->error : "?");
+        fff_free_result(create_result);
+        return 1;
+    }
+
+    void *file_picker = create_result->handle;
+    fff_free_result(create_result); // safe to drop now: handle outlives the envelope
+
+    struct FffResult *scan_result = fff_wait_for_scan(file_picker, 5000);
+    if (!scan_result->success) {
+        fprintf(stderr, "wait_for_scan failed: %s\n",
+                scan_result->error ? scan_result->error : "?");
+        fff_free_result(scan_result);
+        fff_destroy(file_picker);
+        return 1;
+    }
+    // int_value: 1 = scan completed in time, 0 = timed out.
+    if (scan_result->int_value == 0) {
+        fprintf(stderr, "wait_for_scan: timed out before initial scan finished\n");
+        fff_free_result(scan_result);
+        fff_destroy(file_picker);
+        return 1;
+    }
+    fff_free_result(scan_result);
+
+    struct FffResult *res = fff_search(file_picker, "smkoe.c", "", 0, 0, 50, 0, 0);
+    if (!res->success) {
+        fprintf(stderr, "search failed: %s\n", res->error ? res->error : "?");
+        fff_free_result(res);
+        fff_destroy(file_picker);
+        return 1;
+    }
+
+    struct FffSearchResult *sr = (struct FffSearchResult *)res->handle;
+    uint32_t total = sr->count;
+    int found = 0;
+    for (uint32_t i = 0; i < sr->count; i++) {
+        const char *path = sr->items[i].relative_path;
+        if (path && strstr(path, "smoke.c")) {
+            found = 1;
+            fprintf(stderr, "found self: %s\n", path);
+            break;
+        }
+    }
+
+    fff_free_search_result(sr);
+    fff_free_result(res);
+    fff_destroy(file_picker);
+
+    if (!found) {
+        fprintf(stderr, "FAIL: smoke.c not in search results (count=%u)\n", total);
+        return 1;
+    }
+
+    if (watch_smoke() != 0) {
+        fprintf(stderr, "FAIL: watch test failed\n");
+        return 1;
+    }
+
+    fprintf(stderr, "PASS\n");
+    return 0;
+}

crates/fff-core/Cargo.toml

@@ -1,19 +1,56 @@
 [package]
 name = "fff-search"
-version = "0.4.2"
+version = "0.10.5"
 edition = "2024"
 license = "MIT"
 authors = ["Dmitriy Kovalenko <dmtr.kovalenko@outlook.com>"]
 description = "Faboulous & Fast File Finder - a fast and extremely correct file finder SDK with typo resistance, SIMD, prefiltering, and more"
 
+[lints]
+workspace = true
+
 [lib]
 path = "src/lib.rs"
-crate-type = ["rlib", "staticlib", "cdylib"]
+crate-type = ["rlib"]
+
+[[bench]]
+name = "parse_bench"
+harness = false
+
+[[bench]]
+name = "bigram_bench"
+harness = false
+
+[[bench]]
+name = "memmem_bench"
+harness = false
+
+[[bench]]
+name = "glob_bench"
+harness = false
+required-features = ["zlob"]
+
+[[bench]]
+name = "grep_bench"
+harness = false
 
 [features]
-default = []
+# `ripgrep` is the pure-Rust walker/glob backend and is on by default so
+# consumers build without a Zig toolchain. CI/release opt into zlob via
+# `--no-default-features --features zlob`.
+default = ["ripgrep"]
 # Enable C FFI exports
 ffi = []
+# Count full rescans and their causes. Always on in debug builds; enable this
+# to keep the accounting in a release build (used by the rescan_probe binary).
+rescan-stats = []
+# Enables POC definition classification for grep result matched lines
+definitions = []
+# Pure-Rust filesystem walker + glob matcher (ignore + globset crates).
+ripgrep = ["dep:ignore", "dep:globset", "fff-query-parser/ripgrep"]
+# Call mi_collect(true) after large allocator churn (bigram build).
+# Requires mimalloc to be the global allocator (linked by fff-nvim).
+mimalloc-collect = ["dep:libmimalloc-sys"]
 # Use zlob (Zig-compiled C globbing library) for glob matching.
 # Requires Zig to be installed. When disabled, falls back to globset (pure Rust).
 zlob = ["dep:zlob", "fff-query-parser/zlob"]
@@ -24,42 +61,45 @@ rayon = { workspace = true }
 smallvec = { workspace = true }
 thiserror = { workspace = true }
 tracing = { workspace = true }
-fff-query-parser = { workspace = true  }
-
-# External dependencies
-bindet = { workspace = true }
+fff-query-parser = { workspace = true , version = "0.9.0" }
 blake3 = { workspace = true }
-chrono = { workspace = true }
 dirs = { workspace = true }
+libc = "0.2"
 git2 = { workspace = true }
 glidesort = { workspace = true }
-globset = { workspace = true }
-grep-matcher = { workspace = true }
-fff-grep = { workspace = true  }
+globset = { workspace = true, optional = true }
+fff-grep = { workspace = true , version = "0.9.0" }
 aho-corasick = "1"
 memchr = "2"
 heed = { workspace = true }
-ignore = { workspace = true }
+ignore = { workspace = true, optional = true }
 memmap2 = { workspace = true }
 neo_frizbee = { workspace = true }
 notify = { workspace = true }
 notify-debouncer-full = { workspace = true }
-once_cell = { workspace = true }
 parking_lot = { workspace = true }
 pathdiff = { workspace = true }
 regex = { workspace = true }
+regex-syntax = "0.8"
 serde = { version = "1.0", features = ["derive"] }
-serde_json = "1.0"
-smartstring = { version = "1.0.1", features = ["serde"] }
-toml = "0.8"
 tracing-appender = "0.2"
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
 zlob = { workspace = true, optional = true }
+libmimalloc-sys = { version = "0.1", optional = true, features = ["extended", "local_dynamic_tls"] }
+mimalloc = { version = "0.1", optional = true, features = ["local_dynamic_tls"] }
 # Platform-specific: dunce for Windows to avoid \\?\ extended path prefix
 [target.'cfg(windows)'.dependencies]
 dunce = { workspace = true }
 
+# signal-hook only compiles on unix; we wrap the SIGSEGV handler behind cfg(unix)
+[target.'cfg(unix)'.dependencies]
+signal-hook-registry = { workspace = true }
+
 [dev-dependencies]
 criterion = { version = "0.5", features = ["html_reports"] }
+ctor = "0.2"
+proptest = { version = "1", default-features = false, features = ["std", "fork"] }
 rand = { version = "0.8", features = ["small_rng"] }
 tempfile = "3.8"
+tracing-subscriber = { version = "0.3", features = ["env-filter"] }
+

crates/fff-core/README.md

@@ -2,6 +2,9 @@
 
 fff is a file search toolkit. It is faster than ripgrep and fzf and designed for a long running applications like file editors, ai agents, or file exploerers.
 
+> [!Important performance information]  
+> For the most optimized fff build use `zlob` feature. It requires zig v0.16.0 to be installed on the machine.
+
 ## Features
 
 - Fuzzy file name search 

crates/fff-core/benches/bigram_bench.rs

@@ -0,0 +1,164 @@
+use criterion::{BenchmarkId, Criterion, black_box, criterion_group, criterion_main};
+use fff_search::bigram_filter::{BigramFilter, BigramIndexBuilder};
+
+/// Build a realistic bigram index for benchmarking.
+/// Simulates a large repo by generating varied content per file.
+fn build_test_index(file_count: usize) -> BigramFilter {
+    let builder = BigramIndexBuilder::new(file_count);
+    let skip_builder = BigramIndexBuilder::new(file_count);
+
+    for i in 0..file_count {
+        // Generate varied content so we get a mix of sparse and dense columns
+        let content = format!(
+            "struct File{i} {{ fn process() {{ let controller = read(path); }} }} // module {i}"
+        );
+        builder.add_file_content(&skip_builder, i, content.as_bytes());
+    }
+
+    let mut index = builder.compress(None);
+    let skip_index = skip_builder.compress(Some(12));
+    index.set_skip_index(skip_index);
+    index
+}
+
+fn bench_bigram_query(c: &mut Criterion) {
+    let file_counts = [10_000, 100_000, 500_000];
+
+    for &file_count in &file_counts {
+        let index = build_test_index(file_count);
+        eprintln!(
+            "Index ({} files): {} columns",
+            file_count,
+            index.columns_used(),
+        );
+
+        let mut group = c.benchmark_group(format!("bigram_query_{file_count}"));
+        group.sample_size(500);
+
+        let queries: &[(&str, &[u8])] = &[
+            ("short_2char", b"st"),
+            ("medium_6char", b"struct"),
+            ("long_14char", b"let controller"),
+            ("multi_word", b"fn process"),
+        ];
+
+        for (name, query) in queries {
+            group.bench_with_input(BenchmarkId::from_parameter(name), query, |b, q| {
+                b.iter(|| {
+                    let result = index.query(black_box(q));
+                    black_box(&result);
+                });
+            });
+        }
+
+        group.finish();
+    }
+}
+
+fn bench_bigram_is_candidate(c: &mut Criterion) {
+    let index = build_test_index(500_000);
+    let candidates = match index.query(b"struct") {
+        Some(c) => c,
+        None => {
+            // All bigrams ubiquitous at this size — skip candidate benches
+            eprintln!("Skipping is_candidate bench: query returned None (all bigrams ubiquitous)");
+            return;
+        }
+    };
+
+    c.bench_function("is_candidate_500k", |b| {
+        b.iter(|| {
+            let mut count = 0u32;
+            for i in 0..500_000 {
+                if BigramFilter::is_candidate(black_box(&candidates), i) {
+                    count += 1;
+                }
+            }
+            black_box(count)
+        });
+    });
+
+    c.bench_function("count_candidates_500k", |b| {
+        b.iter(|| BigramFilter::count_candidates(black_box(&candidates)));
+    });
+}
+
+fn bench_bigram_build(c: &mut Criterion) {
+    let mut group = c.benchmark_group("bigram_build");
+    group.sample_size(10);
+
+    let file_counts = [10_000, 100_000];
+
+    for &file_count in &file_counts {
+        // Pre-generate content so we only measure index building.
+        // Short content (~85 bytes/file) exercises the scalar fast path.
+        let contents: Vec<String> = (0..file_count)
+            .map(|i| {
+                format!(
+                    "struct File{i} {{ fn process() {{ let controller = read(path); }} }} // mod {i}"
+                )
+            })
+            .collect();
+
+        group.bench_with_input(
+            BenchmarkId::new("short_content", file_count),
+            &file_count,
+            |b, &fc| {
+                b.iter(|| {
+                    let builder = BigramIndexBuilder::new(fc);
+                    let skip_builder = BigramIndexBuilder::new(fc);
+                    for (i, content) in contents.iter().enumerate() {
+                        builder.add_file_content(&skip_builder, i, content.as_bytes());
+                    }
+                    let index = builder.compress(None);
+                    black_box(index.columns_used())
+                });
+            },
+        );
+
+        // Long content (~4 KB/file) exercises the SIMD pre-pass path.
+        // Build a realistic-looking source-like blob by repeating snippets.
+        let long_contents: Vec<String> = (0..file_count)
+            .map(|i| {
+                let mut s = String::with_capacity(4096);
+                for j in 0..50 {
+                    s.push_str(&format!(
+                        "pub fn handler_{i}_{j}(ctx: &Context) -> Result<Response, Error> {{\n"
+                    ));
+                    s.push_str("    let parsed = ctx.parse()?;\n");
+                    s.push_str("    let validated = parsed.validate()?;\n");
+                    s.push_str(&format!("    ctx.respond(validated, {}).await\n", j));
+                    s.push_str("}\n\n");
+                }
+                s
+            })
+            .collect();
+
+        group.bench_with_input(
+            BenchmarkId::new("long_content", file_count),
+            &file_count,
+            |b, &fc| {
+                b.iter(|| {
+                    let builder = BigramIndexBuilder::new(fc);
+                    let skip_builder = BigramIndexBuilder::new(fc);
+                    for (i, content) in long_contents.iter().enumerate() {
+                        builder.add_file_content(&skip_builder, i, content.as_bytes());
+                    }
+                    let index = builder.compress(None);
+                    black_box(index.columns_used())
+                });
+            },
+        );
+    }
+
+    group.finish();
+}
+
+criterion_group!(
+    benches,
+    bench_bigram_query,
+    bench_bigram_is_candidate,
+    bench_bigram_build,
+);
+
+criterion_main!(benches);

crates/fff-core/benches/glob_bench.rs

@@ -0,0 +1,380 @@
+//! Compare three glob-matching strategies for `match_glob_pattern` in constraints.rs
+//! need to make sure that my zlob changes are not affecting fff's internal globbing
+use ahash::AHashSet;
+use criterion::{BenchmarkId, Criterion, black_box, criterion_group, criterion_main};
+use zlob::{ZlobFlags, ZlobPattern, zlob_match_paths, zlob_match_paths_indices};
+
+fn make_paths(n: usize) -> Vec<String> {
+    let exts = ["rs", "ts", "lua", "md", "toml", "go", "py", "c", "h", "txt"];
+    let dirs = [
+        "src/core",
+        "src/ui",
+        "crates/fff-core/src",
+        "lua/fff",
+        "tests/integration",
+        "vendor/lib",
+        "node_modules/foo/bar",
+        "docs/internal",
+    ];
+    let mut out = Vec::with_capacity(n);
+    for i in 0..n {
+        let dir = dirs[i % dirs.len()];
+        let ext = exts[i % exts.len()];
+        out.push(format!("{dir}/file_{i}.{ext}"));
+    }
+    out
+}
+
+fn current_impl(pattern: &str, paths: &[&str]) -> AHashSet<usize> {
+    let Ok(Some(matches)) = zlob_match_paths(pattern, paths, ZlobFlags::RECOMMENDED) else {
+        return AHashSet::new();
+    };
+    let matched_set: AHashSet<usize> = matches.iter().map(|s| s.as_ptr() as usize).collect();
+    paths
+        .iter()
+        .enumerate()
+        .filter(|(_, p)| matched_set.contains(&(p.as_ptr() as usize)))
+        .map(|(i, _)| i)
+        .collect()
+}
+
+fn indices_free_fn(pattern: &str, paths: &[&str]) -> AHashSet<usize> {
+    let Ok(hits) = zlob_match_paths_indices(pattern, paths, ZlobFlags::RECOMMENDED) else {
+        return AHashSet::new();
+    };
+    hits.to_iter().collect()
+}
+
+fn compiled_pattern(pattern: &str, paths: &[&str]) -> AHashSet<usize> {
+    let Ok(p) = ZlobPattern::compile(pattern, ZlobFlags::RECOMMENDED) else {
+        return AHashSet::new();
+    };
+    let Ok(hits) = p.match_indices(paths, ZlobFlags::RECOMMENDED) else {
+        return AHashSet::new();
+    };
+    hits.to_iter().collect()
+}
+
+fn bench_glob_strategies(c: &mut Criterion) {
+    let path_counts = [1_000usize, 10_000, 100_000];
+    let patterns: &[(&str, &str)] = &[
+        ("ext_rs", "**/*.rs"),
+        ("dir_glob", "src/**/*.{ts,lua}"),
+        ("literal_seg", "**/node_modules/**"),
+        ("brace_multi", "**/*.{rs,ts,lua,md}"),
+    ];
+
+    for &count in &path_counts {
+        let owned = make_paths(count);
+        let paths: Vec<&str> = owned.iter().map(|s| s.as_str()).collect();
+
+        let mut group = c.benchmark_group(format!("glob_{count}"));
+        group.sample_size(50);
+
+        for &(name, pat) in patterns {
+            let id_curr = BenchmarkId::new("current_ptr_trick", name);
+            group.bench_with_input(id_curr, &pat, |b, &pat| {
+                b.iter(|| {
+                    let r = current_impl(black_box(pat), black_box(&paths));
+                    black_box(r);
+                });
+            });
+
+            let id_idx = BenchmarkId::new("match_indices_fn", name);
+            group.bench_with_input(id_idx, &pat, |b, &pat| {
+                b.iter(|| {
+                    let r = indices_free_fn(black_box(pat), black_box(&paths));
+                    black_box(r);
+                });
+            });
+
+            let id_comp = BenchmarkId::new("compiled_pattern", name);
+            group.bench_with_input(id_comp, &pat, |b, &pat| {
+                b.iter(|| {
+                    let r = compiled_pattern(black_box(pat), black_box(&paths));
+                    black_box(r);
+                });
+            });
+        }
+
+        group.finish();
+    }
+}
+
+/// Hot-loop: pattern compiled ONCE, matched many times against fresh path slices.
+/// Models a hypothetical change where we cache compiled patterns across calls.
+fn bench_compiled_reuse(c: &mut Criterion) {
+    let owned = make_paths(10_000);
+    let paths: Vec<&str> = owned.iter().map(|s| s.as_str()).collect();
+    let pat = "**/*.{rs,ts,lua,md}";
+
+    let mut group = c.benchmark_group("glob_reuse_10k");
+    group.sample_size(100);
+
+    group.bench_function("recompile_each_time", |b| {
+        b.iter(|| {
+            let p = ZlobPattern::compile(black_box(pat), ZlobFlags::RECOMMENDED).unwrap();
+            let hits = p
+                .match_indices(black_box(&paths), ZlobFlags::RECOMMENDED)
+                .unwrap();
+            black_box(hits.len());
+        });
+    });
+
+    let compiled = ZlobPattern::compile(pat, ZlobFlags::RECOMMENDED).unwrap();
+    group.bench_function("reuse_compiled", |b| {
+        b.iter(|| {
+            let hits = compiled
+                .match_indices(black_box(&paths), ZlobFlags::RECOMMENDED)
+                .unwrap();
+            black_box(hits.len());
+        });
+    });
+
+    group.finish();
+}
+
+/// End-to-end: build the lookup AND iterate items checking membership, modeling the
+/// real call shape in `apply_constraints` (filter loop reads the result for every item).
+fn bench_full_pipeline(c: &mut Criterion) {
+    bench_full_pipeline_size(c, 100_000);
+    bench_full_pipeline_size(c, 500_000);
+}
+
+fn bench_full_pipeline_size(c: &mut Criterion, count: usize) {
+    let owned = make_paths(count);
+    let paths: Vec<&str> = owned.iter().map(|s| s.as_str()).collect();
+    let pat = "**/*.{rs,ts,lua,md}";
+
+    let mut group = c.benchmark_group(format!("glob_full_pipeline_{count}"));
+    group.sample_size(50);
+
+    // (A) current: indices -> AHashSet -> per-item set.contains
+    group.bench_function("indices_to_ahashset_then_filter", |b| {
+        b.iter(|| {
+            let hits =
+                zlob_match_paths_indices(black_box(pat), &paths, ZlobFlags::RECOMMENDED).unwrap();
+            let set: AHashSet<usize> = hits.to_iter().collect();
+            let count = (0..paths.len()).filter(|i| set.contains(i)).count();
+            black_box(count);
+        });
+    });
+
+    // (B) indices -> Vec<bool> bitmap -> per-item array lookup
+    group.bench_function("indices_to_bitmap_then_filter", |b| {
+        b.iter(|| {
+            let hits =
+                zlob_match_paths_indices(black_box(pat), &paths, ZlobFlags::RECOMMENDED).unwrap();
+            let mut mask = vec![false; paths.len()];
+            for i in hits.to_iter() {
+                mask[i] = true;
+            }
+            let count = (0..paths.len()).filter(|&i| mask[i]).count();
+            black_box(count);
+        });
+    });
+
+    // (C) compiled pattern + per-item matches() inside the filter loop. No batch.
+    group.bench_function("compiled_per_item_matches", |b| {
+        b.iter(|| {
+            let p = ZlobPattern::compile(black_box(pat), ZlobFlags::RECOMMENDED).unwrap();
+            let count = paths.iter().filter(|path| p.matches_default(path)).count();
+            black_box(count);
+        });
+    });
+
+    // (D) compiled pattern + chunked batch -> Vec<bool> bitmap. Best of both:
+    //     SIMD batch wins inside chunks, no global allocation pressure, O(1) lookup.
+    group.bench_function("compiled_chunked_to_bitmap", |b| {
+        b.iter(|| {
+            let p = ZlobPattern::compile(black_box(pat), ZlobFlags::RECOMMENDED).unwrap();
+            let mut mask = vec![false; paths.len()];
+            for (chunk_idx, chunk) in paths.chunks(512).enumerate() {
+                let base = chunk_idx * 512;
+                let hits = p.match_indices(chunk, ZlobFlags::RECOMMENDED).unwrap();
+                for i in hits.to_iter() {
+                    mask[base + i] = true;
+                }
+            }
+            let count = (0..paths.len()).filter(|&i| mask[i]).count();
+            black_box(count);
+        });
+    });
+
+    // (E') indices -> bit-packed Vec<u64> -> per-item bit test
+    group.bench_function("indices_to_bitset_then_filter", |b| {
+        b.iter(|| {
+            let hits =
+                zlob_match_paths_indices(black_box(pat), &paths, ZlobFlags::RECOMMENDED).unwrap();
+            let words = paths.len().div_ceil(64);
+            let mut bits = vec![0u64; words];
+            for i in hits.to_iter() {
+                bits[i >> 6] |= 1u64 << (i & 63);
+            }
+            let count = (0..paths.len())
+                .filter(|&i| (bits[i >> 6] >> (i & 63)) & 1 == 1)
+                .count();
+            black_box(count);
+        });
+    });
+
+    // (E) (D) but larger chunk
+    group.bench_function("compiled_chunked_4096_to_bitmap", |b| {
+        b.iter(|| {
+            let p = ZlobPattern::compile(black_box(pat), ZlobFlags::RECOMMENDED).unwrap();
+            let mut mask = vec![false; paths.len()];
+            for (chunk_idx, chunk) in paths.chunks(4096).enumerate() {
+                let base = chunk_idx * 4096;
+                let hits = p.match_indices(chunk, ZlobFlags::RECOMMENDED).unwrap();
+                for i in hits.to_iter() {
+                    mask[base + i] = true;
+                }
+            }
+            let count = (0..paths.len()).filter(|&i| mask[i]).count();
+            black_box(count);
+        });
+    });
+
+    group.finish();
+}
+
+/// Mixed-constraint pipeline: glob + ext. Compare pre-pass batch (current) vs
+/// inline `ZlobPattern::matches` after the cheap ext check rejects items.
+///
+/// Variables: ext rejection rate. Extreme cases reveal where each strategy wins.
+fn bench_mixed_pipeline(c: &mut Criterion) {
+    let count = 100_000;
+    let owned = make_paths(count);
+    let paths: Vec<&str> = owned.iter().map(|s| s.as_str()).collect();
+    let glob_pat = "**/*.{rs,ts,lua,md}";
+
+    // 4 ext sets: from very selective (1/10 paths kept) to permissive (kept all).
+    let scenarios: &[(&str, &[&str])] = &[
+        ("ext_1of10", &["rs"]),
+        ("ext_4of10", &["rs", "ts", "lua", "md"]),
+        (
+            "ext_8of10",
+            &["rs", "ts", "lua", "md", "toml", "go", "py", "c"],
+        ),
+        (
+            "ext_all",
+            &["rs", "ts", "lua", "md", "toml", "go", "py", "c", "h", "txt"],
+        ),
+    ];
+
+    fn ext_match(name: &str, exts: &[&str]) -> bool {
+        exts.iter().any(|e| {
+            let bytes = name.as_bytes();
+            let elen = e.len();
+            bytes.len() > elen + 1
+                && bytes[bytes.len() - elen - 1] == b'.'
+                && bytes[bytes.len() - elen..].eq_ignore_ascii_case(e.as_bytes())
+        })
+    }
+
+    let mut group = c.benchmark_group("glob_mixed_100k");
+    group.sample_size(50);
+
+    for &(name, exts) in scenarios {
+        // (A) PRE-PASS: build bitmap for ALL paths, then per-item ext-then-bitmap.
+        let id_pre = BenchmarkId::new("prepass_bitmap", name);
+        group.bench_with_input(id_pre, &exts, |b, &exts| {
+            b.iter(|| {
+                let hits =
+                    zlob_match_paths_indices(black_box(glob_pat), &paths, ZlobFlags::RECOMMENDED)
+                        .unwrap();
+                let mut mask = vec![false; paths.len()];
+                for i in hits.to_iter() {
+                    mask[i] = true;
+                }
+                let count = paths
+                    .iter()
+                    .enumerate()
+                    .filter(|&(_, p)| ext_match(p, exts))
+                    .filter(|&(i, _)| mask[i])
+                    .count();
+                black_box(count);
+            });
+        });
+
+        // (B) INLINE: compile once, per-item ext check first, then matches() only on survivors.
+        let id_inline = BenchmarkId::new("inline_compiled", name);
+        group.bench_with_input(id_inline, &exts, |b, &exts| {
+            b.iter(|| {
+                let p = ZlobPattern::compile(black_box(glob_pat), ZlobFlags::RECOMMENDED).unwrap();
+                let count = paths
+                    .iter()
+                    .filter(|path| ext_match(path, exts))
+                    .filter(|path| p.matches_default(path))
+                    .count();
+                black_box(count);
+            });
+        });
+    }
+
+    group.finish();
+}
+
+/// Compare hand-rolled `file_has_extension` byte compare vs compiling extensions
+/// into a single brace glob `**/*.{rs,ts,lua,md}` and dispatching through zlob.
+/// Both share the same per-item "filter then count" shape.
+fn bench_extensions_vs_glob(c: &mut Criterion) {
+    let owned = make_paths(100_000);
+    let paths: Vec<&str> = owned.iter().map(|s| s.as_str()).collect();
+    let exts = ["rs", "ts", "lua", "md"];
+    let glob_pat = "**/*.{rs,ts,lua,md}";
+
+    fn ext_match(name: &str, exts: &[&str]) -> bool {
+        let bytes = name.as_bytes();
+        exts.iter().any(|e| {
+            let elen = e.len();
+            bytes.len() > elen + 1
+                && bytes[bytes.len() - elen - 1] == b'.'
+                && bytes[bytes.len() - elen..].eq_ignore_ascii_case(e.as_bytes())
+        })
+    }
+
+    let mut group = c.benchmark_group("ext_vs_glob_100k");
+    group.sample_size(50);
+
+    group.bench_function("file_has_extension_loop", |b| {
+        b.iter(|| {
+            let count = paths.iter().filter(|p| ext_match(p, &exts)).count();
+            black_box(count);
+        });
+    });
+
+    group.bench_function("compiled_brace_glob_inline", |b| {
+        b.iter(|| {
+            let p = ZlobPattern::compile(black_box(glob_pat), ZlobFlags::RECOMMENDED).unwrap();
+            let count = paths.iter().filter(|path| p.matches_default(path)).count();
+            black_box(count);
+        });
+    });
+
+    group.bench_function("brace_glob_prepass_bitmap", |b| {
+        b.iter(|| {
+            let hits =
+                zlob_match_paths_indices(black_box(glob_pat), &paths, ZlobFlags::RECOMMENDED)
+                    .unwrap();
+            let mut mask = vec![false; paths.len()];
+            for i in hits.to_iter() {
+                mask[i] = true;
+            }
+            let count = (0..paths.len()).filter(|&i| mask[i]).count();
+            black_box(count);
+        });
+    });
+
+    group.finish();
+}
+
+criterion_group!(
+    benches,
+    bench_glob_strategies,
+    bench_compiled_reuse,
+    bench_full_pipeline,
+    bench_mixed_pipeline,
+    bench_extensions_vs_glob
+);
+criterion_main!(benches);

crates/fff-core/benches/grep_bench.rs

@@ -0,0 +1,105 @@
+use criterion::{Criterion, criterion_group, criterion_main};
+use fff_search::file_picker::{FilePicker, FilePickerOptions};
+use fff_search::{GrepMode, GrepSearchOptions, parse_grep_query};
+use std::io::Write;
+
+/// Synthetic repo: half the files contain the needle on every line (stresses
+/// the per-match find/highlight path), half are pure noise (stresses the
+/// whole-file prefilter path).
+fn setup_repo(dir: &std::path::Path) {
+    for i in 0..400 {
+        let mut f = std::fs::File::create(dir.join(format!("match_{i}.rs"))).unwrap();
+        for j in 0..100 {
+            writeln!(
+                f,
+                "fn handle_{j}() {{ let controller = Controller::new({j}); controller.run(); }}"
+            )
+            .unwrap();
+        }
+    }
+    for i in 0..400 {
+        let mut f = std::fs::File::create(dir.join(format!("noise_{i}.rs"))).unwrap();
+        for j in 0..100 {
+            writeln!(
+                f,
+                "fn compute_{j}() {{ let value = {j} * 42; process(value); }}"
+            )
+            .unwrap();
+        }
+    }
+}
+
+fn options(mode: GrepMode) -> GrepSearchOptions {
+    GrepSearchOptions {
+        // Force a full scan of every file so we measure matcher/sink work,
+        // not pagination early-exit.
+        page_limit: usize::MAX,
+        max_matches_per_file: 0,
+        mode,
+        ..Default::default()
+    }
+}
+
+fn bench_grep(c: &mut Criterion) {
+    let dir = tempfile::tempdir().unwrap();
+    setup_repo(dir.path());
+
+    let mut picker = FilePicker::new(FilePickerOptions {
+        base_path: dir.path().to_str().unwrap().into(),
+        watch: false,
+        ..Default::default()
+    })
+    .unwrap();
+    picker.collect_files().unwrap();
+    assert_eq!(picker.get_files().len(), 800);
+
+    let mut group = c.benchmark_group("grep_e2e");
+    group.sample_size(30);
+
+    // Case-sensitive, 40k matched lines: hottest find_at/highlight path
+    let query = parse_grep_query("Controller");
+    let opts = options(GrepMode::PlainText);
+    group.bench_function("plain_case_sensitive_many_matches", |b| {
+        b.iter(|| {
+            let r = picker.grep(&query, &opts);
+            assert_eq!(r.files_with_matches, 400);
+            std::hint::black_box(r.matches.len())
+        });
+    });
+
+    // Case-insensitive (SIMD folding path), 120k matched spans
+    let query = parse_grep_query("controller");
+    group.bench_function("plain_case_insensitive_many_matches", |b| {
+        b.iter(|| {
+            let r = picker.grep(&query, &opts);
+            assert_eq!(r.files_with_matches, 400);
+            std::hint::black_box(r.matches.len())
+        });
+    });
+
+    // No matches anywhere: whole-file prefilter dominates
+    let query = parse_grep_query("Qqzyx");
+    group.bench_function("plain_no_matches", |b| {
+        b.iter(|| {
+            let r = picker.grep(&query, &opts);
+            assert_eq!(r.files_with_matches, 0);
+            std::hint::black_box(r.total_files_searched)
+        });
+    });
+
+    // Regex mode: must be unaffected by NeedleFinder changes
+    let query = parse_grep_query("Contr[a-z]+ller");
+    let regex_opts = options(GrepMode::Regex);
+    group.bench_function("regex_many_matches", |b| {
+        b.iter(|| {
+            let r = picker.grep(&query, &regex_opts);
+            assert_eq!(r.files_with_matches, 400);
+            std::hint::black_box(r.matches.len())
+        });
+    });
+
+    group.finish();
+}
+
+criterion_group!(benches, bench_grep);
+criterion_main!(benches);

crates/fff-core/benches/memmem_bench.rs

@@ -0,0 +1,85 @@
+use criterion::{BenchmarkId, Criterion, black_box, criterion_group, criterion_main};
+use fff_search::simd_string_utils::memmem;
+use std::path::Path;
+
+/// Load real source files from the repository as benchmark haystacks.
+/// Falls back to concatenating all .rs files under crates/ if specific files are missing.
+fn load_real_files() -> Vec<(&'static str, Vec<u8>)> {
+    let manifest_dir = env!("CARGO_MANIFEST_DIR"); // crates/fff-core
+    let repo_root = Path::new(manifest_dir).parent().unwrap().parent().unwrap();
+
+    let files: &[(&str, &str)] = &[
+        ("grep.rs/80KB", "crates/fff-core/src/grep.rs"),
+        ("file_picker.rs/53KB", "crates/fff-core/src/file_picker.rs"),
+        ("picker_ui.lua/96KB", "lua/fff/picker_ui.lua"),
+    ];
+
+    let mut result = Vec::new();
+    for &(label, rel_path) in files {
+        let full_path = repo_root.join(rel_path);
+        if let Ok(data) = std::fs::read(&full_path) {
+            result.push((label, data));
+        }
+    }
+
+    // Also create a large synthetic file by concatenating all three
+    if result.len() == 3 {
+        let mut combined = Vec::new();
+        for (_, data) in &result {
+            combined.extend_from_slice(data);
+        }
+        // Repeat to get ~1MB
+        let base = combined.clone();
+        while combined.len() < 1024 * 1024 {
+            combined.extend_from_slice(&base);
+        }
+        combined.truncate(1024 * 1024);
+        result.push(("combined/1MB", combined));
+    }
+
+    result
+}
+
+fn bench_memmem(c: &mut Criterion) {
+    let mut group = c.benchmark_group("simd_string_utils_memmem");
+
+    let files = load_real_files();
+    assert!(!files.is_empty(), "No source files found for benchmarking");
+
+    // Needles chosen to exercise different false-positive rates:
+    //
+    // "hit" needles: strings that actually appear in these source files.
+    // "miss" needles: strings with common first-bytes (lots of false positives
+    //   for memchr2) but that don't exist in any of the files.
+    let needles: &[(&str, &[u8])] = &[
+        // Hits — real identifiers from the codebase
+        ("short/hit/fn", b"fn"),
+        ("short/hit/self", b"self"),
+        ("medium/hit", b"search_file"),
+        ("long/hit", b"content_cache_budget"),
+        // Misses — common first-bytes, guaranteed not in source
+        ("short/miss", b"zqxjv"),
+        ("medium/miss", b"fluxcapacitor"),
+        ("long/miss", b"quantum_entanglement_resolver"),
+    ];
+
+    for (file_label, haystack) in &files {
+        for &(needle_label, needle) in needles {
+            let needle_lower: Vec<u8> = needle.iter().map(|b| b.to_ascii_lowercase()).collect();
+            let id = format!("{file_label}/{needle_label}");
+
+            group.bench_with_input(
+                BenchmarkId::new("find", &id),
+                &(haystack, &needle_lower),
+                |b, &(h, n)| {
+                    b.iter(|| black_box(memmem::find(h, n)));
+                },
+            );
+        }
+    }
+
+    group.finish();
+}
+
+criterion_group!(benches, bench_memmem);
+criterion_main!(benches);

crates/fff-core/benches/parse_bench.rs

@@ -0,0 +1,180 @@
+use criterion::{BenchmarkId, Criterion, Throughput, black_box, criterion_group, criterion_main};
+use fff_query_parser::*;
+
+fn bench_parse_simple(c: &mut Criterion) {
+    let parser = QueryParser::default();
+
+    c.bench_function("parse_simple_text", |b| {
+        b.iter(|| parser.parse(black_box("hello world")));
+    });
+
+    c.bench_function("parse_extension", |b| {
+        b.iter(|| parser.parse(black_box("*.rs")));
+    });
+
+    c.bench_function("parse_text_with_extension", |b| {
+        b.iter(|| parser.parse(black_box("name *.rs")));
+    });
+}
+
+fn bench_parse_complex(c: &mut Criterion) {
+    let parser = QueryParser::default();
+
+    c.bench_function("parse_complex_mixed", |b| {
+        b.iter(|| parser.parse(black_box("src name *.rs !test /lib/ status:modified")));
+    });
+
+    c.bench_function("parse_glob", |b| {
+        b.iter(|| parser.parse(black_box("**/*.rs")));
+    });
+
+    c.bench_function("parse_multiple_constraints", |b| {
+        b.iter(|| parser.parse(black_box("*.rs *.toml *.md !test !node_modules /src/")));
+    });
+}
+
+fn bench_parse_realistic_queries(c: &mut Criterion) {
+    let parser = QueryParser::default();
+
+    let queries = vec![
+        "file",
+        "test",
+        "mod.rs",
+        "src/*.rs",
+        "lib test",
+        "*.rs !test",
+        "src/lib/*.rs",
+        "/src/ name",
+        "status:modified *.rs",
+        "type:rust test !node_modules",
+    ];
+
+    let mut group = c.benchmark_group("realistic_queries");
+    for query in queries.iter() {
+        group.throughput(Throughput::Bytes(query.len() as u64));
+        group.bench_with_input(BenchmarkId::from_parameter(query), query, |b, q| {
+            b.iter(|| parser.parse(black_box(q)));
+        });
+    }
+    group.finish();
+}
+
+fn bench_parse_various_lengths(c: &mut Criterion) {
+    let parser = QueryParser::default();
+
+    let short = "*.rs";
+    let medium = "src name *.rs !test";
+    let long = "src lib test name *.rs *.toml !node_modules !test /src/ /lib/ status:modified";
+    let very_long =
+        "a b c d e f g h i j k l m n o p q r s t u v w x y z *.rs *.toml *.md *.txt *.js";
+
+    let mut group = c.benchmark_group("query_lengths");
+
+    group.throughput(Throughput::Bytes(short.len() as u64));
+    group.bench_with_input(BenchmarkId::new("short", short.len()), &short, |b, q| {
+        b.iter(|| parser.parse(black_box(q)));
+    });
+
+    group.throughput(Throughput::Bytes(medium.len() as u64));
+    group.bench_with_input(BenchmarkId::new("medium", medium.len()), &medium, |b, q| {
+        b.iter(|| parser.parse(black_box(q)));
+    });
+
+    group.throughput(Throughput::Bytes(long.len() as u64));
+    group.bench_with_input(BenchmarkId::new("long", long.len()), &long, |b, q| {
+        b.iter(|| parser.parse(black_box(q)));
+    });
+
+    group.throughput(Throughput::Bytes(very_long.len() as u64));
+    group.bench_with_input(
+        BenchmarkId::new("very_long", very_long.len()),
+        &very_long,
+        |b, q| {
+            b.iter(|| parser.parse(black_box(q)));
+        },
+    );
+
+    group.finish();
+}
+
+fn bench_config_comparison(c: &mut Criterion) {
+    let file_picker = QueryParser::new(FileSearchConfig);
+    let grep = QueryParser::new(GrepConfig);
+
+    let query = "src name *.rs !test";
+
+    let mut group = c.benchmark_group("config_comparison");
+
+    group.bench_function("file_picker_config", |b| {
+        b.iter(|| file_picker.parse(black_box(query)));
+    });
+
+    group.bench_function("grep_config", |b| {
+        b.iter(|| grep.parse(black_box(query)));
+    });
+
+    group.finish();
+}
+
+fn bench_constraint_types(c: &mut Criterion) {
+    let parser = QueryParser::default();
+
+    let mut group = c.benchmark_group("constraint_types");
+
+    group.bench_function("extension", |b| {
+        b.iter(|| parser.parse(black_box("*.rs")));
+    });
+
+    group.bench_function("glob", |b| {
+        b.iter(|| parser.parse(black_box("**/*.rs")));
+    });
+
+    group.bench_function("exclude", |b| {
+        b.iter(|| parser.parse(black_box("!test")));
+    });
+
+    group.bench_function("path_segment", |b| {
+        b.iter(|| parser.parse(black_box("/src/")));
+    });
+
+    group.bench_function("git_status", |b| {
+        b.iter(|| parser.parse(black_box("status:modified")));
+    });
+
+    group.bench_function("file_type", |b| {
+        b.iter(|| parser.parse(black_box("type:rust")));
+    });
+
+    group.finish();
+}
+
+fn bench_worst_case(c: &mut Criterion) {
+    let parser = QueryParser::default();
+
+    // Worst case: many constraints that all need to be checked
+    let worst_case = "a b c d e f g h i j k l m n o p q r s t u v w x y z";
+
+    c.bench_function("worst_case_many_text_tokens", |b| {
+        b.iter(|| parser.parse(black_box(worst_case)));
+    });
+
+    // Many constraints
+    let many_constraints = "*.rs *.toml *.md *.txt *.js *.ts *.jsx *.tsx *.vue *.svelte";
+
+    c.bench_function("worst_case_many_constraints", |b| {
+        b.iter(|| parser.parse(black_box(many_constraints)));
+    });
+}
+
+criterion_group!(
+    benches,
+    bench_parse_simple,
+    bench_parse_complex,
+    bench_parse_realistic_queries,
+    bench_parse_various_lengths,
+    bench_config_comparison,
+    bench_constraint_types,
+    bench_worst_case,
+);
+
+criterion_main!(benches);

crates/fff-core/build.rs

@@ -1,36 +1,45 @@
 fn main() {
+    // Opt-in cfg for the long-running randomized stress tests
+    // used by tests/fuzz_git_watcher_stress.rs
+    println!("cargo::rustc-check-cfg=cfg(stress)");
+
+    // Full-rescan accounting. Debug builds get it for free; a release build has
+    // to opt in with `--features rescan-stats` (what the rescan_probe needs).
+    println!("cargo::rustc-check-cfg=cfg(rescan_stats)");
+    if std::env::var("DEBUG").is_ok_and(|debug| debug != "false")
+        || std::env::var("CARGO_FEATURE_RESCAN_STATS").is_ok()
+    {
+        println!("cargo::rustc-cfg=rescan_stats");
+    }
+
     // When the `zlob` feature is enabled (Zig-compiled C library):
     // On Windows MSVC, explicitly link the C runtime libraries.
     // Zig-compiled static libraries don't emit /DEFAULTLIB directives for the
     // MSVC CRT, so symbols like strcmp, memcpy etc. would be unresolved.
     if std::env::var("CARGO_FEATURE_ZLOB").is_ok() {
+        if !zig_available() {
+            panic!(
+                "The `zlob` feature is enabled but Zig is not installed. \
+                 Install Zig (https://ziglang.org/download/) or build without \
+                 `--features zlob`."
+            );
+        }
+
         let target = std::env::var("TARGET").unwrap_or_default();
         if target.contains("windows") && target.contains("msvc") {
             println!("cargo:rustc-link-lib=msvcrt");
             println!("cargo:rustc-link-lib=ucrt");
             println!("cargo:rustc-link-lib=vcruntime");
         }
-    } else if std::env::var("CI").is_ok() {
-        // CI must always build with zlob for production-quality binaries.
-        if !zig_available() {
-            panic!(
-                "CI detected but Zig is not installed. \
-                 Please install Zig and build with `--features zlob`."
-            );
-        }
-        panic!(
-            "CI detected but `zlob` feature is not enabled. \
-             Build with `--features zlob`."
-        );
-    } else {
+    } else if std::env::var("CARGO_PRIMARY_PACKAGE").is_ok() && zig_available() {
         // Hint: if Zig is available but the zlob feature wasn't enabled,
         // let the developer know they can get faster glob matching.
-        if zig_available() {
-            println!(
-                "cargo:warning=Zig detected but `zlob` feature is not enabled. \
-                 Build with `--features zlob` for faster glob matching."
-            );
-        }
+        // Only emit this hint when this crate is the primary package to
+        // avoid noisy warnings for downstream consumers.
+        println!(
+            "cargo:warning=Zig detected but `zlob` feature is not enabled. \
+             Build with `--features zlob` for faster glob matching."
+        );
     }
 }
 
@@ -41,6 +50,5 @@ fn zig_available() -> bool {
         .stdout(std::process::Stdio::null())
         .stderr(std::process::Stdio::null())
         .status()
-        .map(|s| s.success())
-        .unwrap_or(false)
+        .is_ok()
 }

crates/fff-core/src/background_watcher.rs

@@ -1,604 +0,0 @@
-use crate::error::Error;
-use crate::file_picker::{FFFMode, FilePicker};
-use crate::git::GitStatusCache;
-use crate::sort_buffer::sort_with_buffer;
-use crate::{SharedFrecency, SharedPicker};
-use git2::Repository;
-use notify::event::{AccessKind, AccessMode};
-use notify::{Config, EventKind, RecursiveMode};
-use notify_debouncer_full::{DebounceEventResult, DebouncedEvent, NoCache, new_debouncer_opt};
-use std::path::{Path, PathBuf};
-use std::sync::Arc;
-use std::sync::atomic::{AtomicBool, Ordering};
-use std::time::Duration;
-use tracing::{Level, debug, error, info, warn};
-
-type Debouncer = notify_debouncer_full::Debouncer<notify::RecommendedWatcher, NoCache>;
-
-/// Owns the file-system watcher and guarantees that all background threads
-/// are fully joined before `stop()` / `Drop` returns.
-///
-/// Architecture:
-///   - The debouncer (and its internal watcher) live inside an **owner thread**
-///     that we spawn and hold the `JoinHandle` for.
-///   - `stop()` sets a flag, unparks the owner thread, and **joins** it.
-///   - Inside the owner thread, `Debouncer::stop()` is called which joins the
-///     debouncer's event-processing thread.
-///   - On Windows an additional short sleep is added after `Debouncer::stop()`
-///     because `notify`'s `ReadDirectoryChangesWatcher` discards its thread
-///     `JoinHandle`, so we cannot join it directly. The watcher's `Drop` does
-///     signal the thread via semaphore so it exits almost immediately, but we
-///     need to give the OS a moment to reclaim it.
-pub struct BackgroundWatcher {
-    stop_signal: Arc<AtomicBool>,
-    owner_thread: Option<std::thread::JoinHandle<()>>,
-}
-
-const DEBOUNCE_TIMEOUT: Duration = Duration::from_millis(250);
-const MAX_PATHS_THRESHOLD: usize = 1024;
-const MAX_SELECTIVE_WATCH_DIRS: usize = 100;
-/// Minimum seconds between frecency tracks of the same file in AI mode.
-/// Prevents score inflation from rapid burst edits by AI agents.
-const AI_MODE_COOLDOWN_SECS: u64 = 5 * 60;
-
-impl BackgroundWatcher {
-    pub fn new(
-        base_path: PathBuf,
-        git_workdir: Option<PathBuf>,
-        shared_picker: SharedPicker,
-        shared_frecency: SharedFrecency,
-        mode: FFFMode,
-    ) -> Result<Self, Error> {
-        info!(
-            "Initializing background watcher for path: {}, mode: {:?}",
-            base_path.display(),
-            mode,
-        );
-
-        let debouncer =
-            Self::create_debouncer(base_path, git_workdir, shared_picker, shared_frecency, mode)?;
-        info!("Background file watcher initialized successfully");
-
-        let stop_signal = Arc::new(AtomicBool::new(false));
-        let stop_clone = Arc::clone(&stop_signal);
-
-        // The owner thread keeps the debouncer alive and ensures proper
-        // cleanup: `Debouncer::stop()` joins its internal thread, then the
-        // watcher `Drop` signals its I/O thread to exit.
-        let owner_thread = std::thread::Builder::new()
-            .name("fff-watcher-owner".into())
-            .spawn(move || {
-                while !stop_clone.load(Ordering::Acquire) {
-                    std::thread::park_timeout(Duration::from_secs(1));
-                }
-                // Debouncer::stop() joins the debouncer's event thread, then
-                // drops the watcher (whose Drop signals the I/O thread).
-                debouncer.stop();
-                // On Windows the notify crate discards the ReadDirectoryChangesW
-                // thread's JoinHandle — we cannot join it. Its Drop signals the
-                // thread via semaphore so it exits almost immediately; give the
-                // OS a moment to fully reclaim it.
-                #[cfg(windows)]
-                std::thread::sleep(Duration::from_millis(250));
-            })
-            .expect("failed to spawn fff-watcher-owner thread");
-
-        Ok(Self {
-            stop_signal,
-            owner_thread: Some(owner_thread),
-        })
-    }
-
-    fn create_debouncer(
-        base_path: PathBuf,
-        git_workdir: Option<PathBuf>,
-        shared_picker: SharedPicker,
-        shared_frecency: SharedFrecency,
-        mode: FFFMode,
-    ) -> Result<Debouncer, Error> {
-        // do not follow symlinks as then notifiers spawns a bunch of events for symlinked
-        // files that could be git ignored, we have to property differentiate those and if
-        // the file was edited through a
-        let config = Config::default().with_follow_symlinks(false);
-
-        let git_workdir_for_handler = git_workdir.clone();
-        let mut debouncer = new_debouncer_opt(
-            DEBOUNCE_TIMEOUT,
-            Some(DEBOUNCE_TIMEOUT / 2), // tick rate for the event span
-            {
-                move |result: DebounceEventResult| match result {
-                    Ok(events) => {
-                        handle_debounced_events(
-                            events,
-                            &git_workdir_for_handler,
-                            &shared_picker,
-                            &shared_frecency,
-                            mode,
-                        );
-                    }
-                    Err(errors) => {
-                        error!("File watcher errors: {:?}", errors);
-                    }
-                }
-            },
-            // There is an issue with recommended cache implementation on macos
-            // it keeps track of all the files added to the watcher which is not a problem
-            // for us because any rename to the file will anyway require the removing from the
-            // ordedred index and adding it back with the new name
-            NoCache::new(),
-            config,
-        )?;
-
-        // Watch only non-ignored directories to avoid flooding the OS event buffer.
-        // On macOS, FSEvents has a fixed-size kernel buffer — watching huge gitignored
-        // directories like `target/` in rust causes buffer overflow, which drops real source file
-        // events. Instead we watch the root non-recursively (for top-level file changes
-        // and new directory detection) and each non-ignored subdirectory recursively.
-        let watch_dirs = collect_non_ignored_dirs(&base_path);
-
-        if watch_dirs.len() > MAX_SELECTIVE_WATCH_DIRS {
-            tracing::warn!(
-                "Too many non-ignored directories ({}/{}) can't efficiently watch them",
-                watch_dirs.len(),
-                MAX_SELECTIVE_WATCH_DIRS
-            );
-            debouncer.watch(base_path.as_path(), RecursiveMode::Recursive)?;
-        } else {
-            debouncer.watch(base_path.as_path(), RecursiveMode::NonRecursive)?;
-
-            for dir in &watch_dirs {
-                match debouncer.watch(dir.as_path(), RecursiveMode::Recursive) {
-                    Ok(()) => {}
-                    Err(e) => {
-                        // Non-fatal: directory may have been removed between discovery and watch
-                        warn!("Failed to watch directory {}: {}", dir.display(), e);
-                    }
-                }
-            }
-
-            // In selective mode the .git directory is excluded from the non-ignored
-            // dirs, but we still need to observe changes that affect git status
-            // (staging, unstaging, committing, branch switches, merges, etc.).
-            watch_git_status_paths(&mut debouncer, git_workdir.as_ref());
-        }
-
-        info!(
-            "File watcher initialized for {} directories under {}",
-            watch_dirs.len(),
-            base_path.display()
-        );
-
-        Ok(debouncer)
-    }
-
-    pub fn stop(&mut self) {
-        self.stop_signal.store(true, Ordering::Release);
-        if let Some(handle) = self.owner_thread.take() {
-            handle.thread().unpark();
-
-            if let Err(e) = handle.join() {
-                error!("Watcher owner thread panicked: {:?}", e);
-            }
-        }
-
-        info!("Background file watcher stopped successfully");
-    }
-}
-
-impl Drop for BackgroundWatcher {
-    fn drop(&mut self) {
-        self.stop();
-    }
-}
-
-#[tracing::instrument(name = "fs_events", skip(events, shared_picker, shared_frecency), level = Level::DEBUG)]
-fn handle_debounced_events(
-    events: Vec<DebouncedEvent>,
-    git_workdir: &Option<PathBuf>,
-    shared_picker: &SharedPicker,
-    shared_frecency: &SharedFrecency,
-    mode: FFFMode,
-) {
-    // this will be called very often, we have to minimiy the lock time for file picker
-    let repo = git_workdir.as_ref().and_then(|p| Repository::open(p).ok());
-    let mut need_full_rescan = false;
-    let mut need_full_git_rescan = false;
-    let mut paths_to_remove = Vec::new();
-    let mut paths_to_add_or_modify = Vec::new();
-    let mut affected_paths_count = 0usize;
-
-    for debounced_event in &events {
-        // It is very important to not react to the access errors because we inevitably
-        // gonna trigger the sync by our own preview or other unnecessary noise
-        if matches!(
-            debounced_event.event.kind,
-            EventKind::Access(
-                AccessKind::Read
-                    | AccessKind::Open(_)
-                    | AccessKind::Close(AccessMode::Read | AccessMode::Execute)
-            )
-        ) {
-            continue;
-        }
-
-        // When macOS FSEvents (or other backends) overflow their event buffer, the kernel
-        // drops individual events and emits a Rescan flag telling us to re-scan the subtree.
-        // Without handling this, modified source files can be silently missed.
-        if debounced_event.event.need_rescan() {
-            warn!(
-                "Received rescan event for paths {:?}, triggering full rescan",
-                debounced_event.event.paths
-            );
-            need_full_rescan = true;
-            break;
-        }
-
-        tracing::debug!(event = ?debounced_event.event, "Processing FS event");
-        for path in &debounced_event.event.paths {
-            if is_ignore_definition_path(path) {
-                info!(
-                    "Detected change in ignore definition file: {}",
-                    path.display()
-                );
-                need_full_rescan = true;
-                break;
-            }
-
-            if is_dotgit_change_affecting_status(path, &repo) {
-                need_full_git_rescan = true;
-            }
-
-            if is_git_file(path) {
-                continue;
-            }
-
-            // Use a combination of event kind and filesystem state to decide
-            // whether a path is an addition/modification or a removal.
-            //
-            // We cannot rely on `path.exists()` alone because:
-            //   - A freshly created file might not be visible yet (race).
-            //   - macOS FSEvents uses Modify(Name(Any)) for both rename-in
-            //     and rename-out, so we must stat the path to disambiguate.
-            //
-            // We cannot rely on event kind alone because:
-            //   - Remove events are not always emitted (macOS often sends
-            //     Modify(Name(Any)) instead of Remove).
-            let is_removal = matches!(debounced_event.event.kind, EventKind::Remove(_));
-
-            if is_removal || !path.exists() {
-                paths_to_remove.push(path.as_path());
-            } else {
-                // For additions/modifications, still filter gitignored files.
-                if should_include_file(path, &repo) {
-                    paths_to_add_or_modify.push(path.as_path());
-                }
-            }
-        }
-
-        affected_paths_count += debounced_event.event.paths.len();
-        if affected_paths_count > MAX_PATHS_THRESHOLD {
-            warn!(
-                "Too many affected paths ({}) in a single batch, triggering full rescan",
-                affected_paths_count
-            );
-
-            need_full_rescan = true;
-            break;
-        }
-
-        if need_full_rescan {
-            break;
-        }
-    }
-
-    if need_full_rescan {
-        info!(?affected_paths_count, "Triggering full rescan");
-        trigger_full_rescan(shared_picker, shared_frecency);
-        return;
-    }
-
-    // It's important to get the allocated sort
-    sort_with_buffer(paths_to_add_or_modify.as_mut_slice(), |a, b| {
-        a.as_os_str().cmp(b.as_os_str())
-    });
-    paths_to_add_or_modify.dedup_by(|a, b| a.as_os_str().eq(b.as_os_str()));
-
-    info!(
-        "Event processing summary: {} to remove, {} to add/modify",
-        paths_to_remove.len(),
-        paths_to_add_or_modify.len()
-    );
-
-    // Apply file index updates (add/remove) unconditionally — these must
-    // happen even when there is no git repository.
-    let files_to_update_git_status =
-        if !paths_to_remove.is_empty() || !paths_to_add_or_modify.is_empty() {
-            debug!(
-                "Applying file index changes: {} to remove, {} to add/modify",
-                paths_to_remove.len(),
-                paths_to_add_or_modify.len(),
-            );
-
-            let apply_changes = |picker: &mut FilePicker| -> Vec<PathBuf> {
-                for path in &paths_to_remove {
-                    let removed = picker.remove_file_by_path(path);
-                    debug!("remove_file_by_path({:?}) -> {}", path, removed);
-                }
-
-                let mut files_to_update = Vec::with_capacity(paths_to_add_or_modify.len());
-                for path in &paths_to_add_or_modify {
-                    let result = picker.on_create_or_modify(path);
-                    match result {
-                        Some(file) => {
-                            debug!(
-                                "on_create_or_modify({:?}) -> Some({})",
-                                path,
-                                file.path.display()
-                            );
-                            files_to_update.push(file.path.clone());
-                        }
-                        None => {
-                            error!("on_create_or_modify({:?}) -> None (file not added!)", path);
-                        }
-                    }
-                }
-                info!(
-                    "apply_changes complete: {} files to update git status",
-                    files_to_update.len()
-                );
-                files_to_update
-            };
-
-            let Ok(mut guard) = shared_picker.write() else {
-                error!("Failed to acquire file picker write lock");
-                return;
-            };
-            let Some(ref mut picker) = *guard else {
-                error!("File picker not initialized");
-                return;
-            };
-            apply_changes(picker)
-        } else {
-            debug!("No file index changes to apply");
-            Vec::new()
-        };
-
-    // AI mode: auto-track frecency for all modified/created files.
-    // Uses a 5-minute cooldown per file to prevent score inflation from rapid
-    // burst edits (AI agents often edit the same file many times in minutes).
-    // This runs after apply_changes so the picker write lock is released.
-    if mode.is_ai() && !paths_to_add_or_modify.is_empty() {
-        let mut tracked_count = 0usize;
-        if let Ok(frecency_guard) = shared_frecency.read()
-            && let Some(ref frecency) = *frecency_guard
-        {
-            for path in &paths_to_add_or_modify {
-                // Skip if this file was tracked less than 5 minutes ago
-                let should_track = match frecency.seconds_since_last_access(path) {
-                    Ok(Some(secs)) => secs >= AI_MODE_COOLDOWN_SECS,
-                    Ok(None) => true, // Never tracked before
-                    Err(_) => true,   // DB error, track anyway
-                };
-                if !should_track {
-                    continue;
-                }
-
-                if let Err(e) = frecency.track_access(path) {
-                    error!("Failed to track frecency for {:?}: {:?}", path, e);
-                } else {
-                    tracked_count += 1;
-                }
-            }
-            if tracked_count > 0 {
-                info!("AI mode: tracked frecency for {} files", tracked_count);
-            }
-        }
-
-        // Update in-memory frecency scores for tracked files
-        if tracked_count > 0
-            && let Ok(mut picker_guard) = shared_picker.write()
-            && let Some(ref mut picker) = *picker_guard
-            && let Ok(frecency_guard) = shared_frecency.read()
-            && let Some(ref frecency) = *frecency_guard
-        {
-            for path in &paths_to_add_or_modify {
-                let _ = picker.update_single_file_frecency(path, frecency);
-            }
-        }
-    }
-
-    // Git status updates require a repository.
-    let Some(repo) = repo.as_ref() else {
-        debug!("No git repo available, skipping git status updates");
-        return;
-    };
-
-    if need_full_git_rescan {
-        info!("Triggering full git rescan");
-
-        let result = FilePicker::refresh_git_status(shared_picker, shared_frecency);
-        if let Err(e) = result {
-            error!("Failed to refresh git status: {:?}", e);
-        }
-        return;
-    }
-
-    if !files_to_update_git_status.is_empty() {
-        info!(
-            "Fetching git status for {} files",
-            files_to_update_git_status.len()
-        );
-
-        let status = match GitStatusCache::git_status_for_paths(repo, &files_to_update_git_status) {
-            Ok(status) => status,
-            Err(e) => {
-                tracing::error!(?e, "Failed to query git status");
-                return;
-            }
-        };
-
-        if let Ok(mut guard) = shared_picker.write()
-            && let Some(ref mut picker) = *guard
-        {
-            if let Err(e) = picker.update_git_statuses(status, shared_frecency) {
-                error!("Failed to update git statuses: {:?}", e);
-            } else {
-                info!("Successfully updated git statuses in picker");
-            }
-        } else {
-            error!("Failed to acquire picker lock for git status update");
-        }
-    }
-}
-
-fn trigger_full_rescan(shared_picker: &SharedPicker, shared_frecency: &SharedFrecency) {
-    info!("Triggering full filesystem rescan");
-
-    // Note: no need to clear mmaps — they are backed by the kernel page cache
-    // and automatically reflect file changes. Old FileItems (and their mmaps)
-    // are dropped when the picker rebuilds its file list.
-
-    let Ok(mut guard) = shared_picker.write() else {
-        error!("Failed to acquire file picker write lock for full rescan");
-        return;
-    };
-    let Some(ref mut picker) = *guard else {
-        error!("File picker not initialized, cannot trigger rescan");
-        return;
-    };
-    if let Err(e) = picker.trigger_rescan(shared_frecency) {
-        error!("Failed to trigger full rescan: {:?}", e);
-    } else {
-        info!("Full filesystem rescan completed successfully");
-    }
-}
-
-fn should_include_file(path: &Path, repo: &Option<Repository>) -> bool {
-    // Directories are not indexed — only regular files (and symlinks to files).
-    if path.is_dir() {
-        return false;
-    }
-
-    // If there is a git repo, respect its ignore rules.
-    // If there is no repo (or the check fails), include the file.
-    match repo.as_ref() {
-        Some(repo) => repo.is_path_ignored(path) != Ok(true),
-        None => true,
-    }
-}
-
-#[inline]
-fn is_git_file(path: &Path) -> bool {
-    path.components()
-        .any(|component| component.as_os_str() == ".git")
-}
-
-pub fn is_dotgit_change_affecting_status(changed: &Path, repo: &Option<Repository>) -> bool {
-    let Some(repo) = repo.as_ref() else {
-        return false;
-    };
-
-    let git_dir = repo.path();
-
-    if let Ok(rel) = changed.strip_prefix(git_dir) {
-        if rel.starts_with("objects") || rel.starts_with("logs") || rel.starts_with("hooks") {
-            return false;
-        }
-        if rel == Path::new("index") || rel == Path::new("index.lock") {
-            return true;
-        }
-        if rel == Path::new("HEAD") {
-            return true;
-        }
-        if rel.starts_with("refs") || rel == Path::new("packed-refs") {
-            return true;
-        }
-        if rel == Path::new("info/exclude") || rel == Path::new("info/sparse-checkout") {
-            return true;
-        }
-
-        if let Some(fname) = rel.file_name().and_then(|f| f.to_str())
-            && matches!(fname, "MERGE_HEAD" | "CHERRY_PICK_HEAD" | "REVERT_HEAD")
-        {
-            return true;
-        }
-    }
-
-    false
-}
-
-fn is_ignore_definition_path(path: &Path) -> bool {
-    matches!(
-        path.file_name().and_then(|f| f.to_str()),
-        Some(".ignore") | Some(".gitignore")
-    )
-}
-
-fn watch_git_status_paths(debouncer: &mut Debouncer, git_workdir: Option<&PathBuf>) {
-    let Some(workdir) = git_workdir else {
-        return;
-    };
-
-    let git_dir = workdir.join(".git");
-    if !git_dir.is_dir() {
-        return;
-    }
-
-    // Watch .git/ non-recursively to catch top-level files:
-    // index, index.lock, HEAD, packed-refs, MERGE_HEAD, CHERRY_PICK_HEAD, REVERT_HEAD
-    if let Err(e) = debouncer.watch(&git_dir, RecursiveMode::NonRecursive) {
-        warn!("Failed to watch .git directory: {}", e);
-        return;
-    }
-
-    // Watch refs/ recursively to catch branch/tag changes
-    let refs_dir = git_dir.join("refs");
-    if refs_dir.is_dir()
-        && let Err(e) = debouncer.watch(&refs_dir, RecursiveMode::Recursive)
-    {
-        warn!("Failed to watch .git/refs: {}", e);
-    }
-
-    // Watch info/ non-recursively for exclude and sparse-checkout
-    let info_dir = git_dir.join("info");
-    if info_dir.is_dir()
-        && let Err(e) = debouncer.watch(&info_dir, RecursiveMode::NonRecursive)
-    {
-        warn!("Failed to watch .git/info: {}", e);
-    }
-}
-
-/// Collects immediate non-ignored subdirectories of `base_path` using the `ignore` crate
-/// to respect .gitignore, .ignore, and global gitignore rules. This is used to set up
-/// selective file watching — only non-ignored directories get a recursive watcher,
-/// preventing gitignored directories like `target/` from flooding the OS event buffer.
-fn collect_non_ignored_dirs(base_path: &Path) -> Vec<PathBuf> {
-    use ignore::WalkBuilder;
-
-    let walker = WalkBuilder::new(base_path)
-        .hidden(false)
-        .git_ignore(true)
-        .git_exclude(true)
-        .git_global(true)
-        .ignore(true)
-        .follow_links(false)
-        .max_depth(Some(1))
-        .build();
-
-    let mut dirs = Vec::new();
-    for entry in walker {
-        let Ok(entry) = entry else { continue };
-        let path = entry.path();
-
-        // Skip the root directory itself
-        if path == base_path {
-            continue;
-        }
-
-        if path.is_dir() && !is_git_file(path) {
-            dirs.push(path.to_path_buf());
-        }
-    }
-
-    dirs
-}

crates/fff-core/src/constants.rs

@@ -0,0 +1,48 @@
+/// Largest file whose full content fff will touch: the default grep read cap
+/// (`GrepSearchOptions::max_file_size`) and the content-cache mmap cap
+/// (`ContentCacheBudget::max_file_size`). Binary detection also streams up to
+/// this far so nothing grep would read is left unclassified.
+pub const MAX_FFFILE_SIZE: u64 = 10 * 1024 * 1024;
+
+/// Upper bound on a file the bigram builder will build, if the file is very large there is a
+/// big probability it will only bloat the available bigrams and will anyway pop ut from the prefilter
+pub const MAX_INDEXABLE_FILE_SIZE: usize = 2 * 1024 * 1024;
+
+/// Total bytes the persistent content mmap cache may hold for a small repo.
+pub const MAX_CACHED_CONTENT_BYTES: u64 = 512 * 1024 * 1024;
+
+/// Files below one page waste the remainder when mmapped, so the cache skips
+/// them and falls back to chunked reads. Unused on Windows (no content cache)
+#[cfg(all(not(target_os = "windows"), target_arch = "aarch64"))]
+pub const MMAP_THRESHOLD: u64 = 16 * 1024;
+#[cfg(all(not(target_os = "windows"), not(target_arch = "aarch64")))]
+pub const MMAP_THRESHOLD: u64 = 4 * 1024;
+
+/// Watcher overflow capacity reserved after the initial scan
+pub const MAX_OVERFLOW_FILES: usize = 1024;
+
+/// Minimum delay between watcher-initiated rescans.
+pub const RESCAN_MIN_INTERVAL: std::time::Duration = std::time::Duration::from_secs(30);
+
+/// Rescan delay for large indexes.
+pub const RESCAN_MIN_INTERVAL_LARGE_INDEX: std::time::Duration =
+    std::time::Duration::from_secs(5 * 60);
+
+/// Live-file count at which [`RESCAN_MIN_INTERVAL_LARGE_INDEX`] takes over.
+pub const LARGE_INDEX_FILE_COUNT: usize = 1_000_000;
+
+/// Fresh-mmap threshold: files at or above this size get mmapped directly on
+/// cache miss instead of chunked reads into Vec. Empirically tuned per-platform.
+/// Only referenced on Unix; Windows uses the `std::fs::read` fallback so this
+/// constant is gated to non-Windows targets to keep `-D unused-imports` happy.
+#[cfg(target_os = "macos")]
+pub const FRESH_MMAP_THRESHOLD: u64 = 1024 * 1024;
+#[cfg(all(not(target_os = "macos"), not(target_os = "windows")))]
+pub const FRESH_MMAP_THRESHOLD: u64 = 256 * 1024;
+
+// we do not support 32kb path limit on windows
+#[cfg(target_os = "windows")]
+pub const PATH_BUF_SIZE: usize = 4096;
+
+#[cfg(not(target_os = "windows"))]
+pub const PATH_BUF_SIZE: usize = libc::PATH_MAX as usize;

crates/fff-core/src/constraints.rs

@@ -1,449 +0,0 @@
-//! Constraint filtering engine for fff.
-//!
-//! This module provides the core constraint application logic that filters items
-//! based on parsed query constraints (extensions, path segments, globs, git status, etc.).
-//!
-//! The filtering is generic over the [`Constrainable`] trait, allowing reuse across
-//! different search modes (file picker, live grep, etc.).
-
-use ahash::AHashSet;
-use fff_query_parser::{Constraint, GitStatusFilter};
-use smallvec::SmallVec;
-
-use crate::git::is_modified_status;
-
-/// Minimum item count before switching to parallel iteration with rayon.
-/// Below this threshold, the overhead of thread pool dispatch outweighs the benefit.
-const PAR_THRESHOLD: usize = 10_000;
-
-/// Trait for items that can be filtered by constraints.
-/// Implement this for any searchable item type (files, grep results, etc.).
-pub trait Constrainable {
-    /// The file's relative path (e.g. "src/main.rs")
-    fn relative_path(&self) -> &str;
-
-    /// The file's lowercased relative path for case-insensitive matching
-    fn relative_path_lower(&self) -> &str;
-
-    /// The file name component (e.g. "main.rs")
-    fn file_name(&self) -> &str;
-
-    /// The git status of this item, if available
-    fn git_status(&self) -> Option<git2::Status>;
-}
-
-/// Check if a relative path ends with the given suffix at a `/` boundary (case-insensitive).
-///
-/// Returns `true` when the path equals the suffix or the character before the suffix
-/// in the path is `/`. This ensures partial directory-name matches are rejected.
-///
-/// Examples:
-/// - `path_ends_with_suffix("libswscale/input.c", "libswscale/input.c")` → true (exact)
-/// - `path_ends_with_suffix("foo/libswscale/input.c", "libswscale/input.c")` → true (suffix)
-/// - `path_ends_with_suffix("xlibswscale/input.c", "libswscale/input.c")` → false (no boundary)
-#[inline]
-pub fn path_ends_with_suffix(path: &str, suffix: &str) -> bool {
-    if path.len() < suffix.len() {
-        return false;
-    }
-    let start = path.len() - suffix.len();
-    if !path[start..].eq_ignore_ascii_case(suffix) {
-        return false;
-    }
-    // Exact match, or the character before is /
-    start == 0 || path.as_bytes()[start - 1] == b'/'
-}
-
-/// Check if file extension matches (without allocation)
-#[inline]
-pub fn file_has_extension(file_name: &str, ext: &str) -> bool {
-    if file_name.len() <= ext.len() + 1 {
-        return false;
-    }
-    let start = file_name.len() - ext.len() - 1;
-    file_name.as_bytes().get(start) == Some(&b'.')
-        && file_name[start + 1..].eq_ignore_ascii_case(ext)
-}
-
-/// Check if path contains segment (without allocation)
-/// Supports both single segments ("src") and multi-segment paths ("libswscale/aarch64").
-/// For "libswscale/aarch64", checks that these appear as consecutive path components.
-#[inline]
-pub fn path_contains_segment(path: &str, segment: &str) -> bool {
-    let path_bytes = path.as_bytes();
-    let segment_len = segment.len();
-
-    // Check segment/ at start of path
-    if path.len() > segment_len
-        && path_bytes.get(segment_len) == Some(&b'/')
-        && path[..segment_len].eq_ignore_ascii_case(segment)
-    {
-        return true;
-    }
-
-    // Check /segment/ anywhere using byte scanning
-    if path.len() < segment_len + 2 {
-        return false;
-    }
-
-    for i in 0..path.len().saturating_sub(segment_len + 1) {
-        if path_bytes[i] == b'/' {
-            let start = i + 1;
-            let end = start + segment_len;
-            if end < path.len()
-                && path_bytes[end] == b'/'
-                && path[start..end].eq_ignore_ascii_case(segment)
-            {
-                return true;
-            }
-        }
-    }
-    false
-}
-
-/// Check if an item at given index matches a constraint (single-pass friendly, allocation-free)
-#[inline]
-fn item_matches_constraint_at_index<T: Constrainable>(
-    item: &T,
-    item_index: usize,
-    constraint: &Constraint<'_>,
-    glob_results: &[(bool, AHashSet<usize>)],
-    glob_idx: &mut usize,
-    negate: bool,
-) -> bool {
-    let matches = match constraint {
-        Constraint::Extension(ext) => file_has_extension(item.file_name(), ext),
-        Constraint::Glob(_) => {
-            let result = glob_results
-                .get(*glob_idx)
-                .map(|(is_neg, set)| {
-                    let matched = set.contains(&item_index);
-                    if *is_neg { !matched } else { matched }
-                })
-                .unwrap_or(true);
-            *glob_idx += 1;
-            return if negate { !result } else { result };
-        }
-        Constraint::PathSegment(segment) => path_contains_segment(item.relative_path(), segment),
-        Constraint::FilePath(suffix) => path_ends_with_suffix(item.relative_path(), suffix),
-        Constraint::GitStatus(status_filter) => match (item.git_status(), status_filter) {
-            (Some(status), GitStatusFilter::Modified) => is_modified_status(status),
-            (Some(status), GitStatusFilter::Untracked) => status.contains(git2::Status::WT_NEW),
-            (Some(status), GitStatusFilter::Staged) => status.intersects(
-                git2::Status::INDEX_NEW
-                    | git2::Status::INDEX_MODIFIED
-                    | git2::Status::INDEX_DELETED
-                    | git2::Status::INDEX_RENAMED
-                    | git2::Status::INDEX_TYPECHANGE,
-            ),
-            (Some(status), GitStatusFilter::Unmodified) => status.is_empty(),
-            (None, GitStatusFilter::Unmodified) => true,
-            (None, _) => false,
-        },
-        Constraint::Not(inner) => {
-            return item_matches_constraint_at_index(
-                item,
-                item_index,
-                inner,
-                glob_results,
-                glob_idx,
-                !negate,
-            );
-        }
-
-        // only works with negation
-        Constraint::Text(text) => item.relative_path_lower().contains(text),
-
-        // Parts and Exclude are handled at a higher level
-        Constraint::Parts(_) | Constraint::Exclude(_) | Constraint::FileType(_) => true,
-    };
-
-    if negate { !matches } else { matches }
-}
-
-/// Apply constraint-based prefiltering in a single pass over all items.
-/// Returns `None` if no constraints are present, `Some(filtered)` otherwise.
-/// Multiple extension constraints (*.rs *.ts) are combined with OR logic.
-/// All other constraints are combined with AND logic.
-///
-/// Uses parallel iteration via rayon when the item count exceeds [`PAR_THRESHOLD`].
-pub fn apply_constraints<'a, T: Constrainable + Sync>(
-    items: &'a [T],
-    constraints: &[Constraint<'_>],
-) -> Option<Vec<&'a T>> {
-    if constraints.is_empty() {
-        return None;
-    }
-
-    // Separate extension constraints from other constraints — they use OR logic
-    let mut extensions: SmallVec<[&str; 8]> = SmallVec::new();
-    let mut other_constraints: SmallVec<[&Constraint<'_>; 8]> = SmallVec::new();
-
-    for constraint in constraints {
-        match constraint {
-            Constraint::Extension(ext) => extensions.push(ext),
-            _ => other_constraints.push(constraint),
-        }
-    }
-
-    // Only collect paths if we have glob constraints (expensive)
-    let has_globs = other_constraints
-        .iter()
-        .any(|c| matches!(c, Constraint::Glob(_) | Constraint::Not(_)));
-
-    let glob_results = if has_globs {
-        let paths: Vec<&str> = items.iter().map(|f| f.relative_path()).collect();
-        precompute_glob_matches(&other_constraints, &paths)
-    } else {
-        Vec::new()
-    };
-
-    let matches_constraints = |i: usize, item: &T| -> bool {
-        if !extensions.is_empty()
-            && !extensions
-                .iter()
-                .any(|ext| file_has_extension(item.file_name(), ext))
-        {
-            return false;
-        }
-
-        let mut glob_idx = 0;
-        other_constraints.iter().all(|constraint| {
-            item_matches_constraint_at_index(
-                item,
-                i,
-                constraint,
-                &glob_results,
-                &mut glob_idx,
-                false,
-            )
-        })
-    };
-
-    let filtered: Vec<&T> = if items.len() >= PAR_THRESHOLD {
-        use rayon::prelude::*;
-        items
-            .par_iter()
-            .enumerate()
-            .filter(|(i, item)| matches_constraints(*i, item))
-            .map(|(_, item)| item)
-            .collect()
-    } else {
-        items
-            .iter()
-            .enumerate()
-            .filter(|(i, item)| matches_constraints(*i, item))
-            .map(|(_, item)| item)
-            .collect()
-    };
-
-    Some(filtered)
-}
-
-fn precompute_glob_matches<'a>(
-    constraints: &[&Constraint<'a>],
-    paths: &[&str],
-) -> Vec<(bool, AHashSet<usize>)> {
-    let mut results = Vec::new();
-    for constraint in constraints {
-        collect_glob_indices(constraint, paths, &mut results, false);
-    }
-    results
-}
-
-fn collect_glob_indices<'a>(
-    constraint: &Constraint<'a>,
-    paths: &[&str],
-    results: &mut Vec<(bool, AHashSet<usize>)>,
-    is_negated: bool,
-) {
-    match constraint {
-        Constraint::Glob(pattern) => {
-            let indices = match_glob_pattern(pattern, paths);
-            results.push((is_negated, indices));
-        }
-        Constraint::Not(inner) => {
-            collect_glob_indices(inner, paths, results, !is_negated);
-        }
-        _ => {}
-    }
-}
-
-/// Match a glob pattern against a list of paths, returning the set of matching indices.
-///
-/// When the `zlob` feature is enabled, delegates to `zlob::zlob_match_paths` (Zig-compiled
-/// C library, fastest). Otherwise falls back to `globset::Glob` (pure Rust).
-#[cfg(feature = "zlob")]
-fn match_glob_pattern(pattern: &str, paths: &[&str]) -> AHashSet<usize> {
-    let Ok(Some(matches)) = zlob::zlob_match_paths(pattern, paths, zlob::ZlobFlags::RECOMMENDED)
-    else {
-        return AHashSet::new();
-    };
-
-    let matched_set: AHashSet<usize> = matches.iter().map(|s| s.as_ptr() as usize).collect();
-
-    if paths.len() >= PAR_THRESHOLD {
-        use rayon::prelude::*;
-        paths
-            .par_iter()
-            .enumerate()
-            .filter(|(_, p)| matched_set.contains(&(p.as_ptr() as usize)))
-            .map(|(i, _)| i)
-            .collect::<Vec<_>>()
-            .into_iter()
-            .collect()
-    } else {
-        paths
-            .iter()
-            .enumerate()
-            .filter(|(_, p)| matched_set.contains(&(p.as_ptr() as usize)))
-            .map(|(i, _)| i)
-            .collect()
-    }
-}
-
-#[cfg(not(feature = "zlob"))]
-fn match_glob_pattern(pattern: &str, paths: &[&str]) -> AHashSet<usize> {
-    let Ok(glob) = globset::Glob::new(pattern) else {
-        return AHashSet::new();
-    };
-    let matcher = glob.compile_matcher();
-
-    if paths.len() >= PAR_THRESHOLD {
-        use rayon::prelude::*;
-        paths
-            .par_iter()
-            .enumerate()
-            .filter(|(_, p)| matcher.is_match(p))
-            .map(|(i, _)| i)
-            .collect::<Vec<_>>()
-            .into_iter()
-            .collect()
-    } else {
-        paths
-            .iter()
-            .enumerate()
-            .filter(|(_, p)| matcher.is_match(p))
-            .map(|(i, _)| i)
-            .collect()
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_file_has_extension() {
-        assert!(file_has_extension("file.rs", "rs"));
-        assert!(file_has_extension("file.RS", "rs")); // case-insensitive
-        assert!(file_has_extension("file.test.rs", "rs"));
-        assert!(file_has_extension("a.rs", "rs"));
-
-        assert!(!file_has_extension("file.tsx", "rs"));
-        assert!(!file_has_extension("rs", "rs")); // too short
-        assert!(!file_has_extension(".rs", "rs")); // just extension
-        assert!(!file_has_extension("file.rsx", "rs")); // different extension
-        assert!(!file_has_extension("filers", "rs")); // no dot
-    }
-
-    #[test]
-    fn test_path_contains_segment() {
-        // Segment at start
-        assert!(path_contains_segment("src/lib.rs", "src"));
-        assert!(path_contains_segment("SRC/lib.rs", "src")); // case-insensitive
-
-        // Segment in middle
-        assert!(path_contains_segment("app/src/lib.rs", "src"));
-        assert!(path_contains_segment("app/SRC/lib.rs", "src"));
-
-        // Multiple levels
-        assert!(path_contains_segment("core/workflow/src/main.rs", "src"));
-        assert!(path_contains_segment(
-            "core/workflow/src/main.rs",
-            "workflow"
-        ));
-        assert!(path_contains_segment("core/workflow/src/main.rs", "core"));
-
-        // Should not match partial segments
-        assert!(!path_contains_segment("source/lib.rs", "src"));
-        assert!(!path_contains_segment("mysrc/lib.rs", "src"));
-
-        // Should not match filename
-        assert!(!path_contains_segment("lib/src", "src"));
-
-        // Multi-segment constraints
-        assert!(path_contains_segment(
-            "libswscale/aarch64/input.S",
-            "libswscale/aarch64"
-        ));
-        assert!(path_contains_segment(
-            "foo/libswscale/aarch64/input.S",
-            "libswscale/aarch64"
-        ));
-        assert!(path_contains_segment(
-            "foo/LibSwscale/AArch64/input.S",
-            "libswscale/aarch64"
-        )); // case-insensitive
-        assert!(!path_contains_segment(
-            "xlibswscale/aarch64/input.S",
-            "libswscale/aarch64"
-        )); // partial match at start
-        assert!(!path_contains_segment(
-            "foo/libswscale/aarch64x/input.S",
-            "libswscale/aarch64"
-        )); // partial match at end
-        assert!(path_contains_segment(
-            "crates/fff-core/src/grep.rs",
-            "fff-core/src"
-        ));
-
-        // Edge cases
-        assert!(!path_contains_segment("", "src"));
-        assert!(!path_contains_segment("src", "src")); // no trailing slash
-    }
-
-    #[test]
-    fn test_path_ends_with_suffix() {
-        // Exact match
-        assert!(path_ends_with_suffix(
-            "libswscale/input.c",
-            "libswscale/input.c"
-        ));
-
-        // Suffix match at / boundary
-        assert!(path_ends_with_suffix(
-            "foo/libswscale/input.c",
-            "libswscale/input.c"
-        ));
-
-        // Deep nesting
-        assert!(path_ends_with_suffix(
-            "a/b/c/libswscale/input.c",
-            "libswscale/input.c"
-        ));
-
-        // No boundary — partial directory name
-        assert!(!path_ends_with_suffix(
-            "xlibswscale/input.c",
-            "libswscale/input.c"
-        ));
-
-        // Case insensitive
-        assert!(path_ends_with_suffix(
-            "foo/LibSwscale/Input.C",
-            "libswscale/input.c"
-        ));
-
-        // Single file name
-        assert!(path_ends_with_suffix("input.c", "input.c"));
-        assert!(!path_ends_with_suffix("xinput.c", "input.c"));
-
-        // Suffix longer than path
-        assert!(!path_ends_with_suffix("input.c", "foo/input.c"));
-
-        // Simple path
-        assert!(path_ends_with_suffix("src/main.rs", "src/main.rs"));
-        assert!(path_ends_with_suffix("crates/src/main.rs", "src/main.rs"));
-    }
-}

crates/fff-core/src/dbs/db_healthcheck.rs

@@ -9,16 +9,23 @@ pub struct DbHealth {
     pub disk_size: u64,
     /// Entry counts by table name
     pub entry_counts: Vec<(&'static str, u64)>,
+    /// Set to `false` if can not acquire the write lock
+    pub healthy: bool,
 }
 
 pub trait DbHealthChecker {
-    fn get_env(&self) -> &heed::Env;
+    fn get_env(&self) -> &heed::Env<heed::WithoutTls>;
+    fn is_healthy(&self) -> bool;
+    /// Entries per database, each group has a static string label
     fn count_entries(&self) -> Result<Vec<(&'static str, u64)>>;
 
+    /// Health summary of the database, returns summary struct
     fn get_health(&self) -> Result<DbHealth> {
         let env = self.get_env();
 
-        let size = env.real_disk_size().map_err(crate::error::Error::EnvOpen)?;
+        let size = env
+            .real_disk_size()
+            .map_err(crate::error::Error::GenericDbError)?;
         let path = env.path().to_string_lossy().to_string();
         let entry_counts = self.count_entries()?;
 
@@ -26,6 +33,7 @@ pub trait DbHealthChecker {
             path,
             disk_size: size,
             entry_counts,
+            healthy: self.is_healthy(),
         })
     }
 }

crates/fff-core/src/dbs/env_pool.rs

@@ -0,0 +1,284 @@
+use heed::{Env, EnvOpenOptions, WithoutTls};
+use std::collections::HashMap;
+use std::fs;
+use std::ops::Deref;
+use std::path::{Path, PathBuf};
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::{Arc, LazyLock, Mutex, MutexGuard, PoisonError, Weak};
+use std::thread;
+use std::time::Duration;
+
+use crate::error::{Error, Result};
+use crate::lmdb::DbHealth;
+
+pub(crate) struct EnvSpec {
+    pub label: &'static str,
+    pub map_size: usize,
+    pub max_dbs: u32,
+    pub size_cap_bytes: u64,
+}
+
+pub(crate) struct PooledEnv {
+    env: Env<WithoutTls>,
+    key: PathBuf,
+    /// lmdb's env spec label
+    label: &'static str,
+    map_size: usize,
+    max_dbs: u32,
+    health: DbHealth,
+    gc_started: AtomicBool,
+    dbi_lock: Mutex<()>,
+}
+
+impl Drop for PooledEnv {
+    fn drop(&mut self) {
+        let mut pool = POOL.lock().unwrap_or_else(PoisonError::into_inner);
+        // Only remove a dead entry: begin_exclusive_destroy may have removed ours.
+        if pool.get(&self.key).is_some_and(|w| w.strong_count() == 0) {
+            pool.remove(&self.key);
+        }
+        // heed closes the env right after this body; a concurrent reopen of the
+        // same path rides out that gap via env_closing_event in get_or_open.
+    }
+}
+
+// Cloneable handle to a process-shared LMDB env, derefs to `heed::Env`.
+#[derive(Clone)]
+pub(crate) struct SharedEnv(Arc<PooledEnv>);
+
+impl Deref for SharedEnv {
+    type Target = Env<WithoutTls>;
+    fn deref(&self) -> &Env<WithoutTls> {
+        &self.0.env
+    }
+}
+
+impl std::fmt::Debug for SharedEnv {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_tuple("SharedEnv").field(&self.0.env).finish()
+    }
+}
+
+impl SharedEnv {
+    pub(crate) fn get_or_open(db_path: &Path, spec: &EnvSpec) -> Result<Self> {
+        fs::create_dir_all(db_path).map_err(Error::CreateDir)?;
+        let path = fs::canonicalize(db_path).map_err(|e| Error::EnvOpen {
+            db: spec.label,
+            source: heed::Error::Io(e),
+        })?;
+
+        let mut close_waits = 0u32;
+        let mut transient_retries = 0u32;
+
+        loop {
+            let mut open_failed = false;
+
+            {
+                let mut pool = POOL.lock().unwrap_or_else(PoisonError::into_inner);
+                if let Some(existing) = pool.get(&path).and_then(Weak::upgrade) {
+                    drop(pool);
+                    if existing.label != spec.label
+                        || existing.map_size != spec.map_size
+                        || existing.max_dbs != spec.max_dbs
+                    {
+                        return Err(Error::EnvSpecMismatch {
+                            path,
+                            open_as: existing.label,
+                            requested_as: spec.label,
+                        });
+                    }
+                    return Ok(Self(existing));
+                }
+
+                erase_if_oversized(&path, spec);
+                let result = unsafe {
+                    // MDB_NOTLS: reader slots are tied to txn objects (freed on
+                    // commit/abort) instead of pinned per thread for its lifetime (#783).
+                    let mut opts = EnvOpenOptions::new().read_txn_without_tls();
+                    opts.map_size(spec.map_size);
+                    opts.max_readers(max_readers());
+                    if spec.max_dbs > 0 {
+                        opts.max_dbs(spec.max_dbs);
+                    }
+                    opts.open(&path)
+                };
+
+                match result {
+                    Ok(env) => {
+                        let entry = Arc::new(PooledEnv {
+                            env,
+                            key: path.clone(),
+                            label: spec.label,
+                            map_size: spec.map_size,
+                            max_dbs: spec.max_dbs,
+                            health: DbHealth::new(),
+                            gc_started: AtomicBool::new(false),
+                            dbi_lock: Mutex::new(()),
+                        });
+                        pool.insert(path.clone(), Arc::downgrade(&entry));
+                        drop(pool);
+                        let shared = Self(entry);
+
+                        match shared.clear_stale_readers() {
+                            Ok(cleared_count) if cleared_count > 0 => {
+                                tracing::info!(
+                                    cleared_count,
+                                    db = spec.label,
+                                    "reclaimed stale LMDB reader slots at open"
+                                );
+                            }
+                            Ok(_) => {}
+                            Err(e) => {
+                                tracing::debug!("clear_stale_readers at open failed: {e}")
+                            }
+                        }
+
+                        return Ok(shared);
+                    }
+                    Err(heed::Error::EnvAlreadyOpened) => open_failed = true,
+                    // special handling cause we know this happens randomly
+                    Err(e)
+                        if is_transient_env_open_error(&e)
+                            && transient_retries < MAX_TRANSIENT_RETRIES =>
+                    {
+                        transient_retries += 1;
+                        tracing::debug!(
+                            path = %path.display(),
+                            transient_retries,
+                            error = ?e,
+                            "transient LMDB env open error, retrying"
+                        );
+                    }
+                    Err(e) => {
+                        return Err(Error::EnvOpen {
+                            db: spec.label,
+                            source: e,
+                        });
+                    }
+                }
+            }
+
+            if open_failed {
+                close_waits += 1;
+                if close_waits > MAX_CLOSE_WAITS {
+                    return Err(Error::EnvOpen {
+                        db: spec.label,
+                        source: heed::Error::EnvAlreadyOpened,
+                    });
+                }
+
+                match heed::env_closing_event(&path) {
+                    Some(event) => {
+                        event.wait_timeout(CLOSE_WAIT);
+                    }
+                    None => thread::sleep(Duration::from_millis(2)),
+                }
+            } else {
+                thread::sleep(TRANSIENT_RETRY_SLEEP);
+            }
+        }
+    }
+
+    pub(crate) fn health(&self) -> &DbHealth {
+        &self.0.health
+    }
+
+    // First caller wins: GC runs once per opened env, not once per tracker.
+    pub(crate) fn try_start_gc(&self) -> bool {
+        !self.0.gc_started.swap(true, Ordering::AcqRel)
+    }
+
+    // LMDB forbids mdb_dbi_open from concurrent txns in the same process.
+    pub(crate) fn lock_dbi_open(&self) -> MutexGuard<'_, ()> {
+        self.0
+            .dbi_lock
+            .lock()
+            .unwrap_or_else(PoisonError::into_inner)
+    }
+
+    pub(crate) fn destroy(&self) -> Result<Option<heed::EnvClosingEvent>> {
+        let mut pool = POOL.lock().unwrap_or_else(PoisonError::into_inner);
+        let holders = Arc::strong_count(&self.0);
+
+        if holders > 1 {
+            return Err(Error::DbInUse {
+                db: self.0.label,
+                path: self.0.key.clone(),
+                holders: holders - 1,
+            });
+        }
+
+        pool.remove(&self.0.key);
+        Ok(heed::env_closing_event(&self.0.key))
+    }
+}
+
+static POOL: LazyLock<Mutex<HashMap<PathBuf, Weak<PooledEnv>>>> = LazyLock::new(Mutex::default);
+
+const CLOSE_WAIT: Duration = Duration::from_millis(100);
+const MAX_CLOSE_WAITS: u32 = 100;
+const TRANSIENT_RETRY_SLEEP: Duration = Duration::from_millis(50);
+const MAX_TRANSIENT_RETRIES: u32 = 8;
+
+// Concurrent mdb_env_open calls on the same path can race on macOS
+// this is for some reason fixable by simple retry of the open
+// heed's default reader table is 126 slots. In TLS mode each thread pins a slot
+// for its lifetime, so long-lived embedders (Neovim, node agents) that share one
+// lock file across many processes/threads exhaust it (#783). Reader slots are
+// tiny (~64B), so raise the ceiling; `FFF_LMDB_MAX_READERS` lets hosts tune it.
+const DEFAULT_MAX_READERS: u32 = 1024;
+
+fn max_readers() -> u32 {
+    parse_max_readers(std::env::var("FFF_LMDB_MAX_READERS").ok())
+}
+
+// Never drop below heed's default 126; ignore missing/garbage/too-small values.
+fn parse_max_readers(raw: Option<String>) -> u32 {
+    raw.and_then(|v| v.trim().parse::<u32>().ok())
+        .filter(|&n| n >= 126)
+        .unwrap_or(DEFAULT_MAX_READERS)
+}
+
+fn is_transient_env_open_error(err: &heed::Error) -> bool {
+    match err {
+        heed::Error::Io(io) => matches!(
+            io.kind(),
+            std::io::ErrorKind::InvalidInput | std::io::ErrorKind::NotFound
+        ),
+        _ => false,
+    }
+}
+
+fn erase_if_oversized(db_path: &Path, spec: &EnvSpec) {
+    let data = db_path.join("data.mdb");
+    let Ok(meta) = fs::metadata(&data) else {
+        return;
+    };
+
+    if meta.len() <= spec.size_cap_bytes {
+        return;
+    }
+
+    tracing::error!(
+        path = %db_path.display(),
+        size = meta.len(),
+        cap = spec.size_cap_bytes,
+        "LMDB db exceeds size cap, erasing"
+    );
+    let _ = fs::remove_file(&data);
+    let _ = fs::remove_file(db_path.join("lock.mdb"));
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{DEFAULT_MAX_READERS, parse_max_readers};
+
+    #[test]
+    fn max_readers_parsing() {
+        assert_eq!(parse_max_readers(None), DEFAULT_MAX_READERS);
+        assert_eq!(parse_max_readers(Some("nan".into())), DEFAULT_MAX_READERS);
+        assert_eq!(parse_max_readers(Some("64".into())), DEFAULT_MAX_READERS); // below 126 floor
+        assert_eq!(parse_max_readers(Some(" 512 ".into())), 512);
+        assert_eq!(parse_max_readers(Some("126".into())), 126);
+    }
+}

crates/fff-core/src/dbs/frecency.rs

@@ -1,20 +1,18 @@
-use crate::db_healthcheck::DbHealthChecker;
+use super::db_healthcheck::DbHealthChecker;
+use super::env_pool::SharedEnv;
 use crate::error::{Error, Result};
 use crate::file_picker::FFFMode;
-use crate::{SharedFrecency, git::is_modified_status};
-use heed::{Database, Env, EnvOpenOptions};
-use heed::{
-    EnvFlags,
-    types::{Bytes, SerdeBincode},
-};
-use std::fs;
-use std::path::PathBuf;
+use crate::git::is_modified_status;
+use crate::lmdb::{DbHealth, LmdbStore, is_map_full};
+use heed::Database;
+use heed::types::{Bytes, SerdeBincode};
 use std::time::{SystemTime, UNIX_EPOCH};
 use std::{collections::VecDeque, path::Path};
 
 const DECAY_CONSTANT: f64 = 0.0693; // ln(2)/10 for 10-day half-life
 const SECONDS_PER_DAY: f64 = 86400.0;
 const MAX_HISTORY_DAYS: f64 = 30.0; // Only consider accesses within 30 days
+const MAX_TIMESTAMPS_PER_FILE: usize = 128;
 
 // AI mode: faster decay since AI sessions are shorter and more intense
 const AI_DECAY_CONSTANT: f64 = 0.231; // ln(2)/3 for 3-day half-life
@@ -22,8 +20,9 @@ const AI_MAX_HISTORY_DAYS: f64 = 7.0; // Only consider accesses within 7 days
 
 #[derive(Debug)]
 pub struct FrecencyTracker {
-    env: Env,
+    env: SharedEnv,
     db: Database<Bytes, SerdeBincode<VecDeque<u64>>>,
+    health: DbHealth,
 }
 
 const MODIFICATION_THRESHOLDS: [(i64, u64); 5] = [
@@ -44,222 +43,117 @@ const AI_MODIFICATION_THRESHOLDS: [(i64, u64); 5] = [
 ];
 
 impl DbHealthChecker for FrecencyTracker {
-    fn get_env(&self) -> &heed::Env {
+    fn get_env(&self) -> &heed::Env<heed::WithoutTls> {
         &self.env
     }
 
+    fn is_healthy(&self) -> bool {
+        self.health.is_healthy()
+    }
+
     fn count_entries(&self) -> Result<Vec<(&'static str, u64)>> {
-        let rtxn = self.env.read_txn().map_err(Error::DbStartReadTxn)?;
-        let count = self.db.len(&rtxn).map_err(Error::DbRead)?;
+        let rtxn = self
+            .env
+            .read_txn()
+            .map_err(|source| Error::DbStartReadTxn {
+                db: Self::LABEL,
+                source,
+            })?;
+        let count = self.db.len(&rtxn).map_err(|source| Error::DbRead {
+            db: Self::LABEL,
+            source,
+        })?;
 
         Ok(vec![("absolute_frecency_entries", count)])
     }
 }
 
-impl FrecencyTracker {
-    pub fn new(db_path: impl AsRef<Path>, use_unsafe_no_lock: bool) -> Result<Self> {
-        let db_path = db_path.as_ref();
-        fs::create_dir_all(db_path).map_err(Error::CreateDir)?;
+impl LmdbStore for FrecencyTracker {
+    const LABEL: &'static str = "frecency";
+    // 10 MiB hard ceiling. Owner's db after years of use is ~560 KiB, so this
+    // leaves ~18× headroom while capping runaway growth (see GH issue #437).
+    const MAP_SIZE: usize = 10 * 1024 * 1024;
+    const MAX_DBS: u32 = 0;
+    // Nuke the db when it exceeds 8 MiB on disk — leaves a small margin under
+    // MAP_SIZE so we don't hit MDB_MAP_FULL before the open-time erase fires.
+    const SIZE_CAP_BYTES: u64 = 12 * 1024 * 1024;
 
-        let env = unsafe {
-            let mut opts = EnvOpenOptions::new();
-            opts.map_size(24 * 1024 * 1024); // 24 MiB
-            if use_unsafe_no_lock {
-                opts.flags(EnvFlags::NO_LOCK | EnvFlags::NO_SYNC | EnvFlags::NO_META_SYNC);
-            }
-            opts.open(db_path).map_err(Error::EnvOpen)?
-        };
-        env.clear_stale_readers()
-            .map_err(Error::DbClearStaleReaders)?;
-
-        // we will open the default unnamed database
-        let mut wtxn = env.write_txn().map_err(Error::DbStartWriteTxn)?;
-        let db = env
-            .create_database(&mut wtxn, None)
-            .map_err(Error::DbCreate)?;
-
-        Ok(FrecencyTracker {
-            db,
-            env: env.clone(),
-        })
+    fn shared_env(&self) -> &SharedEnv {
+        &self.env
     }
 
-    /// Spawns a background thread to purge stale frecency entries and compact the database.
-    /// Run it once in a while to purge old pages and keep DB file size reasonable.
-    ///
-    /// It's okay to not join this thread since it acquires locks for the db access
-    ///
-    /// ```
-    /// use fff_search::frecency::FrecencyTracker;
-    /// use fff_search::SharedFrecency;
-    /// let shared_frecency: SharedFrecency = Default::default();
-    /// let _ = FrecencyTracker::spawn_gc(shared_frecency, "/path/to/frecency_db".into(), true).ok();
-    /// ```
-    pub fn spawn_gc(
-        shared: SharedFrecency,
-        db_path: String,
-        use_unsafe_no_lock: bool,
-    ) -> Result<std::thread::JoinHandle<()>> {
-        Ok(std::thread::Builder::new()
-            .name("fff-frecency-gc".into())
-            .spawn(move || Self::run_frecency_gc(shared, db_path, use_unsafe_no_lock))?)
+    fn health(&self) -> &DbHealth {
+        &self.health
     }
 
-    #[tracing::instrument(skip(shared), fields(db_path = %db_path))]
-    fn run_frecency_gc(shared: SharedFrecency, db_path: String, use_unsafe_no_lock: bool) {
-        let start = std::time::Instant::now();
-        let data_path = PathBuf::from(&db_path).join("data.mdb");
-
-        // Phase 1: Purge stale entries.
-        // The RwLock protects the Option<FrecencyTracker> (not the DB itself),
-        // so a read lock is sufficient — LMDB handles its own write serialization.
-        let (deleted, pruned) = {
-            let guard = match shared.read() {
-                Ok(g) => g,
-                Err(e) => {
-                    tracing::debug!("Failed to acquire read lock: {e}");
-                    return;
-                }
-            };
-            let Some(ref tracker) = *guard else {
-                return;
-            };
-            match tracker.purge_stale_entries() {
-                Ok(result) => result,
-                Err(e) => {
-                    tracing::debug!("Purge failed: {e}");
-                    return;
-                }
-            }
-        };
-
+    fn purge_stale_data(env: &SharedEnv) -> Result<()> {
+        let (deleted, pruned) = Self::purge_stale_entries(env)?;
         if deleted > 0 || pruned > 0 {
-            tracing::info!(deleted, pruned, elapsed = ?start.elapsed(), "Frecency GC purged entries");
+            tracing::info!(deleted, pruned, "Frecency GC purged entries");
         }
 
-        // Compact if we purged entries OR the file has significant freelist bloat
-        let file_size = fs::metadata(&data_path).map(|m| m.len()).unwrap_or(0);
-        if deleted == 0 && pruned == 0 && file_size <= 512 * 1024 {
-            return;
-        }
+        Ok(())
+    }
+}
 
-        // Phase 2: Manual compaction under a single write lock
-        let mut guard = match shared.write() {
-            Ok(g) => g,
-            Err(e) => {
-                tracing::debug!("Failed to acquire write lock: {e}");
-                return;
-            }
-        };
+impl FrecencyTracker {
+    /// Returns the on-disk path of the LMDB environment directory.
+    pub fn db_path(&self) -> &Path {
+        self.env.path()
+    }
 
-        // Read all entries from current env
-        let entries: Vec<(Vec<u8>, VecDeque<u64>)> = match guard.as_ref() {
-            Some(tracker) => {
-                let rtxn = match tracker.env.read_txn() {
-                    Ok(t) => t,
-                    Err(e) => {
-                        tracing::debug!("Compaction read_txn failed: {e}");
-                        return;
-                    }
-                };
-                let iter = match tracker.db.iter(&rtxn) {
-                    Ok(i) => i,
-                    Err(e) => {
-                        tracing::debug!("Compaction iter failed: {e}");
-                        return;
-                    }
-                };
-                let mut entries = Vec::new();
-                let mut read_errors = 0u32;
-                for result in iter {
-                    match result {
-                        Ok((key, value)) => entries.push((key.to_vec(), value)),
-                        Err(_) => read_errors += 1,
-                    }
-                }
-                if read_errors > 0 {
-                    tracing::warn!(
-                        read_errors,
-                        "Skipped corrupted entries during compaction read"
-                    );
-                }
-                entries
-            }
-            None => return,
-        };
+    pub fn open(db_path: impl AsRef<Path>) -> Result<Self> {
+        let db_path = db_path.as_ref();
+        let (env, health) = Self::open_env(db_path)?;
 
-        // Drop old tracker, delete files, create fresh env, write back
-        *guard = None;
+        let db = Self::open_database_safe(&env, None)?;
+        Ok(FrecencyTracker { db, env, health })
+    }
 
-        let lock_path = PathBuf::from(&db_path).join("lock.mdb");
-        let _ = fs::remove_file(&data_path);
-        let _ = fs::remove_file(&lock_path);
-
-        let tracker = match FrecencyTracker::new(&db_path, use_unsafe_no_lock) {
-            Ok(t) => t,
-            Err(e) => {
-                tracing::error!("Compaction reopen failed, frecency disabled: {e}");
-                return;
-            }
-        };
-
-        let write_result = (|| -> std::result::Result<(), heed::Error> {
-            let mut wtxn = tracker.env.write_txn()?;
-            for (key, value) in &entries {
-                tracker.db.put(&mut wtxn, key.as_slice(), value)?;
-            }
-            wtxn.commit()?;
-            Ok(())
-        })();
-
-        match write_result {
-            Ok(()) => {
-                let new_size = fs::metadata(&data_path).map(|m| m.len()).unwrap_or(0);
-                *guard = Some(tracker);
-                tracing::debug!(
-                    entries = entries.len(),
-                    old_size = file_size,
-                    new_size,
-                    elapsed = ?start.elapsed(),
-                    "Frecency DB compacted"
-                );
-            }
-            Err(e) => {
-                tracing::error!("Compaction write failed, frecency data may be incomplete: {e}");
-                *guard = Some(tracker);
-            }
-        }
+    #[deprecated(
+        since = "0.7.0",
+        note = "LMDB unsafe no-lock mode is no longer supported; use `FrecencyTracker::open` instead. \
+                The `_use_unsafe_no_lock` argument is ignored."
+    )]
+    pub fn new(db_path: impl AsRef<Path>, _use_unsafe_no_lock: bool) -> Result<Self> {
+        Self::open(db_path)
     }
 
     /// Removes entries where all timestamps are older than MAX_HISTORY_DAYS,
     /// and prunes stale timestamps from entries that still have recent ones.
     /// Returns (deleted_count, pruned_count).
-    fn purge_stale_entries(&self) -> Result<(usize, usize)> {
-        let now = self.get_now();
+    fn purge_stale_entries(env: &SharedEnv) -> Result<(usize, usize)> {
+        let now = SystemTime::now()
+            .duration_since(UNIX_EPOCH)
+            .unwrap()
+            .as_secs();
         let cutoff_time = now.saturating_sub((MAX_HISTORY_DAYS * SECONDS_PER_DAY) as u64);
 
-        // Collect entries to delete or update
-        let rtxn = self.env.read_txn().map_err(Error::DbStartReadTxn)?;
+        let db: Database<Bytes, SerdeBincode<VecDeque<u64>>> = Self::open_database_safe(env, None)?;
+
+        let rtxn = env.read_txn().map_err(|source| Error::DbStartReadTxn {
+            db: Self::LABEL,
+            source,
+        })?;
         let mut to_delete: Vec<Vec<u8>> = Vec::new();
         let mut to_update: Vec<(Vec<u8>, VecDeque<u64>)> = Vec::new();
 
-        let iter = self.db.iter(&rtxn).map_err(Error::DbRead)?;
+        let iter = db.iter(&rtxn).map_err(|source| Error::DbRead {
+            db: Self::LABEL,
+            source,
+        })?;
         for result in iter {
-            let (key, accesses) = result.map_err(Error::DbRead)?;
+            let (key, accesses) = result.map_err(|source| Error::DbRead {
+                db: Self::LABEL,
+                source,
+            })?;
 
-            // Timestamps are chronologically ordered (oldest at front).
-            // Find the first timestamp that is still within the retention window.
+            // Timestamps chronologically ordered (oldest at front).
             let fresh_start = accesses.iter().position(|&ts| ts >= cutoff_time);
             match fresh_start {
-                None => {
-                    // All timestamps are stale — delete the entire entry
-                    to_delete.push(key.to_vec());
-                }
-                Some(0) => {
-                    // All timestamps are fresh — nothing to do
-                }
+                None => to_delete.push(key.to_vec()),
+                Some(0) => {}
                 Some(start) => {
-                    // Some timestamps are stale — keep only the fresh ones
                     let pruned: VecDeque<u64> = accesses.iter().skip(start).copied().collect();
                     to_update.push((key.to_vec(), pruned));
                 }
@@ -271,26 +165,56 @@ impl FrecencyTracker {
             return Ok((0, 0));
         }
 
-        // Apply all changes in a single write transaction
-        let mut wtxn = self.env.write_txn().map_err(Error::DbStartWriteTxn)?;
+        let mut wtxn = env.write_txn().map_err(|source| Error::DbStartWriteTxn {
+            db: Self::LABEL,
+            source,
+        })?;
+
         for key in &to_delete {
-            self.db.delete(&mut wtxn, key).map_err(Error::DbWrite)?;
+            db.delete(&mut wtxn, key).map_err(|source| Error::DbWrite {
+                db: Self::LABEL,
+                source,
+            })?;
         }
+
         for (key, accesses) in &to_update {
-            self.db
-                .put(&mut wtxn, key, accesses)
-                .map_err(Error::DbWrite)?;
+            db.put(&mut wtxn, key, accesses)
+                .map_err(|source| Error::DbWrite {
+                    db: Self::LABEL,
+                    source,
+                })?;
         }
-        wtxn.commit().map_err(Error::DbCommit)?;
+        wtxn.commit().map_err(|source| Error::DbCommit {
+            db: Self::LABEL,
+            source,
+        })?;
 
         Ok((to_delete.len(), to_update.len()))
     }
 
     fn get_accesses(&self, path: &Path) -> Result<Option<VecDeque<u64>>> {
-        let rtxn = self.env.read_txn().map_err(Error::DbStartReadTxn)?;
-
         let key_hash = Self::path_to_hash_bytes(path)?;
-        self.db.get(&rtxn, &key_hash).map_err(Error::DbRead)
+
+        let rtxn = self
+            .env
+            .read_txn()
+            .map_err(|source| Error::DbStartReadTxn {
+                db: Self::LABEL,
+                source,
+            })?;
+        let result = self
+            .db
+            .get(&rtxn, &key_hash)
+            .map_err(|source| Error::DbRead {
+                db: Self::LABEL,
+                source,
+            })?;
+        rtxn.commit().map_err(|source| Error::DbCommit {
+            db: Self::LABEL,
+            source,
+        })?;
+
+        Ok(result)
     }
 
     fn get_now(&self) -> u64 {
@@ -301,6 +225,16 @@ impl FrecencyTracker {
     }
 
     fn path_to_hash_bytes(path: &Path) -> Result<[u8; 32]> {
+        // On Windows, resolve to the canonical form (short-name/case/symlink)
+        // so the same file always hashes to one key regardless of how the
+        // caller spelled it. Falls back to the raw path when the file no
+        // longer exists (e.g. watcher delete events), so the op is never
+        // dropped. No-op on other platforms.
+        #[cfg(windows)]
+        let canonical: Option<std::path::PathBuf> = crate::path_utils::canonicalize(path).ok();
+        #[cfg(windows)]
+        let path: &Path = canonical.as_deref().unwrap_or(path);
+
         let Some(key) = path.to_str() else {
             return Err(Error::InvalidPath(path.to_path_buf()));
         };
@@ -316,16 +250,22 @@ impl FrecencyTracker {
         Ok(last.map(|ts| self.get_now().saturating_sub(ts)))
     }
 
-    pub fn track_access(&self, path: &Path) -> Result<()> {
-        let mut wtxn = self.env.write_txn().map_err(Error::DbStartWriteTxn)?;
+    /// Number of tracked access for file path
+    pub fn access_count(&self, path: &Path) -> Result<usize> {
+        Ok(self.get_accesses(path)?.map_or(0, |a| a.len()))
+    }
 
+    pub fn track_access(&self, path: &Path) -> Result<()> {
         let key_hash = Self::path_to_hash_bytes(path)?;
         let mut accesses = self.get_accesses(path)?.unwrap_or_default();
 
         let now = self.get_now();
         let cutoff_time = now.saturating_sub((MAX_HISTORY_DAYS * SECONDS_PER_DAY) as u64);
+
+        // Drop stale timestamps from the front while also enforcing the
+        // per-file cap. Reserves one slot for the `push_back` below.
         while let Some(&front_time) = accesses.front() {
-            if front_time < cutoff_time {
+            if front_time < cutoff_time || accesses.len() >= MAX_TIMESTAMPS_PER_FILE {
                 accesses.pop_front();
             } else {
                 break;
@@ -335,13 +275,43 @@ impl FrecencyTracker {
         accesses.push_back(now);
         tracing::debug!(?path, accesses = accesses.len(), "Tracking access");
 
-        self.db
-            .put(&mut wtxn, &key_hash, &accesses)
-            .map_err(Error::DbWrite)?;
+        let mut wtxn = self
+            .env
+            .write_txn()
+            .map_err(|source| Error::DbStartWriteTxn {
+                db: Self::LABEL,
+                source,
+            })?;
+        if let Err(e) = self.db.put(&mut wtxn, &key_hash, &accesses) {
+            if is_map_full(&e) {
+                self.health.mark_unhealthy("MDB_MAP_FULL on put");
+                tracing::error!(
+                    ?path,
+                    "Frecency DB hit MDB_MAP_FULL; dropping write — db will be \
+                     erased on next open via LmdbStore::erase_if_oversized"
+                );
+                return Ok(());
+            }
+            return Err(Error::DbWrite {
+                db: Self::LABEL,
+                source: e,
+            });
+        }
 
-        wtxn.commit().map_err(Error::DbCommit)?;
-
-        Ok(())
+        wtxn.commit()
+            .inspect_err(|e| {
+                if is_map_full(e) {
+                    self.health.mark_unhealthy("MDB_MAP_FULL on commit");
+                    tracing::error!(
+                        ?path,
+                        "Frecency DB hit MDB_MAP_FULL on commit; dropping write"
+                    );
+                }
+            })
+            .map_err(|source| Error::DbCommit {
+                db: Self::LABEL,
+                source,
+            })
     }
 
     pub fn get_access_score(&self, file_path: &Path, mode: FFFMode) -> i64 {
@@ -441,6 +411,15 @@ mod tests {
     use super::*;
     use crate::file_picker::FFFMode;
 
+    // A path that doesn't exist on disk must still hash (canonicalize fails on
+    // Windows → falls back to the raw string), so watcher delete events and
+    // raced files never drop their frecency op.
+    #[test]
+    fn hashes_nonexistent_path_without_error() {
+        let missing = Path::new("/this/path/definitely/does/not/exist/frecency_test_xyz");
+        assert!(FrecencyTracker::path_to_hash_bytes(missing).is_ok());
+    }
+
     fn calculate_test_frecency_score(access_timestamps: &[u64], current_time: u64) -> i64 {
         let mut total_frecency = 0.0;
 
@@ -510,7 +489,7 @@ mod tests {
     fn test_modification_score_interpolation() {
         let temp_dir = std::env::temp_dir().join("fff_test_interpolation");
         let _ = std::fs::remove_dir_all(&temp_dir);
-        let tracker = FrecencyTracker::new(temp_dir.to_str().unwrap(), true).unwrap();
+        let tracker = FrecencyTracker::open(temp_dir.to_str().unwrap()).unwrap();
 
         let current_time = tracker.get_now();
         let git_status = Some(git2::Status::WT_MODIFIED);

crates/fff-core/src/dbs/lmdb.rs

@@ -0,0 +1,197 @@
+use heed::{Database, Env, WithoutTls};
+use std::path::Path;
+use std::sync::Arc;
+use std::sync::RwLock;
+use std::sync::atomic::{AtomicU8, Ordering};
+use std::thread;
+
+use super::env_pool::{EnvSpec, SharedEnv};
+use crate::error::{Error, Result};
+
+pub(crate) fn is_map_full(err: &heed::Error) -> bool {
+    matches!(err, heed::Error::Mdb(heed::MdbError::MapFull))
+}
+
+#[repr(u8)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum DbHealthState {
+    Pending = 0,
+    Healthy = 1,
+    Degraded = 2,
+}
+
+impl DbHealthState {
+    fn from_u8(v: u8) -> Self {
+        debug_assert!(v <= 2);
+
+        match v {
+            0 => Self::Pending,
+            1 => Self::Healthy,
+            _ => Self::Degraded,
+        }
+    }
+}
+
+#[derive(Debug, Clone, Default)]
+pub(crate) struct DbHealth(Arc<AtomicU8>);
+
+impl DbHealth {
+    pub(crate) fn new() -> Self {
+        Self(Arc::new(AtomicU8::new(DbHealthState::Pending as u8)))
+    }
+
+    pub(crate) fn is_healthy(&self) -> bool {
+        // Pending counts as unhealthy: if the GC thread never flipped to
+        // Healthy, something's wrong (deadlocked clear_stale_readers, stuck
+        // writer mutex, etc.) and we want that surfaced to the user.
+        DbHealthState::from_u8(self.0.load(Ordering::Acquire)) == DbHealthState::Healthy
+    }
+
+    pub(crate) fn mark_healthy(&self) {
+        let _ = self.0.compare_exchange(
+            DbHealthState::Pending as u8,
+            DbHealthState::Healthy as u8,
+            Ordering::AcqRel,
+            Ordering::Acquire,
+        );
+    }
+
+    pub(crate) fn mark_unhealthy(&self, reason: &'static str) {
+        let prev = self.0.swap(DbHealthState::Degraded as u8, Ordering::AcqRel);
+        if DbHealthState::from_u8(prev) != DbHealthState::Degraded {
+            tracing::error!(reason, "LMDB tracker marked unhealthy");
+        }
+    }
+}
+
+/// Spawns a background thread that is ensuring that the environment that was previously
+/// open is safe, accessible and doesn't have a corrupted lock.md file. If it does this thread will
+/// hang indefinitely but we will have the information that the database is in failure mode
+pub(crate) fn spawn_lmdb_gc<T: LmdbStore>(shared: Arc<RwLock<Option<T>>>) {
+    let thread_shared = shared.clone();
+    let spawn_result = thread::Builder::new()
+        .name("fff-lmdb-gc".into())
+        .spawn(move || {
+            // Holding a read guard blocks `destroy` / re-init's write
+            // guard until this thread finishes — natural serialization.
+            let guard = match thread_shared.read() {
+                Ok(g) => g,
+                Err(e) => {
+                    tracing::debug!("gc: read lock poisoned: {e}");
+                    return;
+                }
+            };
+            let Some(ref tracker) = *guard else {
+                return; // destroyed before we started
+            };
+            // Trackers attaching to an already-pooled env must not repeat the
+            // GC; the first opener's run flips the shared health flag.
+            if !tracker.shared_env().try_start_gc() {
+                return;
+            }
+
+            if let Err(e) = T::purge_stale_data(tracker.shared_env()) {
+                tracing::debug!("purge_stale_data failed: {e}");
+            }
+
+            tracker.health().mark_healthy();
+        });
+
+    if let Err(e) = spawn_result {
+        tracing::debug!(?e, "failed to spawn fff-lmdb-gc thread");
+        // No thread = mark healthy now so healthcheck isn't stuck Pending.
+        if let Ok(guard) = shared.read()
+            && let Some(ref tracker) = *guard
+        {
+            tracker.health().mark_healthy();
+        }
+    }
+}
+
+pub(crate) trait LmdbStore: Sized + Send + Sync + 'static {
+    /// Short label used to defferintiate different instances of this trait
+    const LABEL: &'static str;
+    /// LMDB map size in bytes. Must be a multiple of the OS page size.
+    const MAP_SIZE: usize;
+    /// Number of named sub-databases. `0` for single-db envs.
+    const MAX_DBS: u32;
+    /// Hard cap on `data.mdb` size.
+    const SIZE_CAP_BYTES: u64;
+
+    /// Borrow the pooled env handle shared by every tracker of this path.
+    fn shared_env(&self) -> &SharedEnv;
+    /// Borrow the health flag from the tracker.
+    fn health(&self) -> &DbHealth;
+
+    /// Borrow the raw heed env.
+    fn env(&self) -> &Env<WithoutTls> {
+        self.shared_env()
+    }
+
+    /// Override to purge stale rows, compact, etc. Default no-op. Runs on
+    /// the GC thread while a read lock is held against the shared handle,
+    /// so destroy / re-init naturally wait for it.
+    fn purge_stale_data(_env: &SharedEnv) -> Result<()> {
+        Ok(())
+    }
+
+    /// Open (or join) the process-shared LMDB env for `db_path`. The health
+    /// flag is per-env: the GC of the first opener flips it for everyone.
+    #[tracing::instrument]
+    fn open_env(db_path: &Path) -> Result<(SharedEnv, DbHealth)> {
+        let shared = SharedEnv::get_or_open(
+            db_path,
+            &EnvSpec {
+                label: Self::LABEL,
+                map_size: Self::MAP_SIZE,
+                max_dbs: Self::MAX_DBS,
+                size_cap_bytes: Self::SIZE_CAP_BYTES,
+            },
+        )?;
+        let health = shared.health().clone();
+        Ok((shared, health))
+    }
+
+    /// Open or create a database without blocking on the LMDB writer mutex
+    /// when the database already exists.
+    fn open_database_safe<KC, DC>(env: &SharedEnv, name: Option<&str>) -> Result<Database<KC, DC>>
+    where
+        KC: 'static,
+        DC: 'static,
+    {
+        let db = Self::LABEL;
+        // mdb_dbi_open must not run from concurrent txns in this process.
+        let _dbi_guard = env.lock_dbi_open();
+
+        let rtxn = env
+            .read_txn()
+            .map_err(|source| Error::DbStartReadTxn { db, source })?;
+        let maybe_db: Option<Database<KC, DC>> = env
+            .open_database(&rtxn, name)
+            .map_err(|source| Error::DbOpen { db, source })?;
+
+        // do not drop the DB here
+        rtxn.commit()
+            .map_err(|source| Error::DbCommit { db, source })?;
+
+        match maybe_db {
+            Some(handle) => Ok(handle),
+            None => {
+                // First time: create the database (requires write lock).
+                // unfortunately this CAN be deadlocking and this is what we see happens
+                // if the other part of the code is segfaulting, so the only rule to prevent this
+                // write the good code mf, okay?
+                let mut wtxn = env
+                    .write_txn()
+                    .map_err(|source| Error::DbStartWriteTxn { db, source })?;
+                let handle = env
+                    .create_database(&mut wtxn, name)
+                    .map_err(|source| Error::DbCreate { db, source })?;
+
+                wtxn.commit()
+                    .map_err(|source| Error::DbCommit { db, source })?;
+                Ok(handle)
+            }
+        }
+    }
+}

crates/fff-core/src/dbs/mod.rs

@@ -0,0 +1,11 @@
+pub(crate) mod env_pool;
+pub(crate) mod lmdb;
+
+pub mod db_healthcheck;
+pub use db_healthcheck::{DbHealth, DbHealthChecker};
+
+pub mod frecency;
+pub use frecency::*;
+
+pub mod query_tracker;
+pub use query_tracker::*;

crates/fff-core/src/dbs/query_tracker.rs

@@ -1,11 +1,11 @@
-use crate::db_healthcheck::DbHealthChecker;
+use super::db_healthcheck::DbHealthChecker;
+use super::env_pool::SharedEnv;
 use crate::error::Error;
-use heed::types::Bytes;
-use heed::{Database, Env, EnvOpenOptions};
-use heed::{EnvFlags, types::SerdeBincode};
+use crate::lmdb::{DbHealth, LmdbStore, is_map_full};
+use heed::types::{Bytes, SerdeBincode};
+use heed::{Database, Env};
 use serde::{Deserialize, Serialize};
 use std::collections::VecDeque;
-use std::fs;
 use std::path::{Path, PathBuf};
 use std::time::{SystemTime, UNIX_EPOCH};
 
@@ -28,29 +28,55 @@ struct HistoryEntry {
 
 #[derive(Debug)]
 pub struct QueryTracker {
-    env: Env,
+    env: SharedEnv,
     // Database for (project_path, query) -> QueryMatchEntry mappings
     query_file_db: Database<Bytes, SerdeBincode<QueryMatchEntry>>,
     // Database for project_path -> VecDeque<HistoryEntry> mappings (file picker)
     query_history_db: Database<Bytes, SerdeBincode<VecDeque<HistoryEntry>>>,
     // Database for project_path -> VecDeque<HistoryEntry> mappings (grep)
     grep_query_history_db: Database<Bytes, SerdeBincode<VecDeque<HistoryEntry>>>,
+    health: DbHealth,
 }
 
 impl DbHealthChecker for QueryTracker {
-    fn get_env(&self) -> &Env {
+    fn get_env(&self) -> &Env<heed::WithoutTls> {
         &self.env
     }
 
-    fn count_entries(&self) -> Result<Vec<(&'static str, u64)>, Error> {
-        let rtxn = self.env.read_txn().map_err(Error::DbStartReadTxn)?;
+    fn is_healthy(&self) -> bool {
+        self.health.is_healthy()
+    }
 
-        let count_queries = self.query_file_db.len(&rtxn).map_err(Error::DbRead)?;
-        let count_histories = self.query_history_db.len(&rtxn).map_err(Error::DbRead)?;
-        let count_grep_histories = self
-            .grep_query_history_db
+    fn count_entries(&self) -> Result<Vec<(&'static str, u64)>, Error> {
+        let rtxn = self
+            .env
+            .read_txn()
+            .map_err(|source| Error::DbStartReadTxn {
+                db: Self::LABEL,
+                source,
+            })?;
+
+        let count_queries = self
+            .query_file_db
             .len(&rtxn)
-            .map_err(Error::DbRead)?;
+            .map_err(|source| Error::DbRead {
+                db: Self::LABEL,
+                source,
+            })?;
+        let count_histories = self
+            .query_history_db
+            .len(&rtxn)
+            .map_err(|source| Error::DbRead {
+                db: Self::LABEL,
+                source,
+            })?;
+        let count_grep_histories =
+            self.grep_query_history_db
+                .len(&rtxn)
+                .map_err(|source| Error::DbRead {
+                    db: Self::LABEL,
+                    source,
+                })?;
 
         Ok(vec![
             ("query_file_entries", count_queries),
@@ -60,47 +86,54 @@ impl DbHealthChecker for QueryTracker {
     }
 }
 
+impl LmdbStore for QueryTracker {
+    const LABEL: &'static str = "query";
+    // 10 MiB hard ceiling. Same reasoning as FrecencyTracker (GH issue #437).
+    const MAP_SIZE: usize = 10 * 1024 * 1024;
+    const MAX_DBS: u32 = 16;
+    const SIZE_CAP_BYTES: u64 = 8 * 1024 * 1024;
+
+    fn shared_env(&self) -> &SharedEnv {
+        &self.env
+    }
+
+    fn health(&self) -> &DbHealth {
+        &self.health
+    }
+}
+
 impl QueryTracker {
-    pub fn new(db_path: impl AsRef<Path>, use_unsafe_no_lock: bool) -> Result<Self, Error> {
+    /// Returns the on-disk path of the LMDB environment directory.
+    pub fn db_path(&self) -> &Path {
+        self.env.path()
+    }
+
+    pub fn open(db_path: impl AsRef<Path>) -> Result<Self, Error> {
         let db_path = db_path.as_ref();
-        fs::create_dir_all(db_path).map_err(Error::CreateDir)?;
+        let (env, health) = Self::open_env(db_path)?;
 
-        let env = unsafe {
-            let mut opts = EnvOpenOptions::new();
-            opts.map_size(10 * 1024 * 1024); // 100 MiB
-            opts.max_dbs(16); // Allow up to 16 databases per environment
-            if use_unsafe_no_lock {
-                opts.flags(EnvFlags::NO_LOCK | EnvFlags::NO_SYNC | EnvFlags::NO_META_SYNC);
-            }
-            opts.open(db_path).map_err(Error::EnvOpen)?
-        };
-
-        env.clear_stale_readers()
-            .map_err(Error::DbClearStaleReaders)?;
-
-        let mut wtxn = env.write_txn().map_err(Error::DbStartWriteTxn)?;
-
-        // Create two named databases
-        let query_file_db = env
-            .create_database(&mut wtxn, Some("query_file_associations"))
-            .map_err(Error::DbCreate)?;
-        let query_history_db = env
-            .create_database(&mut wtxn, Some("query_history"))
-            .map_err(Error::DbCreate)?;
-        let grep_query_history_db = env
-            .create_database(&mut wtxn, Some("grep_query_history"))
-            .map_err(Error::DbCreate)?;
-
-        wtxn.commit().map_err(Error::DbCommit)?;
+        let query_file_db = Self::open_database_safe(&env, Some("query_file_associations"))?;
+        let query_history_db = Self::open_database_safe(&env, Some("query_history"))?;
+        let grep_query_history_db = Self::open_database_safe(&env, Some("grep_query_history"))?;
 
         Ok(QueryTracker {
             env,
             query_file_db,
             query_history_db,
             grep_query_history_db,
+            health,
         })
     }
 
+    #[deprecated(
+        since = "0.7.0",
+        note = "LMDB unsafe no-lock mode is no longer supported; use `QueryTracker::open` instead. \
+                The `_use_unsafe_no_lock` argument is ignored."
+    )]
+    pub fn new(db_path: impl AsRef<Path>, _use_unsafe_no_lock: bool) -> Result<Self, Error> {
+        Self::open(db_path)
+    }
+
     fn get_now(&self) -> u64 {
         SystemTime::now()
             .duration_since(UNIX_EPOCH)
@@ -139,7 +172,10 @@ impl QueryTracker {
     ) -> Result<(), Error> {
         let mut history = db
             .get(wtxn, project_key)
-            .map_err(Error::DbRead)?
+            .map_err(|source| Error::DbRead {
+                db: Self::LABEL,
+                source,
+            })?
             .unwrap_or_default();
 
         history.push_back(HistoryEntry {
@@ -151,7 +187,10 @@ impl QueryTracker {
         }
 
         db.put(wtxn, project_key, &history)
-            .map_err(Error::DbWrite)?;
+            .map_err(|source| Error::DbWrite {
+                db: Self::LABEL,
+                source,
+            })?;
         Ok(())
     }
 
@@ -159,15 +198,21 @@ impl QueryTracker {
     /// offset=0 returns most recent, offset=1 returns 2nd most recent, etc.
     fn read_history_at_offset(
         db: &Database<Bytes, SerdeBincode<VecDeque<HistoryEntry>>>,
-        env: &Env,
+        env: &Env<heed::WithoutTls>,
         project_key: &[u8; 32],
         offset: usize,
     ) -> Result<Option<String>, Error> {
-        let rtxn = env.read_txn().map_err(Error::DbStartReadTxn)?;
+        let rtxn = env.read_txn().map_err(|source| Error::DbStartReadTxn {
+            db: Self::LABEL,
+            source,
+        })?;
 
         let mut history = db
             .get(&rtxn, project_key)
-            .map_err(Error::DbRead)?
+            .map_err(|source| Error::DbRead {
+                db: Self::LABEL,
+                source,
+            })?
             .unwrap_or_default();
 
         // history is FIFO, last element is most recent
@@ -190,12 +235,21 @@ impl QueryTracker {
         let file_path_buf = file_path.to_path_buf();
 
         let query_key = Self::create_query_key(project_path, query)?;
-        let mut wtxn = self.env.write_txn().map_err(Error::DbStartWriteTxn)?;
+        let mut wtxn = self
+            .env
+            .write_txn()
+            .map_err(|source| Error::DbStartWriteTxn {
+                db: Self::LABEL,
+                source,
+            })?;
 
         let mut entry = self
             .query_file_db
             .get(&wtxn, &query_key)
-            .map_err(Error::DbRead)?
+            .map_err(|source| Error::DbRead {
+                db: Self::LABEL,
+                source,
+            })?
             .unwrap_or_else(|| QueryMatchEntry {
                 file_path: file_path_buf.clone(),
                 open_count: 0,
@@ -225,15 +279,50 @@ impl QueryTracker {
 
         entry.last_opened = now;
 
-        self.query_file_db
-            .put(&mut wtxn, &query_key, &entry)
-            .map_err(Error::DbWrite)?;
+        if let Err(e) = self.query_file_db.put(&mut wtxn, &query_key, &entry) {
+            if is_map_full(&e) {
+                self.health.mark_unhealthy("MDB_MAP_FULL on put");
+                tracing::error!(
+                    ?query,
+                    "Query tracker DB hit MDB_MAP_FULL; dropping write — db will \
+                     be erased on next open"
+                );
+                return Ok(());
+            }
+            return Err(Error::DbWrite {
+                db: Self::LABEL,
+                source: e,
+            });
+        }
 
         // Update query history database
         let project_key = Self::create_project_key(project_path)?;
-        Self::append_to_history(&self.query_history_db, &mut wtxn, &project_key, query, now)?;
+        if let Err(e) =
+            Self::append_to_history(&self.query_history_db, &mut wtxn, &project_key, query, now)
+        {
+            if let Error::DbWrite {
+                source: ref inner, ..
+            } = e
+                && is_map_full(inner)
+            {
+                self.health.mark_unhealthy("MDB_MAP_FULL on history append");
+                tracing::error!(?query, "Query tracker DB map full while appending history");
+                return Ok(());
+            }
+            return Err(e);
+        }
 
-        wtxn.commit().map_err(Error::DbCommit)?;
+        if let Err(e) = wtxn.commit() {
+            if is_map_full(&e) {
+                self.health.mark_unhealthy("MDB_MAP_FULL on commit");
+                tracing::error!(?query, "Query tracker DB map full on commit");
+                return Ok(());
+            }
+            return Err(Error::DbCommit {
+                db: Self::LABEL,
+                source: e,
+            });
+        }
 
         tracing::debug!(?query, ?file_path, "Tracked query completion");
         Ok(())
@@ -246,13 +335,21 @@ impl QueryTracker {
         min_combo_count: u32,
     ) -> Result<Option<QueryMatchEntry>, Error> {
         let query_key = Self::create_query_key(project_path, query)?;
-        tracing::debug!(?query_key, "HASH");
-        let rtxn = self.env.read_txn().map_err(Error::DbStartReadTxn)?;
+        let rtxn = self
+            .env
+            .read_txn()
+            .map_err(|source| Error::DbStartReadTxn {
+                db: Self::LABEL,
+                source,
+            })?;
 
         let last_match = self
             .query_file_db
             .get(&rtxn, &query_key)
-            .map_err(Error::DbRead)?;
+            .map_err(|source| Error::DbRead {
+                db: Self::LABEL,
+                source,
+            })?;
 
         Ok(last_match.filter(|entry| entry.open_count >= min_combo_count))
     }
@@ -266,13 +363,21 @@ impl QueryTracker {
     ) -> Result<i32, Error> {
         let query_key = Self::create_query_key(project_path, query)?;
         tracing::debug!(?query_key, "HASH");
-        let rtxn = self.env.read_txn().map_err(Error::DbStartReadTxn)?;
+        let rtxn = self
+            .env
+            .read_txn()
+            .map_err(|source| Error::DbStartReadTxn {
+                db: Self::LABEL,
+                source,
+            })?;
 
         match self
             .query_file_db
             .get(&rtxn, &query_key)
-            .map_err(Error::DbRead)?
-        {
+            .map_err(|source| Error::DbRead {
+                db: Self::LABEL,
+                source,
+            })? {
             Some(entry) => {
                 // Check if the file path matches and return boost
                 if entry.file_path == file_path && entry.open_count >= 2 {
@@ -301,17 +406,45 @@ impl QueryTracker {
     pub fn track_grep_query(&mut self, query: &str, project_path: &Path) -> Result<(), Error> {
         let now = self.get_now();
         let project_key = Self::create_project_key(project_path)?;
-        let mut wtxn = self.env.write_txn().map_err(Error::DbStartWriteTxn)?;
+        let mut wtxn = self
+            .env
+            .write_txn()
+            .map_err(|source| Error::DbStartWriteTxn {
+                db: Self::LABEL,
+                source,
+            })?;
 
-        Self::append_to_history(
+        if let Err(e) = Self::append_to_history(
             &self.grep_query_history_db,
             &mut wtxn,
             &project_key,
             query,
             now,
-        )?;
+        ) {
+            if let Error::DbWrite {
+                source: ref inner, ..
+            } = e
+                && is_map_full(inner)
+            {
+                self.health
+                    .mark_unhealthy("MDB_MAP_FULL on grep history append");
+                tracing::error!(?query, "Grep query history DB map full; dropping write");
+                return Ok(());
+            }
+            return Err(e);
+        }
 
-        wtxn.commit().map_err(Error::DbCommit)?;
+        if let Err(e) = wtxn.commit() {
+            if is_map_full(&e) {
+                self.health.mark_unhealthy("MDB_MAP_FULL on commit");
+                tracing::error!(?query, "Grep query history DB map full on commit");
+                return Ok(());
+            }
+            return Err(Error::DbCommit {
+                db: Self::LABEL,
+                source: e,
+            });
+        }
 
         tracing::debug!(?query, "Tracked grep query");
         Ok(())
@@ -339,7 +472,7 @@ mod tests {
         let temp_dir = env::temp_dir().join("fff_test_query_tracking_new");
         let _ = std::fs::remove_dir_all(&temp_dir);
 
-        let mut tracker = QueryTracker::new(temp_dir.to_str().unwrap(), true).unwrap();
+        let mut tracker = QueryTracker::open(temp_dir.to_str().unwrap()).unwrap();
 
         let project_path = PathBuf::from("/test/project");
         let file_path = PathBuf::from("/test/project/src/main.rs");

crates/fff-core/src/error.rs

@@ -7,6 +7,10 @@ pub enum Error {
     ThreadPanic,
     #[error("Invalid path {0}")]
     InvalidPath(std::path::PathBuf),
+    #[error(
+        "Can not run certain FFF features in a file system root or home directories. Consider smaller per-project directories."
+    )]
+    FilesystemRoot(std::path::PathBuf),
     #[error("File picker not initialized")]
     FilePickerMissing,
     #[error("Failed to acquire lock for frecency")]
@@ -17,24 +21,84 @@ pub enum Error {
     AcquirePathCacheLock,
     #[error("Failed to create directory: {0}")]
     CreateDir(#[from] std::io::Error),
-    #[error("Failed to open frecency database env: {0}")]
-    EnvOpen(#[source] heed::Error),
-    #[error("Failed to create frecency database: {0}")]
-    DbCreate(#[source] heed::Error),
-    #[error("Failed to clear stale readers for frecency database: {0}")]
-    DbClearStaleReaders(#[source] heed::Error),
+    #[error("Failed to remove database directory {path}: {source}")]
+    RemoveDbDir {
+        path: std::path::PathBuf,
+        source: std::io::Error,
+    },
+    #[error("Something is wrong with the local db instance: {0}")]
+    GenericDbError(#[from] heed::Error),
+    #[error("Failed to open {db} database env: {source}")]
+    EnvOpen {
+        db: &'static str,
+        #[source]
+        source: heed::Error,
+    },
+    #[error(
+        "LMDB env at {path} is already open as the '{open_as}' database with different options; requested by '{requested_as}'. Use a distinct path per database."
+    )]
+    EnvSpecMismatch {
+        path: std::path::PathBuf,
+        open_as: &'static str,
+        requested_as: &'static str,
+    },
+    #[error(
+        "The {db} database at {path} is still used by {holders} other tracker(s) in this process"
+    )]
+    DbInUse {
+        db: &'static str,
+        path: std::path::PathBuf,
+        holders: usize,
+    },
+    #[error("Failed to create {db} database: {source}")]
+    DbCreate {
+        db: &'static str,
+        #[source]
+        source: heed::Error,
+    },
+    #[error("Failed to open {db} database: {source}")]
+    DbOpen {
+        db: &'static str,
+        #[source]
+        source: heed::Error,
+    },
+    #[error("Failed to clear stale readers for {db} database: {source}")]
+    DbClearStaleReaders {
+        db: &'static str,
+        #[source]
+        source: heed::Error,
+    },
 
-    #[error("Failed to start read transaction for frecency database: {0}")]
-    DbStartReadTxn(#[source] heed::Error),
-    #[error("Failed to start write transaction for frecency database: {0}")]
-    DbStartWriteTxn(#[source] heed::Error),
-
-    #[error("Failed to read from frecency database: {0}")]
-    DbRead(#[source] heed::Error),
-    #[error("Failed to write to frecency database: {0}")]
-    DbWrite(#[source] heed::Error),
-    #[error("Failed to commit write transaction to frecency database: {0}")]
-    DbCommit(#[source] heed::Error),
+    #[error("Failed to start read transaction for {db} database: {source}")]
+    DbStartReadTxn {
+        db: &'static str,
+        #[source]
+        source: heed::Error,
+    },
+    #[error("Failed to start write transaction for {db} database: {source}")]
+    DbStartWriteTxn {
+        db: &'static str,
+        #[source]
+        source: heed::Error,
+    },
+    #[error("Failed to read from {db} database: {source}")]
+    DbRead {
+        db: &'static str,
+        #[source]
+        source: heed::Error,
+    },
+    #[error("Failed to write to {db} database: {source}")]
+    DbWrite {
+        db: &'static str,
+        #[source]
+        source: heed::Error,
+    },
+    #[error("Failed to commit write transaction to {db} database: {source}")]
+    DbCommit {
+        db: &'static str,
+        #[source]
+        source: heed::Error,
+    },
     #[error("Failed to start file system watcher: {0}")]
     FileSystemWatch(#[from] notify::Error),
 
@@ -43,6 +107,24 @@ pub enum Error {
 
     #[error("libgit2 error occurred: {0}")]
     Git(#[from] git2::Error),
+
+    #[error("Filesystem walk failed: {0}")]
+    WalkFailed(String),
+
+    #[error("Invalid glob pattern '{pattern}': {reason}")]
+    InvalidGlobPattern { pattern: String, reason: String },
+
+    #[error("File system watching is disabled for this picker")]
+    WatcherDisabled,
+
+    #[error("File system watcher is not ready")]
+    WatcherNotReady,
+
+    #[error("Indexed base path changed while creating the watch subscription")]
+    WatchBaseChanged,
+
+    #[error("Failed to start watch callback dispatcher: {0}")]
+    WatchDispatcherStart(#[source] std::io::Error),
 }
 
 pub type Result<T> = std::result::Result<T, Error>;

crates/fff-core/src/git.rs

@@ -1,20 +1,40 @@
 use crate::error::Result;
+use ahash::AHashMap;
 use git2::{Repository, Status, StatusOptions};
 use std::{
     fmt::Debug,
     path::{Path, PathBuf},
 };
-use tracing::debug;
 
-/// Represents a cache of a single git status query, if there is no
-/// status aka file is clear but it was specifically requested to updated
-/// the status is `None` otherwise contains only actual file statuses.
-#[derive(Debug, Clone)]
-pub struct GitStatusCache(Vec<(PathBuf, Status)>);
+pub(crate) fn default_status_options() -> StatusOptions {
+    let mut opts = StatusOptions::new();
+    opts.include_untracked(true)
+        .recurse_untracked_dirs(true)
+        .include_unmodified(true)
+        .exclude_submodules(true);
+    opts
+}
+
+/// Status options for the initial scan / rescan.
+///
+/// Skips `include_unmodified` because every `FileItem` starts with
+/// `git_status: None` (== clean), so a missing cache entry already means
+/// "clean" — no need to ask libgit2 to enumerate every tracked path.
+/// Saves seconds on huge dirty trees (e.g. chromium with 400k+ entries).
+pub(crate) fn initial_scan_status_options() -> StatusOptions {
+    let mut opts = StatusOptions::new();
+    opts.include_untracked(true)
+        .recurse_untracked_dirs(true)
+        .exclude_submodules(true);
+    opts
+}
+
+#[derive(Debug, Clone, Default)]
+pub(crate) struct GitStatusCache(AHashMap<PathBuf, Status>);
 
 impl IntoIterator for GitStatusCache {
     type Item = (PathBuf, Status);
-    type IntoIter = std::vec::IntoIter<Self::Item>;
+    type IntoIter = <AHashMap<PathBuf, Status> as IntoIterator>::IntoIter;
 
     fn into_iter(self) -> Self::IntoIter {
         self.0.into_iter()
@@ -26,25 +46,27 @@ impl GitStatusCache {
         self.0.len()
     }
 
+    #[inline]
     pub fn lookup_status(&self, full_path: &Path) -> Option<Status> {
-        self.0
-            .binary_search_by(|(path, _)| path.as_path().cmp(full_path))
-            .ok()
-            .and_then(|idx| self.0.get(idx).map(|(_, status)| *status))
+        self.0.get(full_path).copied()
     }
 
     #[tracing::instrument(skip(repo, status_options))]
     fn read_status_impl(repo: &Repository, status_options: &mut StatusOptions) -> Result<Self> {
         let statuses = repo.statuses(Some(status_options))?;
         let Some(repo_path) = repo.workdir() else {
-            return Ok(Self(vec![])); // repo is bare
+            return Ok(Self(AHashMap::new())); // repo is bare
         };
 
-        let mut entries = Vec::with_capacity(statuses.len());
+        let repo_path = crate::path_utils::normalize(repo_path.to_path_buf());
+
+        let mut entries = AHashMap::with_capacity(statuses.len());
         for entry in &statuses {
-            if let Some(entry_path) = entry.path() {
-                let full_path = repo_path.join(entry_path);
-                entries.push((full_path, entry.status()));
+            if let Ok(entry_path) = entry.path() {
+                // libgit2 returns entry paths with forward slashes on every platform
+                // fff stores native paths - meaning we have forward slash issue on windows
+                let full_path = crate::path_utils::normalize(repo_path.join(entry_path));
+                entries.insert(full_path, entry.status());
             }
         }
 
@@ -70,46 +92,38 @@ impl GitStatusCache {
         }
     }
 
-    #[tracing::instrument(skip(repo), level = tracing::Level::DEBUG)]
+    #[tracing::instrument(skip(repo), fields(paths_count = paths.len()), level = tracing::Level::DEBUG)]
     pub fn git_status_for_paths<TPath: AsRef<Path> + Debug>(
         repo: &Repository,
         paths: &[TPath],
     ) -> Result<Self> {
         if paths.is_empty() {
-            return Ok(Self(vec![]));
+            return Ok(Self(AHashMap::new()));
         }
 
         let Some(workdir) = repo.workdir() else {
-            return Ok(Self(vec![]));
+            return Ok(Self(AHashMap::new()));
         };
+        let workdir = crate::path_utils::normalize(workdir.to_path_buf());
 
         // git pathspec is pretty slow and requires to walk the whole directory
         // so for a single file which is the most general use case we query directly the file
         if paths.len() == 1 {
             let full_path = paths[0].as_ref();
-            let relative_path = full_path.strip_prefix(workdir)?;
+            let relative_path = full_path.strip_prefix(&workdir)?;
             let status = repo.status_file(relative_path)?;
 
-            return Ok(Self(vec![(full_path.to_path_buf(), status)]));
+            let mut map = AHashMap::with_capacity(1);
+            map.insert(full_path.to_path_buf(), status);
+            return Ok(Self(map));
         }
 
-        let mut status_options = StatusOptions::new();
-        status_options
-            .include_untracked(true)
-            .recurse_untracked_dirs(true)
-            // when reading partial status it's important to include all files requested
-            .include_unmodified(true);
-
+        let mut status_options = default_status_options();
         for path in paths {
-            status_options.pathspec(path.as_ref().strip_prefix(workdir)?);
+            status_options.pathspec(path.as_ref().strip_prefix(&workdir)?);
         }
 
         let git_status_cache = Self::read_status_impl(repo, &mut status_options)?;
-        debug!(
-            status_len = git_status_cache.statuses_len(),
-            "Multiple files git status"
-        );
-
         Ok(git_status_cache)
     }
 }
@@ -157,3 +171,83 @@ pub fn format_git_status_opt(status: Option<Status>) -> Option<&'static str> {
 pub fn format_git_status(status: Option<Status>) -> &'static str {
     format_git_status_opt(status).unwrap_or("unknown")
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::fs;
+    use std::process::Command;
+    use tempfile::TempDir;
+
+    fn git(dir: &Path, args: &[&str]) {
+        let out = Command::new("git")
+            .args(args)
+            .current_dir(dir)
+            .env("GIT_AUTHOR_NAME", "t")
+            .env("GIT_AUTHOR_EMAIL", "t@t")
+            .env("GIT_COMMITTER_NAME", "t")
+            .env("GIT_COMMITTER_EMAIL", "t@t")
+            .output()
+            .unwrap();
+        assert!(out.status.success(), "git {args:?} failed");
+    }
+
+    /// Regression: on case-insensitive filesystems libgit2 returns
+    /// statuses in a case-insensitive order. Our previous sorted-`Vec` +
+    /// `binary_search_by(Path::cmp)` lookup silently missed entries
+    /// because `Path::cmp` is byte-wise.
+    ///
+    /// This test uses deliberately mixed-case filenames so the two
+    /// orderings disagree, then checks every lookup succeeds.
+    #[test]
+    fn lookup_is_case_exact_regardless_of_libgit2_sort_order() {
+        let tmp = TempDir::new().unwrap();
+        // `std::fs::canonicalize` on Windows adds a `\\?\` UNC prefix that
+        // libgit2's workdir string lacks. Use dunce so both sides match.
+        let base = crate::path_utils::canonicalize(tmp.path()).unwrap();
+
+        // Mixed-case names that sort differently under byte-wise vs
+        // case-insensitive comparators.
+        let names = [
+            "README.md",
+            "a_lower.rs",
+            "Z_upper.rs",
+            "mixed_Case.txt",
+            "nested/Inner_File.rs",
+        ];
+        for n in &names {
+            let p = base.join(n);
+            fs::create_dir_all(p.parent().unwrap()).unwrap();
+            fs::write(&p, format!("// {n}\n")).unwrap();
+        }
+
+        git(&base, &["init", "-b", "main"]);
+        git(&base, &["add", "-A"]);
+        git(&base, &["commit", "-m", "seed", "--no-gpg-sign"]);
+
+        // Modify every file so they all end up in the status output as
+        // WT_MODIFIED — guarantees a non-trivial map we have to look up.
+        for n in &names {
+            let p = base.join(n);
+            fs::write(&p, format!("// {n}\n// edit\n")).unwrap();
+        }
+
+        let repo = Repository::open(&base).unwrap();
+        let paths: Vec<PathBuf> = names.iter().map(|n| base.join(n)).collect();
+        let cache = GitStatusCache::git_status_for_paths(&repo, &paths).unwrap();
+
+        for (n, abs) in names.iter().zip(paths.iter()) {
+            let status = cache.lookup_status(abs);
+            assert!(
+                status.is_some(),
+                "lookup for {n} returned None; cache holds {} entries",
+                cache.statuses_len(),
+            );
+            assert!(
+                status.unwrap().contains(Status::WT_MODIFIED),
+                "expected WT_MODIFIED for {n}, got {:?}",
+                status
+            );
+        }
+    }
+}

crates/fff-core/src/git_status_worker.rs

@@ -0,0 +1,120 @@
+use crate::shared::{SharedFrecency, WeakFilePicker};
+use ahash::AHashSet;
+use parking_lot::{Condvar, Mutex};
+use std::path::PathBuf;
+use std::sync::Arc;
+use std::sync::atomic::{AtomicBool, Ordering};
+
+// we don't really need a queue here
+#[derive(Default)]
+struct Pending {
+    paths: AHashSet<PathBuf>,
+    full_rescan: bool,
+    shutdown: bool,
+}
+
+impl Pending {
+    fn has_work(&self) -> bool {
+        self.full_rescan || !self.paths.is_empty()
+    }
+}
+
+/// Condvar based queue that is used for batch processing events
+pub(crate) struct GitStatusWorker {
+    state: Mutex<Pending>,
+    cv: Condvar,
+    consumer_spawned: AtomicBool,
+}
+
+impl GitStatusWorker {
+    pub(crate) fn new() -> Arc<Self> {
+        Arc::new(Self {
+            state: Mutex::new(Pending::default()),
+            cv: Condvar::new(),
+            consumer_spawned: AtomicBool::new(false),
+        })
+    }
+
+    pub(crate) fn spawn_once(
+        self: &Arc<Self>,
+        weak_picker: WeakFilePicker,
+        frecency: SharedFrecency,
+    ) {
+        if self
+            .consumer_spawned
+            .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
+            .is_ok()
+        {
+            Self::spawn_consumer(Arc::clone(self), weak_picker, frecency);
+        }
+    }
+
+    pub(crate) fn enqueue_paths<I>(&self, paths: I)
+    where
+        I: IntoIterator<Item = PathBuf>,
+    {
+        let mut guard = self.state.lock();
+        guard.paths.extend(paths);
+        drop(guard);
+        self.cv.notify_one();
+    }
+
+    pub(crate) fn request_full_rescan(&self) {
+        let mut guard = self.state.lock();
+        guard.full_rescan = true;
+        drop(guard);
+        self.cv.notify_one();
+    }
+
+    pub(crate) fn signal_shutdown(&self) {
+        let mut guard = self.state.lock();
+        guard.shutdown = true;
+        drop(guard);
+        self.cv.notify_one();
+    }
+
+    fn wait_and_take(&self) -> Option<Pending> {
+        let mut guard = self.state.lock();
+        while !guard.shutdown && !guard.has_work() {
+            self.cv.wait(&mut guard);
+        }
+        if guard.shutdown {
+            return None;
+        }
+        Some(std::mem::take(&mut *guard))
+    }
+
+    // the problem: git status update can take a lot of time especially on big repositories
+    // and there is unpredictable wait time on the lock file if huge commit is going so we have to
+    // spawn a separate thread to guartee that notify handler is unlocked even if git update takes a
+    // lot of time on every event burst (pretty cheap as this thread is going to sleep 99.9% of time)
+    fn spawn_consumer(
+        mailbox: Arc<GitStatusWorker>,
+        weak_picker: WeakFilePicker,
+        frecency: SharedFrecency,
+    ) {
+        let _ = std::thread::Builder::new()
+            .name("fff-git-status".into())
+            .spawn(move || {
+                while let Some(work) = mailbox.wait_and_take() {
+                    let Some(picker) = weak_picker.upgrade() else {
+                        break;
+                    };
+
+                    if work.full_rescan {
+                        if let Err(e) = picker.refresh_git_status(&frecency) {
+                            tracing::error!("git-status worker: full rescan failed: {e:?}");
+                        }
+                    } else if !work.paths.is_empty() {
+                        let paths: Vec<PathBuf> = work.paths.into_iter().collect();
+                        if let Err(e) = picker.update_git_status_for_paths(&paths, &frecency) {
+                            tracing::error!("git-status worker: path update failed: {e:?}");
+                        }
+                    }
+                }
+
+                tracing::info!("git-status worker stopped");
+            })
+            .inspect_err(|err| tracing::error!(?err, "Failed to spawn git status worker"));
+    }
+}

crates/fff-core/src/grep/classify.rs

@@ -0,0 +1,131 @@
+//! Definition and import line classification (vibe coded POC)
+//!
+//! Byte-level heuristics that tag a matched line as a code definition
+//! (`struct`, `fn`, `class`, …) or an import/use statement. Used to
+//! rank/annotate grep results for AI/MCP consumers. Gated behind the
+//! `definitions` feature since only such consumers need it.
+
+/// Detect if a line looks like a code definition (struct, fn, class, etc.)
+pub fn is_definition_line(line: &str) -> bool {
+    let s = line.trim_start().as_bytes();
+    let s = skip_modifiers(s);
+    is_definition_keyword(s)
+}
+
+/// Modifier keywords that can precede a definition keyword.
+/// Each must be followed by whitespace to be consumed.
+const MODIFIERS: &[&[u8]] = &[
+    b"pub",
+    b"export",
+    b"default",
+    b"async",
+    b"abstract",
+    b"unsafe",
+    b"static",
+    b"protected",
+    b"private",
+    b"public",
+];
+
+/// Definition keywords to detect.
+const DEF_KEYWORDS: &[&[u8]] = &[
+    b"struct",
+    b"fn",
+    b"enum",
+    b"trait",
+    b"impl",
+    b"class",
+    b"interface",
+    b"function",
+    b"def",
+    b"func",
+    b"type",
+    b"module",
+    b"object",
+];
+
+/// Skip zero or more modifier keywords (including `pub(crate)` style visibility).
+fn skip_modifiers(mut s: &[u8]) -> &[u8] {
+    loop {
+        // Handle `pub(...)` — e.g. `pub(crate)`, `pub(super)`
+        if s.starts_with(b"pub(")
+            && let Some(end) = s.iter().position(|&b| b == b')')
+        {
+            s = skip_ws(&s[end + 1..]);
+            continue;
+        }
+        let mut matched = false;
+        for &kw in MODIFIERS {
+            if s.starts_with(kw) {
+                let rest = &s[kw.len()..];
+                if rest.first().is_some_and(|b| b.is_ascii_whitespace()) {
+                    s = skip_ws(rest);
+                    matched = true;
+                    break;
+                }
+            }
+        }
+        if !matched {
+            return s;
+        }
+    }
+}
+
+/// Check if `s` starts with a definition keyword followed by a word boundary.
+fn is_definition_keyword(s: &[u8]) -> bool {
+    for &kw in DEF_KEYWORDS {
+        if s.starts_with(kw) {
+            let after = s.get(kw.len());
+            // Word boundary: end of input, or next byte is not alphanumeric/underscore
+            if after.is_none_or(|b| !b.is_ascii_alphanumeric() && *b != b'_') {
+                return true;
+            }
+        }
+    }
+    false
+}
+
+/// Skip ASCII whitespace.
+#[inline]
+fn skip_ws(s: &[u8]) -> &[u8] {
+    let n = s
+        .iter()
+        .position(|b| !b.is_ascii_whitespace())
+        .unwrap_or(s.len());
+    &s[n..]
+}
+
+/// Detect import/use lines — lower value than definitions or usages.
+///
+/// Checks if the line (after leading whitespace) starts with a common
+/// import statement prefix. Pure byte-level checks, no regex.
+pub fn is_import_line(line: &str) -> bool {
+    let s = line.trim_start().as_bytes();
+    s.starts_with(b"import ")
+        || s.starts_with(b"import\t")
+        || (s.starts_with(b"from ") && s.get(5).is_some_and(|&b| b == b'\'' || b == b'"'))
+        || s.starts_with(b"use ")
+        || s.starts_with(b"use\t")
+        || starts_with_require(s)
+        || starts_with_include(s)
+}
+
+/// Match `require(` or `require (`.
+#[inline]
+fn starts_with_require(s: &[u8]) -> bool {
+    if !s.starts_with(b"require") {
+        return false;
+    }
+    let rest = &s[b"require".len()..];
+    rest.first() == Some(&b'(') || (rest.first() == Some(&b' ') && rest.get(1) == Some(&b'('))
+}
+
+/// Match `# include ` (with optional spaces after `#`).
+#[inline]
+fn starts_with_include(s: &[u8]) -> bool {
+    if s.first() != Some(&b'#') {
+        return false;
+    }
+    let rest = skip_ws(&s[1..]);
+    rest.starts_with(b"include ") || rest.starts_with(b"include\t")
+}

crates/fff-core/src/grep/fuzzy_grep.rs

@@ -0,0 +1,358 @@
+use crate::simd_path::ArenaPtr;
+use crate::types::{ContentCacheBudget, FileItem, MmapSlot};
+use fff_grep::lines::LineStep;
+use rayon::prelude::*;
+use std::path::Path;
+use std::sync::atomic::{AtomicBool, Ordering};
+
+use super::sink::{
+    char_indices_to_byte_offsets, classify_definition, strip_line_terminators,
+    truncate_display_bytes,
+};
+use super::types::{GrepMatch, GrepResult, GrepSearchOptions};
+
+#[allow(clippy::too_many_arguments)]
+pub(super) fn fuzzy_grep_search<'a>(
+    grep_text: &str,
+    files_to_search: &[&'a FileItem],
+    options: &GrepSearchOptions,
+    total_files: usize,
+    filtered_file_count: usize,
+    case_insensitive: bool,
+    budget: &ContentCacheBudget,
+    abort_signal: &AtomicBool,
+    base_path: &Path,
+    arena: ArenaPtr,
+    overflow_arena: ArenaPtr,
+) -> GrepResult<'a> {
+    // max_typos controls how many *needle* characters can be unmatched.
+    // A transposition (e.g. "shcema" -> "schema") costs ~1 typo with
+    // default gap penalties. We scale max_typos by needle length:
+    //   1-2 chars -> 0 typos (exact subsequence only)
+    //   3-5 chars -> 1 typo
+    //   6+  chars -> 2 typos
+    // Cap at 2: higher values (3+) let the SIMD prefilter pass lines
+    // missing key characters entirely (e.g. query "flvencodeX" matching
+    // lines without 'l' or 'v'). Quality comes from the post-match filters.
+    let max_typos = (grep_text.len() / 3).min(2);
+    let scoring = neo_frizbee::Scoring {
+        // Use default gap penalties. Higher values (e.g. 20) cause
+        // smith-waterman to prefer *dropping needle chars* over paying
+        // gap costs, which inflates the typo count and breaks
+        // transposition matching ("shcema" -> "schema" becomes 3 typos instead of 1)
+        exact_match_bonus: 100,
+        // gap_open_penalty: 4,
+        // gap_extend_penalty: 2,
+        prefix_bonus: 0,
+        capitalization_bonus: if case_insensitive { 0 } else { 4 },
+        ..neo_frizbee::Scoring::default()
+    };
+
+    let matcher = neo_frizbee::Matcher::new(
+        grep_text,
+        &neo_frizbee::Config {
+            // Use the real max_typos so frizbee's SIMD prefilter actually rejects non-matching lines (~2 SIMD instructions per line vs full SW scoring).
+            max_typos: Some(max_typos as u16),
+            sort: false,
+            scoring,
+            ..Default::default()
+        },
+    );
+
+    // Minimum score threshold: 50% of a perfect contiguous match.
+    // With default scoring (match_score=12, matching_case_bonus=4 = 16/char),
+    // a transposition costs ~5 from a gap, keeping the score well above 50%
+    let perfect_score = (grep_text.len() as u16) * 16;
+    let min_score = (perfect_score * 50) / 100;
+
+    // Target identifiers are often longer than the query due to delimiters
+    // (e.g. query "flvencodepicture" -> "ff_flv_encode_picture_header" from ffmpeg)
+    // Allow 3x needle length to accommodate underscore/dot-separated names
+    let max_match_span = grep_text.len() * 3;
+    let needle_len = grep_text.len();
+
+    // Each delimiter (_, .) in the target creates a gap. A typical C/Rust
+    // identifier like "ff_flv_encode_picture_header" has 4-5 underscores.
+    // Scale generously so delimiter gaps don't reject valid matches.
+    let max_gaps = (needle_len / 3).max(2);
+
+    // If a file doesn't contain enough distinct needle characters just skip it
+    let needle_bytes = grep_text.as_bytes();
+    let mut unique_needle_chars: Vec<u8> = Vec::new();
+    for &b in needle_bytes {
+        let lo = b.to_ascii_lowercase();
+        let hi = b.to_ascii_uppercase();
+        if !unique_needle_chars.contains(&lo) {
+            unique_needle_chars.push(lo);
+        }
+        if lo != hi && !unique_needle_chars.contains(&hi) {
+            unique_needle_chars.push(hi);
+        }
+    }
+
+    // How many distinct needle chars must appear in the file.
+    // With max_typos allowed, we need at least (unique_count - max_typos)
+    let unique_count = {
+        let mut seen = [false; 256];
+        for &b in needle_bytes {
+            seen[b.to_ascii_lowercase() as usize] = true;
+        }
+        seen.iter().filter(|&&v| v).count()
+    };
+    let min_chars_required = unique_count.saturating_sub(max_typos);
+
+    let time_budget = if options.time_budget_ms > 0 {
+        Some(std::time::Duration::from_millis(options.time_budget_ms))
+    } else {
+        None
+    };
+    let search_start = std::time::Instant::now();
+    let budget_exceeded = AtomicBool::new(false);
+    let max_matches_per_file = options.max_matches_per_file;
+
+    // for fuzzy match we need a bit smarter chunking as the amount of work we have to perform is
+    // exponentially larger than the original grep (and the nature of work is heavier), so in short we have to
+    // understand if the approximate index prefilter got us a lot of candidates or not
+    //
+    // if we have a few candidates -> likely we have a lot of matches, so verify the check faster
+    // if we have a lot of candidates -> rely on a larger chunk pipelining more parallel lines at once
+    let page_limit = options.page_limit;
+    let base_chunk = rayon::current_num_threads() * 4;
+    let prefilter_strong = total_files > 0 && files_to_search.len() * 2 < total_files;
+    let max_chunk = if prefilter_strong {
+        base_chunk
+    } else {
+        (base_chunk * 256).max(8 * 1024)
+    };
+
+    let growth = if prefilter_strong { 1 } else { 2 };
+    let mut chunk_size = base_chunk;
+    let mut chunk_start = 0;
+    let mut running_matches = 0usize;
+    let mut per_file_results: Vec<(usize, &'a FileItem, Vec<GrepMatch>)> = Vec::new();
+
+    while chunk_start < files_to_search.len() {
+        let chunk_end = (chunk_start + chunk_size).min(files_to_search.len());
+        let chunk = &files_to_search[chunk_start..chunk_end];
+        let chunk_offset = chunk_start;
+        chunk_start = chunk_end;
+        chunk_size = (chunk_size * growth).min(max_chunk);
+
+        // Parallel phase with `map_init`: each rayon worker thread clones the
+        // matcher once and gets a reusable read buffer + mmap slot. Buffer holds
+        // small files, slot holds fresh mmap for cache-miss files ≥ FRESH_MMAP_THRESHOLD.
+        let chunk_results: Vec<(usize, &'a FileItem, Vec<GrepMatch>)> = chunk
+            .par_iter()
+            .enumerate()
+            .map_init(
+                || {
+                    (
+                        matcher.clone(),
+                        Vec::with_capacity(64 * 1024),
+                        MmapSlot::default(),
+                    )
+                },
+                |(matcher, buf, mmap_slot), (local_idx, file)| {
+                    if abort_signal.load(Ordering::Relaxed) {
+                        budget_exceeded.store(true, Ordering::Relaxed);
+                        return None;
+                    }
+
+                    if let Some(budget) = time_budget
+                        && search_start.elapsed() > budget
+                    {
+                        budget_exceeded.store(true, Ordering::Relaxed);
+                        return None;
+                    }
+
+                    let file_arena = if file.is_overflow() {
+                        overflow_arena
+                    } else {
+                        arena
+                    };
+
+                    let file_bytes =
+                        file.get_content_for_search(buf, mmap_slot, file_arena, base_path, budget)?;
+
+                    if min_chars_required > 0 {
+                        let mut chars_found = 0usize;
+                        for &ch in &unique_needle_chars {
+                            if memchr::memchr(ch, file_bytes).is_some() {
+                                chars_found += 1;
+                                if chars_found >= min_chars_required {
+                                    break;
+                                }
+                            }
+                        }
+                        if chars_found < min_chars_required {
+                            return None;
+                        }
+                    }
+
+                    // Validate the whole file as UTF-8 once upfront. Source code
+                    // files are virtually always valid UTF-8; this single check
+                    // replaces per-line from_utf8 calls (~8% of fuzzy grep time)
+                    let file_is_utf8 = std::str::from_utf8(file_bytes).is_ok();
+
+                    let mut stepper = LineStep::new(b'\n', 0, file_bytes.len());
+                    let estimated_lines = (file_bytes.len() / 40).max(64);
+                    let mut file_lines: Vec<&str> = Vec::with_capacity(estimated_lines);
+                    let mut line_meta: Vec<(u64, u64)> = Vec::with_capacity(estimated_lines);
+
+                    let mut line_number: u64 = 1;
+                    while let Some(line_match) = stepper.next_match(file_bytes) {
+                        let byte_offset = line_match.start() as u64;
+                        let trimmed = strip_line_terminators(&file_bytes[line_match]);
+
+                        if !trimmed.is_empty() {
+                            // we know for sure that the file is UTF-8 at this point
+                            let line_str = if file_is_utf8 {
+                                unsafe { std::str::from_utf8_unchecked(trimmed) }
+                            } else if let Ok(s) = std::str::from_utf8(trimmed) {
+                                s
+                            } else {
+                                line_number += 1;
+                                continue;
+                            };
+                            file_lines.push(line_str);
+                            line_meta.push((line_number, byte_offset));
+                        }
+
+                        line_number += 1;
+                    }
+
+                    if file_lines.is_empty() {
+                        return None;
+                    }
+
+                    // Single-pass: score + indices in one Smith-Waterman run per line (not parallel)
+                    let matches_with_indices = matcher.match_list_indices(&file_lines);
+                    let mut file_matches: Vec<GrepMatch> = Vec::new();
+
+                    for mut match_indices in matches_with_indices {
+                        if match_indices.score < min_score {
+                            continue;
+                        }
+
+                        let idx = match_indices.index as usize;
+                        let raw_line = file_lines[idx];
+
+                        let truncated = truncate_display_bytes(raw_line.as_bytes());
+                        let display_line = if truncated.len() < raw_line.len() {
+                            // SAFETY: truncate_display_bytes preserves UTF-8 char boundaries
+                            &raw_line[..truncated.len()]
+                        } else {
+                            raw_line
+                        };
+
+                        // If the line was truncated, re-compute indices on the shorter string.
+                        if display_line.len() < raw_line.len() {
+                            let Some(re_indices) = matcher
+                                .match_list_indices(&[display_line])
+                                .into_iter()
+                                .next()
+                            else {
+                                continue;
+                            };
+                            match_indices = re_indices;
+                        }
+
+                        match_indices.indices.sort_unstable();
+
+                        // Minimum matched chars: at least (needle_len - max_typos)
+                        // characters must appear. This is consistent with the typo
+                        // budget: each typo can drop one needle char from the alignment.
+                        let min_matched = needle_len.saturating_sub(max_typos).max(1);
+                        if match_indices.indices.len() < min_matched {
+                            continue;
+                        }
+
+                        let indices = &match_indices.indices;
+
+                        if let (Some(&first), Some(&last)) = (indices.first(), indices.last()) {
+                            // reject widely scattered matches
+                            let span = last - first + 1;
+                            if span > max_match_span {
+                                continue;
+                            }
+
+                            // Density check: matched chars / span must be dense enough.
+                            // Relaxed for perfect subsequence matches (all needle chars
+                            // present), slightly relaxed for typo matches to handle
+                            // delimiter-heavy targets
+                            // (e.g. "ff_flv_encode_picture_header" has span inflated by underscores w/ density ~68%)
+                            let density = (indices.len() * 100) / span;
+                            let min_density = if indices.len() >= needle_len {
+                                45 // Perfect subsequence relaxed (delimiters inflate span)
+                            } else {
+                                65 // Has typos filter out a long string
+                            };
+                            if density < min_density {
+                                continue;
+                            }
+
+                            // Gap count check: count discontinuities in the indices
+                            let gap_count = indices.windows(2).filter(|w| w[1] != w[0] + 1).count();
+                            if gap_count > max_gaps {
+                                continue;
+                            }
+                        }
+
+                        let (ln, bo) = line_meta[idx];
+                        let match_byte_offsets =
+                            char_indices_to_byte_offsets(display_line, &match_indices.indices);
+                        let col = match_byte_offsets
+                            .first()
+                            .map(|r| r.0 as usize)
+                            .unwrap_or(0);
+
+                        file_matches.push(GrepMatch {
+                            file_index: 0,
+                            line_number: ln,
+                            col,
+                            byte_offset: bo,
+                            is_definition: classify_definition(
+                                options.classify_definitions,
+                                display_line,
+                            ),
+                            line_content: display_line.to_string(),
+                            match_byte_offsets,
+                            fuzzy_score: Some(match_indices.score),
+                            context_before: Vec::new(),
+                            context_after: Vec::new(),
+                        });
+
+                        if max_matches_per_file != 0 && file_matches.len() >= max_matches_per_file {
+                            break;
+                        }
+                    }
+
+                    if file_matches.is_empty() {
+                        return None;
+                    }
+
+                    Some((chunk_offset + local_idx, *file, file_matches))
+                },
+            )
+            .flatten()
+            .collect();
+
+        for result in chunk_results {
+            running_matches += result.2.len();
+            per_file_results.push(result);
+        }
+
+        if running_matches >= page_limit || budget_exceeded.load(Ordering::Relaxed) {
+            break;
+        }
+    }
+
+    GrepResult::collect(
+        per_file_results,
+        files_to_search.len(),
+        options,
+        total_files,
+        filtered_file_count,
+        budget_exceeded.load(Ordering::Relaxed),
+    )
+}

crates/fff-core/src/grep/grep.rs

@@ -0,0 +1,706 @@
+use super::prefilter::prefilter_with_filepath_retry;
+use super::regex::{RegexMatcher, RegexSink, build_regex};
+use super::sink::{SinkState, debug_assert_newline_terminator};
+use super::types::{GrepMatch, GrepMode, GrepResult, GrepSearchOptions};
+use crate::index::{
+    BigramFilter, BigramOverlay, bigram_boundary, fuzzy_candidates, literal_candidates,
+    regex_candidates,
+};
+use crate::simd_string_utils::memmem;
+use crate::types::{ContentCacheBudget, FileItem, FileSliceExt, MmapSlot};
+use fff_grep::{
+    Searcher, SearcherBuilder, Sink, SinkMatch,
+    matcher::{Match, Matcher, NoError},
+};
+use fff_query_parser::{FFFQuery, GrepConfig, QueryParser};
+use rayon::prelude::*;
+use smallvec::SmallVec;
+use std::path::Path;
+use std::sync::atomic::{AtomicBool, Ordering};
+use tracing::Level;
+
+#[allow(clippy::large_enum_variant)]
+pub(super) enum NeedleFinder<'a> {
+    CaseSensitive(memchr::memmem::Finder<'a>),
+    /// Pre-lowered needle bytes for the SIMD case-insensitive search.
+    CaseInsensitive(&'a [u8]),
+}
+
+impl<'a> NeedleFinder<'a> {
+    fn new(needle: &'a [u8], case_insensitive: bool) -> Self {
+        if case_insensitive {
+            Self::CaseInsensitive(needle)
+        } else {
+            Self::CaseSensitive(memchr::memmem::Finder::new(needle))
+        }
+    }
+
+    #[inline]
+    fn find(&self, haystack: &[u8]) -> Option<usize> {
+        match self {
+            Self::CaseSensitive(finder) => finder.find(haystack),
+            Self::CaseInsensitive(needle_lower) => memmem::find(haystack, needle_lower),
+        }
+    }
+
+    #[inline]
+    fn needle(&self) -> &[u8] {
+        match self {
+            Self::CaseSensitive(finder) => finder.needle(),
+            Self::CaseInsensitive(needle_lower) => needle_lower,
+        }
+    }
+
+    /// Compare `haystack` against a slice of the needle with the same case
+    /// semantics as `find`.
+    #[inline]
+    fn eq_fold(&self, haystack: &[u8], needle_seg: &[u8]) -> bool {
+        match self {
+            Self::CaseSensitive(_) => haystack == needle_seg,
+            Self::CaseInsensitive(_) => {
+                haystack.len() == needle_seg.len() && memmem::find(haystack, needle_seg) == Some(0)
+            }
+        }
+    }
+
+    /// Collect highlight spans for every needle occurrence within a line.
+    /// The case branch is resolved once per line, not once per occurrence.
+    #[inline]
+    fn for_each_occurrence(&self, haystack: &[u8], mut on_match: impl FnMut(usize)) {
+        match self {
+            Self::CaseSensitive(finder) => {
+                let mut start_pos = 0usize;
+                while let Some(pos) = finder.find(&haystack[start_pos..]) {
+                    on_match(start_pos + pos);
+                    start_pos += pos + 1;
+                }
+            }
+            Self::CaseInsensitive(needle_lower) => {
+                let mut start_pos = 0usize;
+                while let Some(pos) = memmem::find(&haystack[start_pos..], needle_lower) {
+                    on_match(start_pos + pos);
+                    start_pos += pos + 1;
+                }
+            }
+        }
+    }
+}
+
+struct PlainTextMatcher<'a> {
+    finder: &'a NeedleFinder<'a>,
+}
+
+impl Matcher for PlainTextMatcher<'_> {
+    type Error = NoError;
+
+    #[inline]
+    fn find_at(&self, haystack: &[u8], at: usize) -> Result<Option<Match>, NoError> {
+        let hay = &haystack[at..];
+        let needle_len = self.finder.needle().len();
+
+        Ok(self
+            .finder
+            .find(hay)
+            .map(|pos| Match::new(at + pos, at + pos + needle_len)))
+    }
+
+    #[inline]
+    fn line_terminator(&self) -> Option<fff_grep::LineTerminator> {
+        Some(fff_grep::LineTerminator::byte(b'\n'))
+    }
+}
+
+struct PlainTextSink<'r> {
+    state: SinkState,
+    finder: &'r NeedleFinder<'r>,
+    pattern_len: u32,
+    multiline_segment_len: Option<usize>,
+}
+
+impl Sink for PlainTextSink<'_> {
+    type Error = std::io::Error;
+
+    fn matched(
+        &mut self,
+        searcher: &Searcher,
+        sink_match: &SinkMatch<'_>,
+    ) -> Result<bool, Self::Error> {
+        debug_assert_newline_terminator(searcher);
+        if self.state.max_matches != 0 && self.state.matches.len() >= self.state.max_matches {
+            return Ok(false);
+        }
+
+        let line_bytes = sink_match.bytes();
+        let (display_bytes, display_len, line_number, byte_offset) =
+            SinkState::prepare_line(line_bytes, sink_match);
+
+        let line_content = String::from_utf8_lossy(display_bytes).into_owned();
+        let mut match_byte_offsets: SmallVec<[(u32, u32); 4]> = SmallVec::new();
+        let mut col = 0usize;
+        let mut first = true;
+
+        if let Some(seg_len) = self.multiline_segment_len {
+            // Multiline needle: the match starts on this line, so the needle's
+            // first segment must be a suffix of the line. Highlight that suffix.
+            let seg = &self.finder.needle()[..seg_len];
+            if !seg.is_empty()
+                && display_bytes.len() >= seg.len()
+                && self
+                    .finder
+                    .eq_fold(&display_bytes[display_bytes.len() - seg.len()..], seg)
+            {
+                col = display_bytes.len() - seg.len();
+                match_byte_offsets.push((col as u32, display_len));
+            }
+        } else {
+            let pattern_len = self.pattern_len;
+            self.finder.for_each_occurrence(display_bytes, |pos| {
+                let abs_start = pos as u32;
+                let abs_end = (abs_start + pattern_len).min(display_len);
+                if first {
+                    col = pos;
+                    first = false;
+                }
+                match_byte_offsets.push((abs_start, abs_end));
+            });
+        }
+
+        let (context_before, context_after) = self.state.extract_context(sink_match);
+        self.state.push_match(
+            line_number,
+            col,
+            byte_offset,
+            line_content,
+            match_byte_offsets,
+            context_before,
+            context_after,
+        );
+        Ok(true)
+    }
+
+    fn finish(&mut self, _: &Searcher, _: &fff_grep::SinkFinish) -> Result<(), Self::Error> {
+        Ok(())
+    }
+}
+
+/// Perform a grep search across all indexed files.
+///
+/// When `query` is empty, returns git-modified/untracked files sorted by
+/// frecency for the "welcome state" UI.
+#[tracing::instrument(skip_all, fields(file_count = files.len()))]
+#[allow(clippy::too_many_arguments)]
+pub(crate) fn grep_search<'a>(
+    files: &'a [FileItem],
+    query: &FFFQuery<'_>,
+    options: &GrepSearchOptions,
+    budget: &ContentCacheBudget,
+    bigram_index: Option<&BigramFilter>,
+    bigram_overlay: Option<&BigramOverlay>,
+    abort_signal: &AtomicBool,
+    base_path: &Path,
+    arena: crate::simd_path::ArenaPtr,
+    overflow_arena: crate::simd_path::ArenaPtr,
+) -> GrepResult<'a> {
+    let result = grep_search_parsed(
+        files,
+        query,
+        options,
+        budget,
+        bigram_index,
+        bigram_overlay,
+        abort_signal,
+        base_path,
+        arena,
+        overflow_arena,
+    );
+
+    // Constraint parsing can swallow tokens the user meant literally (e.g. `!=`
+    // becoming an exclusion). If the constrained search scanned everything and
+    // found nothing, retry the whole raw query as literal text. This also holds
+    // for later pages: an empty full scan at offset 0 stays empty at any offset,
+    // so paging offsets consistently index the literal search's file list.
+    let full_scan_empty = result.matches.is_empty() && result.next_file_offset == 0;
+    if !full_scan_empty || query.constraints.is_empty() || abort_signal.load(Ordering::Relaxed) {
+        return result;
+    }
+
+    let raw = query.raw_query.trim();
+    if raw.is_empty() {
+        return result;
+    }
+
+    // Keep any explicit FilePath scope (AI mode `path/to/file.ext` prefix) so the
+    // fallback can't leak matches outside the file the user pinned. Only the
+    // swallowed operator/glob tokens are dropped. See issue #756.
+    let scoped_constraints: fff_query_parser::ConstraintVec<'_> = query
+        .constraints
+        .iter()
+        .filter(|c| matches!(c, fff_query_parser::Constraint::FilePath(_)))
+        .cloned()
+        .collect();
+
+    let literal_query = FFFQuery {
+        raw_query: query.raw_query,
+        constraints: scoped_constraints,
+        fuzzy_query: fff_query_parser::FuzzyQuery::Text(raw),
+        location: None,
+    };
+
+    let mut fallback = grep_search_parsed(
+        files,
+        &literal_query,
+        options,
+        budget,
+        bigram_index,
+        bigram_overlay,
+        abort_signal,
+        base_path,
+        arena,
+        overflow_arena,
+    );
+
+    if fallback.matches.is_empty() {
+        result
+    } else {
+        fallback.literal_fallback = true;
+        fallback
+    }
+}
+
+#[allow(clippy::too_many_arguments)]
+fn grep_search_parsed<'a>(
+    files: &'a [FileItem],
+    query: &FFFQuery<'_>,
+    options: &GrepSearchOptions,
+    budget: &ContentCacheBudget,
+    bigram_index: Option<&BigramFilter>,
+    bigram_overlay: Option<&BigramOverlay>,
+    abort_signal: &AtomicBool,
+    base_path: &Path,
+    arena: crate::simd_path::ArenaPtr,
+    overflow_arena: crate::simd_path::ArenaPtr,
+) -> GrepResult<'a> {
+    let total_files = files.live_count();
+    let constraints_from_query = &query.constraints[..];
+
+    let grep_text = extract_grep_text(query);
+    if grep_text.is_empty() {
+        return GrepResult::empty(total_files, total_files);
+    }
+
+    let case_insensitive = if options.smart_case {
+        !grep_text.chars().any(|c| c.is_uppercase())
+    } else {
+        false
+    };
+
+    let base_count = bigram_boundary(bigram_overlay, files.len());
+
+    let mut regex_fallback_error: Option<String> = None;
+    let regex = match options.mode {
+        GrepMode::PlainText => None,
+        GrepMode::Fuzzy => {
+            let bigram_candidates = fuzzy_candidates(bigram_index, bigram_overlay, &grep_text);
+
+            let (files_to_search, filtered_file_count) = prefilter_with_filepath_retry(
+                files,
+                constraints_from_query,
+                bigram_candidates.as_deref(),
+                base_count,
+                options,
+                arena,
+                overflow_arena,
+            );
+
+            if files_to_search.is_empty() {
+                return GrepResult::empty(total_files, filtered_file_count);
+            }
+
+            return super::fuzzy_grep::fuzzy_grep_search(
+                &grep_text,
+                &files_to_search,
+                options,
+                total_files,
+                filtered_file_count,
+                case_insensitive,
+                budget,
+                abort_signal,
+                base_path,
+                arena,
+                overflow_arena,
+            );
+        }
+        GrepMode::Regex => build_regex(&grep_text, options.smart_case)
+            .inspect_err(|err| {
+                tracing::warn!("Regex compilation failed for {}. Error {}", grep_text, err);
+
+                regex_fallback_error = Some(err.to_string());
+            })
+            .ok(),
+    };
+
+    let (multiline_segment_len, effective_pattern) = match replace_newline_escapes(&grep_text) {
+        Some((replaced, first_newline_pos)) => (Some(first_newline_pos), replaced),
+        None => (None, grep_text),
+    };
+
+    let is_multiline = multiline_segment_len.is_some();
+
+    // when there is multiple line requested automatically expand the context to include all the lines
+    let after_context = if is_multiline && regex.is_none() && options.after_context == 0 {
+        effective_pattern.bytes().filter(|&b| b == b'\n').count()
+    } else {
+        options.after_context
+    };
+
+    let finder_pattern: Vec<u8> = if case_insensitive {
+        effective_pattern.as_bytes().to_ascii_lowercase()
+    } else {
+        effective_pattern.as_bytes().to_vec()
+    };
+    let finder = NeedleFinder::new(&finder_pattern, case_insensitive);
+    let pattern_len = finder_pattern.len() as u32;
+
+    // PlainText (or regex-fallback-to-plain): literal bigram query.
+    // Regex: decompose the pattern HIR into an AND/OR bigram query tree.
+    let bigram_candidates = if regex.is_none() {
+        literal_candidates(bigram_index, bigram_overlay, &[&effective_pattern])
+    } else {
+        regex_candidates(bigram_index, bigram_overlay, &effective_pattern)
+    };
+
+    let (files_to_search, filtered_file_count) = prefilter_with_filepath_retry(
+        files,
+        constraints_from_query,
+        bigram_candidates.as_deref(),
+        base_count,
+        options,
+        arena,
+        overflow_arena,
+    );
+
+    if files_to_search.is_empty() {
+        return GrepResult::empty(total_files, filtered_file_count);
+    }
+
+    // `PlainTextMatcher` is used by the grep-searcher engine for line detection.
+    // `PlainTextSink` / `RegexSink` handle highlight extraction independently via ripgrep create
+    let plain_matcher = PlainTextMatcher { finder: &finder };
+
+    let searcher = {
+        let mut b = SearcherBuilder::new();
+        b.line_number(true).multi_line(is_multiline);
+        b
+    }
+    .build();
+
+    let should_prefilter = regex.is_none();
+    let mut result = perform_grep(
+        &files_to_search,
+        options,
+        &GrepContext {
+            total_files,
+            filtered_file_count,
+            budget,
+            base_path,
+            arena,
+            overflow_arena,
+            prefilter: should_prefilter.then_some(&finder),
+            abort_signal,
+        },
+        // The single sink-selection point: every mode's matcher/sink pairing
+        // is decided here based on the compiled pattern.
+        |file_bytes: &[u8], max_matches: usize| {
+            let state = SinkState {
+                file_index: 0,
+                matches: Vec::with_capacity(4),
+                max_matches,
+                before_context: options.before_context,
+                after_context,
+                classify_definitions: options.classify_definitions,
+            };
+
+            match regex {
+                Some(ref re) => {
+                    let regex_matcher = RegexMatcher {
+                        regex: re,
+                        is_multiline,
+                    };
+                    let mut sink = RegexSink { state, re };
+                    if let Err(e) = searcher.search_slice(&regex_matcher, file_bytes, &mut sink) {
+                        tracing::error!(error = %e, "Grep (regex) search failed");
+                    }
+                    sink.state.matches
+                }
+                None => {
+                    let mut sink = PlainTextSink {
+                        state,
+                        finder: &finder,
+                        pattern_len,
+                        multiline_segment_len,
+                    };
+                    if let Err(e) = searcher.search_slice(&plain_matcher, file_bytes, &mut sink) {
+                        tracing::error!(error = %e, "Grep (plain text) search failed");
+                    }
+                    sink.state.matches
+                }
+            }
+        },
+    );
+    result.regex_fallback_error = regex_fallback_error;
+    result
+}
+
+/// Replace unescaped `\n` escapes with real newlines in a single pass.
+///
+/// Returns `Some((replaced, first_newline_pos))` when the pattern contained at
+/// least one real `\n` escape (the user wants multiline search), where
+/// `first_newline_pos` is the byte offset of the first inserted newline in the
+/// replaced string. Returns `None` when nothing had to be replaced: `\\n` is
+/// preserved as-is (escaped backslash + literal `n`, e.g. `\\nvim-data`).
+pub(super) fn replace_newline_escapes(text: &str) -> Option<(String, usize)> {
+    let bytes = text.as_bytes();
+    let mut result = Vec::with_capacity(bytes.len());
+    let mut first_newline_pos: Option<usize> = None;
+    let mut i = 0;
+    while i < bytes.len() {
+        if bytes[i] == b'\\' && i + 1 < bytes.len() {
+            if bytes[i + 1] == b'n' {
+                // Odd number of consecutive backslashes before 'n' -> real \n escape
+                let mut backslash_count = 1;
+                while backslash_count <= i && bytes[i - backslash_count] == b'\\' {
+                    backslash_count += 1;
+                }
+                if backslash_count % 2 == 1 {
+                    first_newline_pos.get_or_insert(result.len());
+                    result.push(b'\n');
+                    i += 2;
+                    continue;
+                }
+            }
+            result.push(bytes[i]);
+            i += 1;
+        } else {
+            result.push(bytes[i]);
+            i += 1;
+        }
+    }
+
+    let first_newline_pos = first_newline_pos?;
+    let replaced = String::from_utf8(result).unwrap_or_else(|_| text.to_string());
+    Some((replaced, first_newline_pos))
+}
+
+pub fn parse_grep_query(query: &str) -> FFFQuery<'_> {
+    let parser = QueryParser::new(GrepConfig);
+    parser.parse(query)
+}
+
+/// Extract the grep pattern text from the parsed query: all non-constraint
+/// tokens joined with spaces, e.g. `"name = *.rs someth"` -> `"name = someth"`
+/// with constraint `Extension("rs")`.
+fn extract_grep_text(query: &FFFQuery<'_>) -> String {
+    if !matches!(query.fuzzy_query, fff_query_parser::FuzzyQuery::Empty) {
+        return query.grep_text();
+    }
+
+    // if constraint-only or empty query we use raw_query for backslash-escape handling
+    let t = query.raw_query.trim();
+    if t.starts_with('\\') && t.len() > 1 {
+        let suffix = &t[1..];
+        let parser = QueryParser::new(GrepConfig);
+        if !parser.parse(suffix).constraints.is_empty() {
+            return suffix.to_string();
+        }
+    }
+    t.to_string()
+}
+
+#[derive(Clone, Copy)]
+pub(super) struct GrepContext<'a, 'b> {
+    pub(super) total_files: usize,
+    pub(super) filtered_file_count: usize,
+    pub(super) budget: &'a ContentCacheBudget,
+    pub(super) base_path: &'a Path,
+    pub(super) arena: crate::simd_path::ArenaPtr,
+    pub(super) overflow_arena: crate::simd_path::ArenaPtr,
+    pub(super) prefilter: Option<&'a NeedleFinder<'b>>,
+    pub(super) abort_signal: &'a AtomicBool,
+}
+
+impl GrepContext<'_, '_> {
+    #[inline]
+    fn arena_for_file(&self, file: &FileItem) -> crate::simd_path::ArenaPtr {
+        if file.is_overflow() {
+            self.overflow_arena
+        } else {
+            self.arena
+        }
+    }
+}
+
+#[tracing::instrument(
+    skip_all,
+    level = Level::DEBUG,
+    fields(prefiltered_count = files_to_search.len())
+)]
+pub(super) fn perform_grep<'a, F>(
+    files_to_search: &[&'a FileItem],
+    options: &GrepSearchOptions,
+    ctx: &GrepContext<'_, '_>,
+    search_file: F,
+) -> GrepResult<'a>
+where
+    F: Fn(&[u8], usize) -> Vec<GrepMatch> + Sync,
+{
+    let time_budget = if options.time_budget_ms > 0 {
+        Some(std::time::Duration::from_millis(options.time_budget_ms))
+    } else {
+        None
+    };
+
+    let search_start = std::time::Instant::now();
+    let page_limit = options.page_limit;
+    let budget_exceeded = AtomicBool::new(false);
+
+    let mut result_files: Vec<&'a FileItem> = Vec::new();
+    let mut all_matches: Vec<GrepMatch> = Vec::new();
+    let mut files_consumed: usize = 0;
+    let mut page_filled = false;
+
+    // Each chunk is a rayon barrier. A flat small chunk over 500k files = ~7800
+    // barriers; x2 growth makes it logarithmic. But a too-aggressive growth
+    // over-scans: when a page fills mid-chunk, the whole submitted chunk still
+    // runs.
+    //
+    // So only grow when the prefilter is weak (large candidate set);
+    // when bigram cut the set in half, keep fixed small chunks for cheap page-fill termination.
+    let base_chunk = rayon::current_num_threads() * 4;
+    let prefilter_strong = ctx.total_files > 0 && files_to_search.len() * 2 < ctx.total_files;
+    let max_chunk = if prefilter_strong {
+        base_chunk
+    } else {
+        (base_chunk * 256).max(8 * 1024)
+    };
+    let growth = if prefilter_strong { 1 } else { 2 };
+    let mut chunk_size = base_chunk;
+    let mut chunk_start = 0;
+
+    while chunk_start < files_to_search.len() {
+        let chunk_end = (chunk_start + chunk_size).min(files_to_search.len());
+        let chunk = &files_to_search[chunk_start..chunk_end];
+        chunk_start = chunk_end;
+        chunk_size = (chunk_size * growth).min(max_chunk);
+        let chunk_offset = files_consumed;
+
+        let chunk_results: Vec<(usize, &'a FileItem, Vec<GrepMatch>)> = chunk
+            .par_iter()
+            .enumerate()
+            .map_init(
+                // tested it out a few times, this is just fine for rayon worker in this specific
+                // case it doesn't reallocate this many times and it is actually faster than using
+                // scoped threads with a predefined local scratch buffers because of spawn cost
+                || (Vec::with_capacity(64 * 1024), MmapSlot::default()),
+                |(buf, mmap_slot), (local_idx, file)| {
+                    // perform all the atomic machinery on every 8th
+                    if local_idx % 8 == 0 {
+                        let mut need_abort = ctx.abort_signal.load(Ordering::Relaxed);
+                        if !need_abort
+                            && let Some(budget) = time_budget
+                            && all_matches.len() > 1
+                            && search_start.elapsed() > budget
+                        {
+                            need_abort = true;
+                        }
+
+                        if need_abort {
+                            budget_exceeded.store(true, Ordering::Relaxed);
+                            return None;
+                        }
+                    }
+
+                    let content = file.get_content_for_search(
+                        buf,
+                        mmap_slot,
+                        ctx.arena_for_file(file),
+                        ctx.base_path,
+                        ctx.budget,
+                    )?;
+
+                    // Fast whole-file memmem check before entering the
+                    // grep-searcher machinery. Skips Vec alloc, Searcher
+                    // setup, and line-splitting for files that can't match.
+                    if let Some(pf) = ctx.prefilter
+                        && pf.find(content).is_none()
+                    {
+                        return None;
+                    }
+
+                    let file_matches = search_file(content, options.max_matches_per_file);
+
+                    if file_matches.is_empty() {
+                        return None;
+                    }
+
+                    Some((chunk_offset + local_idx, *file, file_matches))
+                },
+            )
+            .flatten()
+            .collect();
+
+        // Every file in the chunk was visited by rayon (matched or not).
+        files_consumed = chunk_offset + chunk.len();
+
+        // Flatten this chunk's results into the accumulator.
+        for (batch_idx, file, file_matches) in chunk_results {
+            let file_result_idx = result_files.len();
+            result_files.push(file);
+
+            for mut m in file_matches {
+                m.file_index = file_result_idx;
+                if options.trim_whitespace {
+                    m.trim_leading_whitespace();
+                }
+                all_matches.push(m);
+            }
+
+            if all_matches.len() >= page_limit {
+                // Tighten files_consumed to the file that tipped us over so
+                // the next page resumes right after it.
+                files_consumed = batch_idx + 1;
+                page_filled = true;
+                break;
+            }
+        }
+
+        if page_filled || budget_exceeded.load(Ordering::Relaxed) {
+            break;
+        }
+    }
+
+    // If no file had any match, we searched the entire slice.
+    if result_files.is_empty() {
+        files_consumed = files_to_search.len();
+    }
+
+    let has_more = budget_exceeded.load(Ordering::Relaxed)
+        || (page_filled && files_consumed < files_to_search.len());
+
+    let next_file_offset = if has_more {
+        options.file_offset + files_consumed
+    } else {
+        0
+    };
+
+    GrepResult {
+        matches: all_matches,
+        files_with_matches: result_files.len(),
+        files: result_files,
+        total_files_searched: files_consumed,
+        total_files: ctx.total_files,
+        filtered_file_count: ctx.filtered_file_count,
+        next_file_offset,
+        regex_fallback_error: None,
+        literal_fallback: false,
+    }
+}

crates/fff-core/src/grep/grep_tests.rs

@@ -0,0 +1,518 @@
+use super::grep::replace_newline_escapes;
+use super::*;
+
+use crate::file_picker::{FilePicker, FilePickerOptions};
+use crate::index::BigramIndexBuilder;
+use std::io::Write;
+use std::sync::atomic::AtomicBool;
+
+#[test]
+fn test_replace_newline_escapes() {
+    // Single \n → multiline: replaced with a real newline at byte 3
+    assert_eq!(
+        replace_newline_escapes("foo\\nbar"),
+        Some(("foo\nbar".to_string(), 3))
+    );
+    // \\n → escaped backslash + literal n, NOT multiline
+    // (this is what the user types when grepping Rust source with `\\nvim`)
+    assert_eq!(replace_newline_escapes("foo\\\\nvim-data"), None);
+    // Real-world: source file has literal \\AppData\\Local\\nvim-data
+    // (double backslash in the file, so user types double backslash)
+    assert_eq!(
+        replace_newline_escapes(r#"format!("{}\\AppData\\Local\\nvim-data","#),
+        None
+    );
+    // No \n at all
+    assert_eq!(replace_newline_escapes("hello world"), None);
+    // \\\\n → even number of backslashes before n → NOT multiline
+    assert_eq!(replace_newline_escapes("foo\\\\\\\\nbar"), None);
+    // \\\n → 3 backslashes: first two pair up, third + n = \n → multiline,
+    // newline lands after "foo" + 2 kept backslashes = byte 5
+    assert_eq!(
+        replace_newline_escapes("foo\\\\\\nbar"),
+        Some(("foo\\\\\nbar".to_string(), 5))
+    );
+    // Position is for the FIRST newline when there are several
+    assert_eq!(
+        replace_newline_escapes("a\\nb\\nc"),
+        Some(("a\nb\nc".to_string(), 1))
+    );
+}
+
+#[test]
+fn test_fuzzy_typo_scoring() {
+    // Mirror the config from fuzzy_grep_search
+    let needle = "schema";
+    let max_typos = (needle.len() / 3).min(2); // 2
+    let config = neo_frizbee::Config {
+        max_typos: Some(max_typos as u16),
+        sort: false,
+        scoring: neo_frizbee::Scoring {
+            exact_match_bonus: 100,
+            ..neo_frizbee::Scoring::default()
+        },
+        ..Default::default()
+    };
+    let min_matched = needle.len().saturating_sub(1).max(1); // 5
+    let max_match_span = needle.len() + 4; // 10
+
+    // Helper: check if a match would pass our post-filters
+    let passes = |n: &str, h: &str| -> bool {
+        let Some(mut mi) = neo_frizbee::match_list_indices(n, &[h], &config)
+            .into_iter()
+            .next()
+        else {
+            return false;
+        };
+        // upstream returns indices in reverse order, sort ascending
+        mi.indices.sort_unstable();
+        if mi.indices.len() < min_matched {
+            return false;
+        }
+        if let (Some(&first), Some(&last)) = (mi.indices.first(), mi.indices.last()) {
+            let span = last - first + 1;
+            if span > max_match_span {
+                return false;
+            }
+            let density = (mi.indices.len() * 100) / span;
+            if density < 70 {
+                return false;
+            }
+        }
+        true
+    };
+
+    // Exact match: must pass
+    assert!(passes("schema", "schema"));
+    // Exact in longer line: must pass
+    assert!(passes("schema", "  schema: String,"));
+    // In identifier: must pass
+    assert!(passes("schema", "pub fn validate_schema() {}"));
+    // Transposition: must pass
+    assert!(passes("shcema", "schema"));
+    // Partial "ema" only line: must NOT pass
+    assert!(!passes("schema", "it has ema in it"));
+    // Completely unrelated: must NOT pass
+    assert!(!passes("schema", "hello world foo bar"));
+}
+
+#[test]
+fn test_multi_grep_search() {
+    use crate::file_picker::{FilePicker, FilePickerOptions};
+    use std::io::Write;
+
+    let dir = tempfile::tempdir().unwrap();
+
+    // File 1: has "GrepMode" and "GrepMatch"
+    {
+        let mut f = std::fs::File::create(dir.path().join("grep.rs")).unwrap();
+        writeln!(f, "pub enum GrepMode {{").unwrap();
+        writeln!(f, "    PlainText,").unwrap();
+        writeln!(f, "    Regex,").unwrap();
+        writeln!(f, "}}").unwrap();
+        writeln!(f, "pub struct GrepMatch {{").unwrap();
+        writeln!(f, "    pub line_number: u64,").unwrap();
+        writeln!(f, "}}").unwrap();
+    }
+
+    // File 2: has "PlainTextMatcher" only
+    {
+        let mut f = std::fs::File::create(dir.path().join("matcher.rs")).unwrap();
+        writeln!(f, "struct PlainTextMatcher {{").unwrap();
+        writeln!(f, "    needle: Vec<u8>,").unwrap();
+        writeln!(f, "}}").unwrap();
+    }
+
+    // File 3: no matches
+    {
+        let mut f = std::fs::File::create(dir.path().join("other.rs")).unwrap();
+        writeln!(f, "fn main() {{").unwrap();
+        writeln!(f, "    println!(\"hello\");").unwrap();
+        writeln!(f, "}}").unwrap();
+    }
+
+    let mut picker = FilePicker::new(FilePickerOptions {
+        base_path: dir.path().to_str().unwrap().into(),
+        watch: false,
+        ..Default::default()
+    })
+    .unwrap();
+    picker.collect_files().unwrap();
+
+    let files = picker.get_files();
+    let arena = picker.arena_base_ptr();
+
+    let options = super::GrepSearchOptions {
+        max_file_size: MAX_FFFILE_SIZE,
+        max_matches_per_file: 0,
+        smart_case: true,
+        file_offset: 0,
+        page_limit: 100,
+        mode: super::GrepMode::PlainText,
+        time_budget_ms: 0,
+        before_context: 0,
+        after_context: 0,
+        classify_definitions: false,
+        trim_whitespace: false,
+        abort_signal: None,
+    };
+    let no_cancel = AtomicBool::new(false);
+
+    // Test with 3 patterns
+    let result = super::multi_grep_search(
+        files,
+        &["GrepMode", "GrepMatch", "PlainTextMatcher"],
+        &[],
+        &options,
+        picker.cache_budget(),
+        None,
+        None,
+        &no_cancel,
+        dir.path(),
+        arena,
+        arena,
+    );
+
+    assert!(
+        result.matches.len() >= 3,
+        "Expected at least 3 matches, got {}",
+        result.matches.len()
+    );
+
+    let has_grep_mode = result
+        .matches
+        .iter()
+        .any(|m| m.line_content.contains("GrepMode"));
+    let has_grep_match = result
+        .matches
+        .iter()
+        .any(|m| m.line_content.contains("GrepMatch"));
+    let has_plain_text_matcher = result
+        .matches
+        .iter()
+        .any(|m| m.line_content.contains("PlainTextMatcher"));
+
+    assert!(has_grep_mode, "Should find GrepMode");
+    assert!(has_grep_match, "Should find GrepMatch");
+    assert!(has_plain_text_matcher, "Should find PlainTextMatcher");
+
+    assert_eq!(result.files.len(), 2, "Should match exactly 2 files");
+
+    // Test with single pattern
+    let result2 = super::multi_grep_search(
+        files,
+        &["PlainTextMatcher"],
+        &[],
+        &options,
+        picker.cache_budget(),
+        None,
+        None,
+        &no_cancel,
+        dir.path(),
+        arena,
+        arena,
+    );
+    assert_eq!(
+        result2.matches.len(),
+        1,
+        "Single pattern should find 1 match"
+    );
+
+    // Test with empty patterns
+    let result3 = super::multi_grep_search(
+        files,
+        &[],
+        &[],
+        &options,
+        picker.cache_budget(),
+        None,
+        None,
+        &no_cancel,
+        dir.path(),
+        arena,
+        arena,
+    );
+    assert_eq!(
+        result3.matches.len(),
+        0,
+        "Empty patterns should find nothing"
+    );
+}
+
+/// E2E: multiline grep (`\n` in query) and escaped-backslash literals (`\\n`)
+/// through the full picker.grep pipeline, in both PlainText and Regex modes.
+#[test]
+fn test_grep_multiline_and_escaped_newline_e2e() {
+    let dir = tempfile::tempdir().unwrap();
+    let base = crate::path_utils::canonicalize(dir.path()).unwrap();
+
+    // Content spanning two lines: "hello unicorn\nrainbow world"
+    {
+        let mut f = std::fs::File::create(base.join("multi.txt")).unwrap();
+        writeln!(f, "hello unicorn").unwrap();
+        writeln!(f, "rainbow world").unwrap();
+    }
+    // Content with a literal double backslash before "nvim": `C:\\Users\\nvim-data`
+    {
+        let mut f = std::fs::File::create(base.join("winpath.rs")).unwrap();
+        writeln!(f, "let p = \"C:\\\\Users\\\\nvim-data\";").unwrap();
+    }
+    {
+        let mut f = std::fs::File::create(base.join("noise.txt")).unwrap();
+        writeln!(f, "nothing interesting here").unwrap();
+    }
+
+    let mut picker = FilePicker::new(FilePickerOptions {
+        base_path: base.to_str().unwrap().into(),
+        watch: false,
+        ..Default::default()
+    })
+    .unwrap();
+    picker.collect_files().unwrap();
+
+    let options = crate::GrepSearchOptions {
+        page_limit: 100,
+        max_matches_per_file: 0,
+        ..Default::default()
+    };
+
+    // 1. PlainText + `\n`: needle becomes a real newline, match spans two lines
+    let query = super::parse_grep_query("unicorn\\nrainbow");
+    let result = picker.grep(&query, &options);
+    assert_eq!(
+        result.files.len(),
+        1,
+        "multiline plaintext should match multi.txt"
+    );
+    assert_eq!(result.files[0].relative_path(&picker), "multi.txt");
+    let m = &result.matches[0];
+    assert_eq!(m.line_content, "hello unicorn");
+    // Auto after-context: the rest of the matched span is returned
+    assert_eq!(m.context_after, vec!["rainbow world"]);
+    // First needle segment highlighted as the line suffix
+    assert_eq!(m.match_byte_offsets.as_slice(), &[(6, 13)]);
+    assert_eq!(m.col, 6);
+
+    // 2. PlainText + `\\n`: literal backslash + n, must NOT be mangled to newline
+    let query = super::parse_grep_query("\\\\nvim-data");
+    let result = picker.grep(&query, &options);
+    assert_eq!(
+        result.files.len(),
+        1,
+        "escaped backslash should match winpath.rs literally"
+    );
+    assert_eq!(result.files[0].relative_path(&picker), "winpath.rs");
+    assert!(result.matches[0].line_content.contains("\\\\nvim-data"));
+    assert!(result.matches[0].context_after.is_empty());
+
+    // 3. Regex + `\n`: goes through the MultiLine searcher strategy
+    let regex_options = super::GrepSearchOptions {
+        mode: super::GrepMode::Regex,
+        ..options.clone()
+    };
+    let query = super::parse_grep_query("unicorn\\nrainbow");
+    let result = picker.grep(&query, &regex_options);
+    assert!(result.regex_fallback_error.is_none());
+    assert_eq!(
+        result.files.len(),
+        1,
+        "multiline regex should match multi.txt"
+    );
+    assert_eq!(result.files[0].relative_path(&picker), "multi.txt");
+    let m = &result.matches[0];
+    // Blob is normalized: single-line content + remaining lines as context
+    assert_eq!(m.line_content, "hello unicorn");
+    assert_eq!(m.context_after, vec!["rainbow world"]);
+    // Highlight clamped to the visible first line
+    assert_eq!(m.match_byte_offsets.as_slice(), &[(6, 13)]);
+}
+
+/// Regression test for issue #407: Live grep returns duplicate results
+/// when the bigram candidate bitset has trailing bits set beyond
+/// `base_file_count`. The bitset is rounded up to a multiple of 64 bits
+/// so any trailing bit that happens to be set (e.g. from overlay data)
+/// would previously map to an overflow file index, which was then also
+/// unconditionally appended by the overflow loop, producing duplicates.
+#[test]
+fn test_grep_no_duplicates_with_overflow_trailing_bits() {
+    let dir = tempfile::tempdir().unwrap();
+    // Match the picker's internal dunce-canonicalize so paths passed to
+    // on_create_or_modify resolve back to the same base_path on Windows.
+    let base = crate::path_utils::canonicalize(dir.path()).unwrap();
+
+    // Five base files: only three contain the pattern "unicorn".
+    // We need some files WITHOUT the pattern so the bigrams for
+    // "unicorn" aren't treated as ubiquitous (≥90% of files) and
+    // dropped from the index during compress().
+    let base_contents: &[(&str, &str)] = &[
+        ("a.txt", "hello unicorn world"),
+        ("b.txt", "another unicorn line"),
+        ("c.txt", "one more unicorn here"),
+        ("d.txt", "nothing special in here"),
+        ("e.txt", "just some random content"),
+    ];
+    for (name, content) in base_contents {
+        let mut f = std::fs::File::create(base.join(name)).unwrap();
+        writeln!(f, "{}", content).unwrap();
+    }
+
+    let mut picker = FilePicker::new(FilePickerOptions {
+        base_path: base.to_str().unwrap().into(),
+        watch: false,
+        ..Default::default()
+    })
+    .unwrap();
+    picker.collect_files().unwrap();
+    assert_eq!(picker.get_files().len(), 5);
+
+    // Manually build a bigram index over the 5 base files.
+    let base_count = 5usize;
+    let consec_builder = BigramIndexBuilder::new(base_count);
+    let skip_builder = BigramIndexBuilder::new(base_count);
+    for (i, (_, content)) in base_contents.iter().enumerate() {
+        consec_builder.add_file_content(&skip_builder, i, content.as_bytes());
+    }
+    let mut index = consec_builder.compress(Some(0));
+    index.set_skip_index(skip_builder.compress(Some(0)));
+    picker.set_bigram_index(index);
+
+    // Add three overflow files (new after the bigram index was built),
+    // all containing "unicorn".
+    for name in ["f.txt", "g.txt", "h.txt"] {
+        let path = base.join(name);
+        let mut f = std::fs::File::create(&path).unwrap();
+        writeln!(f, "overflow unicorn entry").unwrap();
+        drop(f);
+        picker.handle_create_or_modify(&path);
+    }
+    assert_eq!(picker.get_files().len(), 8);
+
+    // Inject a trailing bit into the overlay at a file index that
+    // corresponds to an overflow file (i.e. >= base_file_count=5 but
+    // < bitset_word_size=64). Without the fix, the bigram-candidate
+    // merge would set this bit in the bitset, and the bitset loop would
+    // push files[6] while the overflow loop also appends files[5..]
+    // which includes files[6], producing a duplicate.
+    let overflow_rel = "g.txt"; // middle overflow file
+    let overflow_abs = picker
+        .get_files()
+        .iter()
+        .position(|f| f.relative_path(&picker) == overflow_rel)
+        .expect("overflow file should be present");
+    assert!(overflow_abs >= base_count);
+    assert!(
+        overflow_abs < 64,
+        "index must fit in the single bitset word"
+    );
+
+    if let Some(overlay) = picker.bigram_overlay() {
+        overlay
+            .write()
+            .modify_file(overflow_abs, b"overflow unicorn entry");
+    }
+
+    // Run a grep for "unicorn": six files match
+    // (a, b, c in base + f, g, h in overflow).
+    let query = super::parse_grep_query("unicorn");
+    let options = super::GrepSearchOptions {
+        max_file_size: MAX_FFFILE_SIZE,
+        max_matches_per_file: 0,
+        smart_case: true,
+        file_offset: 0,
+        page_limit: 100,
+        mode: super::GrepMode::PlainText,
+        time_budget_ms: 0,
+        before_context: 0,
+        after_context: 0,
+        classify_definitions: false,
+        trim_whitespace: false,
+        abort_signal: Some(std::sync::Arc::new(AtomicBool::new(false))),
+    };
+    let result = picker.grep(&query, &options);
+
+    // Collect the matched relative paths via the returned files list.
+    let mut paths: Vec<String> = result
+        .files
+        .iter()
+        .map(|f| f.relative_path(&picker))
+        .collect();
+    paths.sort();
+
+    // Every file (base + overflow) should match exactly once.
+    let mut dedup = paths.clone();
+    dedup.dedup();
+    assert_eq!(
+        dedup, paths,
+        "grep must not return duplicate results (issue #407): {:?}",
+        paths
+    );
+    assert_eq!(
+        paths,
+        vec!["a.txt", "b.txt", "c.txt", "f.txt", "g.txt", "h.txt"],
+    );
+
+    // And the match count must equal the number of files (one line per
+    // file). A duplicate entry in files_to_search would double-count
+    // matches for the duplicated file.
+    assert_eq!(
+        result.matches.len(),
+        6,
+        "expected exactly one match per file, got {}",
+        result.matches.len()
+    );
+}
+
+/// Issue #756: an AI-mode regex query with an inline FilePath scope and
+/// top-level alternation. The regex fragments are swallowed as bogus Glob
+/// constraints, the constrained search finds nothing, and the literal/regex
+/// fallback must NOT drop the FilePath scope — otherwise the `|` branch leaks
+/// matches into files outside the pinned path.
+#[test]
+fn regex_fallback_keeps_file_path_scope_issue_756() {
+    use fff_query_parser::{AiGrepConfig, QueryParser};
+    let dir = tempfile::tempdir().unwrap();
+    let base = crate::path_utils::canonicalize(dir.path()).unwrap();
+    std::fs::create_dir(base.join("scope")).unwrap();
+    std::fs::write(
+        base.join("scope").join("target.css"),
+        "/* ---------- target ---------- */\n",
+    )
+    .unwrap();
+    std::fs::write(
+        base.join("outside.css"),
+        "/* ---------- outside ---------- */\n",
+    )
+    .unwrap();
+
+    let mut picker = FilePicker::new(FilePickerOptions {
+        base_path: base.to_str().unwrap().into(),
+        watch: false,
+        ..Default::default()
+    })
+    .unwrap();
+    picker.collect_files().unwrap();
+
+    let options = crate::GrepSearchOptions {
+        mode: super::GrepMode::Regex,
+        smart_case: true,
+        max_matches_per_file: 80,
+        page_limit: 100,
+        ..Default::default()
+    };
+
+    let raw = r"scope/target.css ^/\* |^\s*/\* ----------";
+    let query = QueryParser::new(AiGrepConfig).parse(raw);
+    let result = picker.grep(&query, &options);
+    let mut paths: Vec<String> = result
+        .files
+        .iter()
+        .map(|f| f.relative_path(&picker))
+        .collect();
+    paths.sort();
+
+    assert_eq!(
+        paths,
+        vec!["scope/target.css"],
+        "regex fallback must not leak outside the FilePath scope"
+    );
+}

crates/fff-core/src/grep/mod.rs

@@ -0,0 +1,27 @@
+//! Live grep. `grep.rs` implements the main plain-text path, the parallel
+//! scan engine, and the `grep_search` entry point that picks the matcher/sink
+//! for every mode in one place; `regex`, `multi_pattern`, and `fuzzy_grep`
+//! hold the mode-specific machinery on top of the shared `prefilter`/`sink`.
+
+#[allow(clippy::module_inception)]
+mod grep;
+pub use grep::*;
+
+mod fuzzy_grep;
+mod multi_pattern;
+mod prefilter;
+mod regex;
+mod sink;
+mod types;
+
+#[cfg(feature = "definitions")]
+mod classify;
+#[cfg(feature = "definitions")]
+pub use classify::*;
+
+pub(crate) use multi_pattern::multi_grep_search;
+pub use regex::has_regex_metacharacters;
+pub use types::*;
+
+#[cfg(test)]
+mod grep_tests;

crates/fff-core/src/grep/multi_pattern.rs

@@ -0,0 +1,191 @@
+use super::grep::{GrepContext, perform_grep};
+use super::prefilter::prefilter_files;
+use super::sink::{SinkState, debug_assert_newline_terminator};
+use super::types::{GrepResult, GrepSearchOptions};
+use crate::index::{BigramFilter, BigramOverlay, bigram_boundary, literal_candidates};
+use crate::types::{ContentCacheBudget, FileItem, FileSliceExt};
+use aho_corasick::AhoCorasick;
+use fff_grep::{
+    Searcher, SearcherBuilder, Sink, SinkMatch,
+    matcher::{Match, Matcher, NoError},
+};
+use smallvec::SmallVec;
+use std::path::Path;
+use std::sync::atomic::AtomicBool;
+
+/// A `grep_matcher::Matcher` backed by Aho-Corasick for multi-pattern search.
+///
+/// Finds the first occurrence of any pattern starting at the given offset.
+/// Always reports `\n` as the line terminator for the fast candidate-line path.
+struct AhoCorasickMatcher<'a> {
+    ac: &'a AhoCorasick,
+}
+
+impl Matcher for AhoCorasickMatcher<'_> {
+    type Error = NoError;
+
+    #[inline]
+    fn find_at(&self, haystack: &[u8], at: usize) -> std::result::Result<Option<Match>, NoError> {
+        let hay = &haystack[at..];
+        let found: Option<aho_corasick::Match> = self.ac.find(hay);
+        Ok(found.map(|m| Match::new(at + m.start(), at + m.end())))
+    }
+
+    #[inline]
+    fn line_terminator(&self) -> Option<fff_grep::LineTerminator> {
+        Some(fff_grep::LineTerminator::byte(b'\n'))
+    }
+}
+
+/// Sink for Aho-Corasick multi-pattern mode.
+///
+/// Collects all pattern match positions on each matched line for highlighting.
+struct AhoCorasickSink<'a> {
+    state: SinkState,
+    ac: &'a AhoCorasick,
+}
+
+impl Sink for AhoCorasickSink<'_> {
+    type Error = std::io::Error;
+
+    fn matched(&mut self, searcher: &Searcher, mat: &SinkMatch<'_>) -> Result<bool, Self::Error> {
+        debug_assert_newline_terminator(searcher);
+        if self.state.max_matches != 0 && self.state.matches.len() >= self.state.max_matches {
+            return Ok(false);
+        }
+
+        let line_bytes = mat.bytes();
+        let (display_bytes, display_len, line_number, byte_offset) =
+            SinkState::prepare_line(line_bytes, mat);
+
+        let line_content = String::from_utf8_lossy(display_bytes).into_owned();
+        let mut match_byte_offsets: SmallVec<[(u32, u32); 4]> = SmallVec::new();
+        let mut col = 0usize;
+        let mut first = true;
+
+        for m in self.ac.find_iter(display_bytes as &[u8]) {
+            let abs_start = m.start() as u32;
+            let abs_end = (m.end() as u32).min(display_len);
+            if first {
+                col = abs_start as usize;
+                first = false;
+            }
+            match_byte_offsets.push((abs_start, abs_end));
+        }
+
+        let (context_before, context_after) = self.state.extract_context(mat);
+        self.state.push_match(
+            line_number,
+            col,
+            byte_offset,
+            line_content,
+            match_byte_offsets,
+            context_before,
+            context_after,
+        );
+        Ok(true)
+    }
+
+    fn finish(&mut self, _: &Searcher, _: &fff_grep::SinkFinish) -> Result<(), Self::Error> {
+        Ok(())
+    }
+}
+
+/// Multi-pattern OR search using Aho-Corasick.
+///
+/// Builds a single automaton from all patterns and searches each file in one
+/// pass. This is significantly faster than regex alternation for literal text
+/// searches because Aho-Corasick uses SIMD-accelerated multi-needle matching.
+///
+/// Returns the same `GrepResult` type as `grep_search`.
+#[allow(clippy::too_many_arguments)]
+pub(crate) fn multi_grep_search<'a>(
+    files: &'a [FileItem],
+    patterns: &[&str],
+    constraints: &[fff_query_parser::Constraint<'_>],
+    options: &GrepSearchOptions,
+    budget: &ContentCacheBudget,
+    bigram_index: Option<&BigramFilter>,
+    bigram_overlay: Option<&BigramOverlay>,
+    abort_signal: &AtomicBool,
+    base_path: &Path,
+    arena: crate::simd_path::ArenaPtr,
+    overflow_arena: crate::simd_path::ArenaPtr,
+) -> GrepResult<'a> {
+    let total_files = files.live_count();
+
+    if patterns.is_empty() || patterns.iter().all(|p| p.is_empty()) {
+        return GrepResult::empty(total_files, total_files);
+    }
+
+    let bigram_candidates = literal_candidates(bigram_index, bigram_overlay, patterns);
+    let base_file_count = bigram_boundary(bigram_overlay, files.len());
+
+    // Constraints are separate from patterns, so a miss must not broaden the search.
+    let (files_to_search, filtered_file_count) = prefilter_files(
+        files,
+        constraints,
+        bigram_candidates.as_deref(),
+        base_file_count,
+        options,
+        arena,
+        overflow_arena,
+    );
+
+    if files_to_search.is_empty() {
+        return GrepResult::empty(total_files, filtered_file_count);
+    }
+
+    // Smart case: case-insensitive when all patterns are lowercase
+    let case_insensitive = if options.smart_case {
+        !patterns.iter().any(|p| p.chars().any(|c| c.is_uppercase()))
+    } else {
+        false
+    };
+
+    let ac = aho_corasick::AhoCorasickBuilder::new()
+        .ascii_case_insensitive(case_insensitive)
+        .build(patterns)
+        .expect("Aho-Corasick build should not fail for literal patterns");
+
+    let searcher = {
+        let mut b = SearcherBuilder::new();
+        b.line_number(true);
+        b
+    }
+    .build();
+
+    let ac_matcher = AhoCorasickMatcher { ac: &ac };
+    perform_grep(
+        &files_to_search,
+        options,
+        &GrepContext {
+            total_files,
+            filtered_file_count,
+            budget,
+            base_path,
+            arena,
+            overflow_arena,
+            prefilter: None, // no memmem prefilter for multi-pattern search
+            abort_signal,
+        },
+        |file_bytes: &[u8], max_matches: usize| {
+            let state = SinkState {
+                file_index: 0,
+                matches: Vec::with_capacity(4),
+                max_matches,
+                before_context: options.before_context,
+                after_context: options.after_context,
+                classify_definitions: options.classify_definitions,
+            };
+
+            let mut sink = AhoCorasickSink { state, ac: &ac };
+
+            if let Err(e) = searcher.search_slice(&ac_matcher, file_bytes, &mut sink) {
+                tracing::error!(error = %e, "Grep (aho-corasick multi) search failed");
+            }
+
+            sink.state.matches
+        },
+    )
+}

crates/fff-core/src/grep/prefilter.rs

@@ -0,0 +1,204 @@
+use super::types::GrepSearchOptions;
+use crate::index::BigramFilter;
+use crate::index::constraints::{ConstraintPlan, ConstraintsBuffers};
+use crate::sort_buffer::sort_with_buffer;
+use crate::types::FileItem;
+use fff_query_parser::Constraint;
+
+/// Prefilter with a FilePath-constraint fallback: when constraints yield 0
+/// files and the query had FilePath constraints, retry without them (the path
+/// token was likely part of the search text).
+#[allow(clippy::too_many_arguments)]
+pub(super) fn prefilter_with_filepath_retry<'a>(
+    files: &'a [FileItem],
+    constraints: &[Constraint<'_>],
+    bigram_candidates: Option<&[u64]>,
+    base_count: usize,
+    options: &GrepSearchOptions,
+    arena: crate::simd_path::ArenaPtr,
+    overflow_arena: crate::simd_path::ArenaPtr,
+) -> (Vec<&'a FileItem>, usize) {
+    let (files_to_search, filtered_file_count) = prefilter_files(
+        files,
+        constraints,
+        bigram_candidates,
+        base_count,
+        options,
+        arena,
+        overflow_arena,
+    );
+
+    if !files_to_search.is_empty() {
+        return (files_to_search, filtered_file_count);
+    }
+
+    let Some(stripped) = strip_file_path_constraint_if_present(constraints) else {
+        return (files_to_search, filtered_file_count);
+    };
+
+    prefilter_files(
+        files,
+        &stripped,
+        bigram_candidates,
+        base_count,
+        options,
+        arena,
+        overflow_arena,
+    )
+}
+
+/// Single pass prefilter that doesn't involve file reading
+/// allocates only amount of memory required for storing references of the FileItems have to be
+/// opened for grepping unaviodably, in the worst case allocates N * <word> memory if no prefilter needed
+pub(crate) fn prefilter_files<'a>(
+    files: &'a [FileItem],
+    constraints: &[Constraint<'_>],
+    bigram_candidates: Option<&[u64]>,
+    base_count: usize,
+    options: &GrepSearchOptions,
+    arena: crate::simd_path::ArenaPtr,
+    overflow_arena: crate::simd_path::ArenaPtr,
+) -> (Vec<&'a FileItem>, usize) {
+    let max_file_size = options.max_file_size;
+    let plan = if constraints.is_empty() {
+        None
+    } else {
+        Some(ConstraintPlan::build(
+            constraints,
+            files,
+            arena,
+            overflow_arena,
+        ))
+    };
+
+    let mut scratch = ConstraintsBuffers::new();
+
+    #[inline(always)]
+    fn basic_prefilter(file: &FileItem, max: u64) -> bool {
+        !file.is_deleted() && !file.is_binary() && file.size > 0 && file.size <= max
+    }
+
+    // squeeze as much prefilters into a single loop as possible
+    let mut prefiltered: Vec<&FileItem> = match bigram_candidates {
+        Some(candidates) => {
+            let boundary = base_count.min(files.len());
+            let (indexed, tail) = files.split_at(boundary);
+
+            let cap = BigramFilter::count_candidates(candidates) + tail.len();
+            let mut out: Vec<&FileItem> = Vec::with_capacity(cap);
+
+            let full_words = boundary / 64;
+            let last_word_bits = boundary % 64;
+
+            // we need this because we already had a regression of the wrong bit
+            // has been set for the very last word based on the overlay, it's pretty cheap
+            macro_rules! evaluate_bigram_match_word {
+                ($word:expr, $base:expr) => {{
+                    let mut bits: u64 = $word;
+                    while bits != 0 {
+                        let bit = bits.trailing_zeros() as usize;
+                        let file_idx = $base + bit;
+                        bits &= bits - 1;
+
+                        let f = unsafe { indexed.get_unchecked(file_idx) };
+                        if !basic_prefilter(f, max_file_size) {
+                            continue;
+                        }
+                        if let Some(plan) = plan.as_ref()
+                            && !plan.matches(f, file_idx, arena, overflow_arena, &mut scratch)
+                        {
+                            continue;
+                        }
+                        out.push(f);
+                    }
+                }};
+            }
+
+            // Full words: every set bit guaranteed `< boundary`.
+            for (word_idx, &word) in candidates.iter().take(full_words).enumerate() {
+                if word != 0 {
+                    evaluate_bigram_match_word!(word, word_idx * 64);
+                }
+            }
+
+            // Last partial word: mask bits past `boundary` once at word load.
+            if last_word_bits != 0 {
+                // this will get only (mod 64) bits from the last word guarantee that it's 0 padded
+                let last_mask: u64 = (1u64 << last_word_bits) - 1;
+                let word = candidates[full_words] & last_mask;
+                if word != 0 {
+                    evaluate_bigram_match_word!(word, full_words * 64);
+                }
+            }
+
+            // Sequential processing for non-bigrammable files: they are always in the end
+            for (offset, f) in tail.iter().enumerate() {
+                if !basic_prefilter(f, max_file_size) {
+                    continue;
+                }
+                if let Some(ref p) = plan
+                    && !p.matches(f, boundary + offset, arena, overflow_arena, &mut scratch)
+                {
+                    continue;
+                }
+                out.push(f);
+            }
+
+            out
+        }
+        // this will be executed if there is no bigram, in the worst case it will allocate
+        // whole array of files but probability in the real repo of NO preflter working is so
+        // low that we just ignore that, usually there would be at least a few files excluded
+        None => {
+            let mut out: Vec<&FileItem> = Vec::new();
+            for (idx, f) in files.iter().enumerate() {
+                if !basic_prefilter(f, max_file_size) {
+                    continue;
+                }
+                if let Some(ref p) = plan
+                    && !p.matches(f, idx, arena, overflow_arena, &mut scratch)
+                {
+                    continue;
+                }
+                out.push(f);
+            }
+            out
+        }
+    };
+
+    let total_count = prefiltered.len();
+
+    sort_with_buffer(&mut prefiltered, |a, b| {
+        b.total_frecency_score()
+            .cmp(&a.total_frecency_score())
+            .then(b.modified.cmp(&a.modified))
+    });
+
+    if options.file_offset > 0 && options.file_offset < total_count {
+        let paginated = prefiltered.split_off(options.file_offset);
+        (paginated, total_count)
+    } else if options.file_offset >= total_count {
+        (Vec::new(), total_count)
+    } else {
+        (prefiltered, total_count)
+    }
+}
+
+fn strip_file_path_constraint_if_present<'a>(
+    constraints: &[Constraint<'a>],
+) -> Option<fff_query_parser::ConstraintVec<'a>> {
+    if !constraints
+        .iter()
+        .any(|c| matches!(c, Constraint::FilePath(_)))
+    {
+        return None;
+    }
+
+    let filtered: fff_query_parser::ConstraintVec<'a> = constraints
+        .iter()
+        .filter(|c| !matches!(c, Constraint::FilePath(_)))
+        .cloned()
+        .collect();
+
+    Some(filtered)
+}

crates/fff-core/src/grep/regex.rs

@@ -0,0 +1,130 @@
+use super::sink::{SinkState, debug_assert_newline_terminator, split_multiline_blob};
+use fff_grep::{
+    Searcher, Sink, SinkMatch,
+    matcher::{Match, Matcher, NoError},
+};
+use smallvec::SmallVec;
+
+pub fn has_regex_metacharacters(text: &str) -> bool {
+    regex::escape(text) != text
+}
+
+pub(super) fn build_regex(pattern: &str, smart_case: bool) -> Result<regex::bytes::Regex, String> {
+    if pattern.is_empty() {
+        return Err("empty pattern".to_string());
+    }
+
+    let regex_pattern = if pattern.contains("\\n") {
+        pattern.replace("\\n", "\n")
+    } else {
+        pattern.to_string()
+    };
+
+    let case_insensitive = if smart_case {
+        !pattern.chars().any(|c| c.is_uppercase())
+    } else {
+        false
+    };
+
+    regex::bytes::RegexBuilder::new(&regex_pattern)
+        .case_insensitive(case_insensitive)
+        .multi_line(true)
+        .unicode(false)
+        .build()
+        .map_err(|e| e.to_string())
+}
+
+pub(super) struct RegexMatcher<'r> {
+    pub(super) regex: &'r regex::bytes::Regex,
+    pub(super) is_multiline: bool,
+}
+
+impl Matcher for RegexMatcher<'_> {
+    type Error = NoError;
+
+    #[inline]
+    fn find_at(&self, haystack: &[u8], at: usize) -> Result<Option<Match>, NoError> {
+        Ok(self
+            .regex
+            .find_at(haystack, at)
+            .map(|m| Match::new(m.start(), m.end())))
+    }
+
+    #[inline]
+    fn line_terminator(&self) -> Option<fff_grep::LineTerminator> {
+        if self.is_multiline {
+            None
+        } else {
+            Some(fff_grep::LineTerminator::byte(b'\n'))
+        }
+    }
+}
+
+pub(super) struct RegexSink<'r> {
+    pub(super) state: SinkState,
+    pub(super) re: &'r regex::bytes::Regex,
+}
+
+impl Sink for RegexSink<'_> {
+    type Error = std::io::Error;
+
+    fn matched(
+        &mut self,
+        searcher: &Searcher,
+        sink_match: &SinkMatch<'_>,
+    ) -> Result<bool, Self::Error> {
+        debug_assert_newline_terminator(searcher);
+        if self.state.max_matches != 0 && self.state.matches.len() >= self.state.max_matches {
+            return Ok(false);
+        }
+
+        let line_bytes = sink_match.bytes();
+        let (display_bytes, _, line_number, byte_offset) =
+            SinkState::prepare_line(line_bytes, sink_match);
+
+        // MultiLine strategy hands over all matched lines as one blob: keep
+        // `line_content` single-line, the remaining lines become after-context.
+        let (first_line, extra_after) = split_multiline_blob(display_bytes);
+        let first_len = first_line.len() as u32;
+        let line_content = String::from_utf8_lossy(first_line).into_owned();
+        let mut match_byte_offsets: SmallVec<[(u32, u32); 4]> = SmallVec::new();
+        let mut col = 0usize;
+        let mut first = true;
+
+        for m in self.re.find_iter(display_bytes) {
+            let abs_start = m.start() as u32;
+            if abs_start >= first_len {
+                continue; // highlight only spans visible in the first line
+            }
+            let abs_end = (m.end() as u32).min(first_len);
+            if first {
+                col = abs_start as usize;
+                first = false;
+            }
+            match_byte_offsets.push((abs_start, abs_end));
+        }
+
+        let (context_before, context_after) = self.state.extract_context(sink_match);
+        let context_after = if extra_after.is_empty() {
+            context_after
+        } else {
+            let mut combined = extra_after;
+            combined.extend(context_after);
+            combined
+        };
+        self.state.push_match(
+            line_number,
+            col,
+            byte_offset,
+            line_content,
+            match_byte_offsets,
+            context_before,
+            context_after,
+        );
+        Ok(true)
+    }
+
+    fn finish(&mut self, _: &Searcher, _: &fff_grep::SinkFinish) -> Result<(), Self::Error> {
+        Ok(())
+    }
+}

crates/fff-core/src/grep/sink.rs

@@ -0,0 +1,242 @@
+use super::types::GrepMatch;
+use fff_grep::{Searcher, SinkMatch};
+use smallvec::SmallVec;
+
+/// Maximum bytes of a matched line to keep for display. Prevents minified
+/// JS or huge single-line files from blowing up memory.
+pub(super) const MAX_LINE_DISPLAY_LEN: usize = 512;
+
+#[cfg(feature = "definitions")]
+#[inline]
+pub(super) fn classify_definition(enabled: bool, line: &str) -> bool {
+    enabled && super::classify::is_definition_line(line)
+}
+
+#[cfg(not(feature = "definitions"))]
+#[inline]
+pub(super) fn classify_definition(_enabled: bool, _line: &str) -> bool {
+    false
+}
+
+#[inline]
+pub(super) fn debug_assert_newline_terminator(searcher: &Searcher) {
+    debug_assert_eq!(
+        searcher.line_terminator(),
+        fff_grep::LineTerminator::byte(b'\n'),
+        "sink helpers assume \\n line terminators (see module invariant)"
+    );
+}
+
+#[inline]
+pub(super) fn strip_line_terminators(bytes: &[u8]) -> &[u8] {
+    let mut len = bytes.len();
+    while len > 0 && matches!(bytes[len - 1], b'\n' | b'\r') {
+        len -= 1;
+    }
+    &bytes[..len]
+}
+
+pub(super) struct SinkState {
+    pub(super) file_index: usize,
+    pub(super) matches: Vec<GrepMatch>,
+    pub(super) max_matches: usize,
+    pub(super) before_context: usize,
+    pub(super) after_context: usize,
+    pub(super) classify_definitions: bool,
+}
+
+impl SinkState {
+    #[inline]
+    pub(super) fn prepare_line<'a>(
+        line_bytes: &'a [u8],
+        mat: &SinkMatch<'_>,
+    ) -> (&'a [u8], u32, u64, u64) {
+        let line_number = mat.line_number().unwrap_or(0);
+        let byte_offset = mat.absolute_byte_offset();
+
+        // Trim trailing newline/CR directly on bytes to avoid UTF-8 conversion.
+        let trimmed_bytes = strip_line_terminators(line_bytes);
+
+        // Truncate for display (floor to a char boundary).
+        let display_bytes = truncate_display_bytes(trimmed_bytes);
+
+        let display_len = display_bytes.len() as u32;
+        (display_bytes, display_len, line_number, byte_offset)
+    }
+
+    #[inline]
+    #[allow(clippy::too_many_arguments)]
+    pub(super) fn push_match(
+        &mut self,
+        line_number: u64,
+        col: usize,
+        byte_offset: u64,
+        line_content: String,
+        match_byte_offsets: SmallVec<[(u32, u32); 4]>,
+        context_before: Vec<String>,
+        context_after: Vec<String>,
+    ) {
+        let is_definition = classify_definition(self.classify_definitions, &line_content);
+        self.matches.push(GrepMatch {
+            file_index: self.file_index,
+            line_number,
+            col,
+            byte_offset,
+            line_content,
+            match_byte_offsets,
+            fuzzy_score: None,
+            is_definition,
+            context_before,
+            context_after,
+        });
+    }
+
+    /// Extract context lines from the full buffer around a matched region.
+    pub(super) fn extract_context(&self, mat: &SinkMatch<'_>) -> (Vec<String>, Vec<String>) {
+        if self.before_context == 0 && self.after_context == 0 {
+            return (Vec::new(), Vec::new());
+        }
+
+        let buffer = mat.buffer();
+        let range = mat.bytes_range_in_buffer();
+
+        let mut before = Vec::new();
+        if self.before_context > 0 && range.start > 0 {
+            // Walk backward from the start of the match line to find preceding lines
+            let mut pos = range.start;
+            let mut lines_found = 0;
+            while lines_found < self.before_context && pos > 0 {
+                // Skip the newline just before our current position
+                pos -= 1;
+                // Find the previous newline
+                let line_start = match memchr::memrchr(b'\n', &buffer[..pos]) {
+                    Some(nl) => nl + 1,
+                    None => 0,
+                };
+                let line = &buffer[line_start..pos];
+                // Trim trailing \r
+                let line = if line.last() == Some(&b'\r') {
+                    &line[..line.len() - 1]
+                } else {
+                    line
+                };
+                let truncated = truncate_display_bytes(line);
+                before.push(String::from_utf8_lossy(truncated).into_owned());
+                pos = line_start;
+                lines_found += 1;
+            }
+            before.reverse();
+        }
+
+        let mut after = Vec::new();
+        if self.after_context > 0 && range.end < buffer.len() {
+            let mut pos = range.end;
+            let mut lines_found = 0;
+            while lines_found < self.after_context && pos < buffer.len() {
+                // Find the next newline
+                let line_end = match memchr::memchr(b'\n', &buffer[pos..]) {
+                    Some(nl) => pos + nl,
+                    None => buffer.len(),
+                };
+                let line = &buffer[pos..line_end];
+                // Trim trailing \r
+                let line = if line.last() == Some(&b'\r') {
+                    &line[..line.len() - 1]
+                } else {
+                    line
+                };
+                let truncated = truncate_display_bytes(line);
+                after.push(String::from_utf8_lossy(truncated).into_owned());
+                pos = if line_end < buffer.len() {
+                    line_end + 1 // skip past \n
+                } else {
+                    buffer.len()
+                };
+                lines_found += 1;
+            }
+        }
+
+        (before, after)
+    }
+}
+
+/// Truncate a byte slice for display, respecting UTF-8 char boundaries.
+#[inline]
+pub(super) fn truncate_display_bytes(bytes: &[u8]) -> &[u8] {
+    if bytes.len() <= MAX_LINE_DISPLAY_LEN {
+        bytes
+    } else {
+        let mut end = MAX_LINE_DISPLAY_LEN;
+        while end > 0 && !is_utf8_char_boundary(bytes[end]) {
+            end -= 1;
+        }
+        &bytes[..end]
+    }
+}
+
+/// Split a multiline match blob (from the MultiLine searcher strategy) into
+/// the first line and the remaining lines so `line_content` stays single-line.
+pub(super) fn split_multiline_blob(display_bytes: &[u8]) -> (&[u8], Vec<String>) {
+    match memchr::memchr(b'\n', display_bytes) {
+        None => (display_bytes, Vec::new()),
+        Some(pos) => {
+            let first = strip_line_terminators(&display_bytes[..pos + 1]);
+            let extra = display_bytes[pos + 1..]
+                .split(|&b| b == b'\n')
+                .map(|l| String::from_utf8_lossy(strip_line_terminators(l)).into_owned())
+                .collect();
+            (first, extra)
+        }
+    }
+}
+
+/// Convert character-position indices from neo_frizbee into byte-offset
+/// pairs (start, end) suitable for `match_byte_offsets`.
+///
+/// frizbee returns character positions (0-based index into the char
+/// iterator). We need byte ranges because the UI renderer and Lua layer
+/// use byte offsets for extmark highlights.
+///
+/// Each matched character becomes its own (byte_start, byte_end) pair.
+/// Adjacent characters are merged into a single contiguous range.
+pub(super) fn char_indices_to_byte_offsets(
+    line: &str,
+    char_indices: &[usize],
+) -> SmallVec<[(u32, u32); 4]> {
+    if char_indices.is_empty() {
+        return SmallVec::new();
+    }
+
+    // Build a map: char_index -> (byte_start, byte_end) for all chars.
+    // Iterating all chars is O(n) in the line length which is bounded by MAX_LINE_DISPLAY_LEN (512).
+    let char_byte_ranges: Vec<(usize, usize)> = line
+        .char_indices()
+        .map(|(byte_pos, ch)| (byte_pos, byte_pos + ch.len_utf8()))
+        .collect();
+
+    // Convert char indices to byte ranges, merging adjacent ranges
+    let mut result: SmallVec<[(u32, u32); 4]> = SmallVec::with_capacity(char_indices.len());
+
+    for &ci in char_indices {
+        if ci >= char_byte_ranges.len() {
+            continue; // out of bounds (shouldn't happen with valid data)
+        }
+        let (start, end) = char_byte_ranges[ci];
+        // Merge with previous range if adjacent
+        if let Some(last) = result.last_mut()
+            && last.1 == start as u32
+        {
+            last.1 = end as u32;
+            continue;
+        }
+        result.push((start as u32, end as u32));
+    }
+
+    result
+}
+
+// copied from the rust u8 private method
+#[inline]
+const fn is_utf8_char_boundary(b: u8) -> bool {
+    (b as i8) >= -0x40
+}

crates/fff-core/src/grep/types.rs

@@ -0,0 +1,243 @@
+use crate::types::FileItem;
+use smallvec::SmallVec;
+use std::sync::Arc;
+use std::sync::atomic::AtomicBool;
+
+pub use crate::constants::MAX_FFFILE_SIZE;
+
+/// Controls how the grep pattern is interpreted.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
+pub enum GrepMode {
+    /// Literal plain text match: default path that doesn't require any regex machinery
+    #[default]
+    PlainText,
+    /// Regex mode: uses the same exact matching engine as ripgrep
+    Regex,
+    /// Smart fuzzy mode, allows user to make either a couple of single char typos or long gaps
+    /// e.g. shcema -> shcema, or UserController -> UserAuthController
+    ///
+    /// Significatnly slower than plain text, especially on unindexed FilePicker
+    Fuzzy,
+}
+
+/// A single content match within a file
+#[derive(Debug, Clone)]
+pub struct GrepMatch {
+    /// Index into the deduplicated `files` vec of the GrepResult.
+    pub file_index: usize,
+    /// 1-based line number.
+    pub line_number: u64,
+    /// 0-based byte column of first match start within the line.
+    pub col: usize,
+    /// Absolute byte offset of the matched line from the start of the file.
+    /// Can be used by the preview to seek directly without scanning from the top.
+    pub byte_offset: u64,
+    /// The matched line text, truncated to `MAX_LINE_DISPLAY_LEN`.
+    pub line_content: String,
+    /// Byte offsets `(start, end)` within `line_content` for each match.
+    /// Stack-allocated for the common case of ≤4 spans per line.
+    pub match_byte_offsets: SmallVec<[(u32, u32); 4]>,
+    /// Fuzzy match score from neo_frizbee (only set in Fuzzy grep mode).
+    pub fuzzy_score: Option<u16>,
+    /// Whether the matched line looks like a definition (struct, fn, class, etc.).
+    /// Computed at match time so output formatters don't need to re-scan.
+    pub is_definition: bool,
+    /// Lines before the match (for context display). Empty when context is 0.
+    pub context_before: Vec<String>,
+    /// Lines after the match (for context display). Empty when context is 0.
+    pub context_after: Vec<String>,
+}
+
+impl GrepMatch {
+    /// Strip leading whitespace from `line_content` and all context lines,
+    /// adjusting `col` and `match_byte_offsets` so highlights remain correct.
+    pub fn trim_leading_whitespace(&mut self) {
+        let strip_len = self.line_content.len() - self.line_content.trim_start().len();
+        if strip_len > 0 {
+            self.line_content.drain(..strip_len);
+            let off = strip_len as u32;
+            self.col = self.col.saturating_sub(strip_len);
+            for range in &mut self.match_byte_offsets {
+                range.0 = range.0.saturating_sub(off);
+                range.1 = range.1.saturating_sub(off);
+            }
+        }
+        for line in &mut self.context_before {
+            let n = line.len() - line.trim_start().len();
+            if n > 0 {
+                line.drain(..n);
+            }
+        }
+        for line in &mut self.context_after {
+            let n = line.len() - line.trim_start().len();
+            if n > 0 {
+                line.drain(..n);
+            }
+        }
+    }
+}
+
+/// Options for grep search.
+#[derive(Debug, Clone)]
+pub struct GrepSearchOptions {
+    pub max_file_size: u64,
+    pub max_matches_per_file: usize,
+    pub smart_case: bool,
+    /// File-based pagination offset: index into the sorted/filtered file list
+    /// to start searching from. Pass 0 for the first page, then use
+    /// `GrepResult::next_file_offset` for subsequent pages.
+    pub file_offset: usize,
+    /// Maximum number of matches to collect before stopping.
+    pub page_limit: usize,
+    /// How to interpret the search pattern. Defaults to `PlainText`.
+    pub mode: GrepMode,
+    /// Maximum time in milliseconds to spend searching before returning partial
+    /// results. Prevents UI freezes on pathological queries. 0 = no limit.
+    pub time_budget_ms: u64,
+    /// Number of context lines to include before each match. 0 = disabled.
+    pub before_context: usize,
+    /// Number of context lines to include after each match. 0 = disabled.
+    pub after_context: usize,
+    /// Whether to classify each match as a definition line. Adds ~2% overhead
+    /// on large repos; disable for interactive grep where it is not needed.
+    pub classify_definitions: bool,
+    /// Strip leading whitespace from matched lines and context lines, adjusting
+    /// highlight byte offsets accordingly. Useful for AI/MCP consumers and UIs
+    /// that don't need indentation. Default: false.
+    pub trim_whitespace: bool,
+    /// External abort signal. When provided, overrides the picker's internal
+    /// cancellation flag. Set to `true` to stop the search early and return
+    /// partial results. Omit (or use `..Default::default()`) to let the
+    /// picker manage cancellation.
+    pub abort_signal: Option<Arc<AtomicBool>>,
+}
+
+impl Default for GrepSearchOptions {
+    fn default() -> Self {
+        Self {
+            max_file_size: MAX_FFFILE_SIZE,
+            max_matches_per_file: 200,
+            smart_case: true,
+            file_offset: 0,
+            page_limit: 50,
+            mode: GrepMode::default(),
+            time_budget_ms: 0,
+            before_context: 0,
+            after_context: 0,
+            classify_definitions: false,
+            trim_whitespace: false,
+            abort_signal: None,
+        }
+    }
+}
+
+/// Result of a grep search with a list of matches, list of matched files, and metadata.
+#[derive(Debug, Clone, Default)]
+pub struct GrepResult<'a> {
+    pub matches: Vec<GrepMatch>,
+    /// Deduplicated file references for the returned matches.
+    pub files: Vec<&'a FileItem>,
+    /// Number of files actually searched in this call.
+    pub total_files_searched: usize,
+    /// Total number of indexed files (before filtering).
+    pub total_files: usize,
+    /// Total number of searchable files (after filtering out binary, too-large, etc.).
+    pub filtered_file_count: usize,
+    /// Number of files that contained at least one match.
+    pub files_with_matches: usize,
+    /// The file offset to pass for the next page. `0` if there are no more files.
+    /// Callers should store this and pass it as `file_offset` in the next call.
+    pub next_file_offset: usize,
+    /// When regex mode fails to compile the pattern, the search falls back to
+    /// literal matching and this field contains the compilation error message.
+    /// The UI can display this to inform the user their regex was invalid.
+    pub regex_fallback_error: Option<String>,
+    /// Set to `true` if the constrained query found nothing and the results come from
+    /// retrying the whole raw query as literal text (ignored all the inferred constraints)
+    pub literal_fallback: bool,
+}
+
+impl<'a> GrepResult<'a> {
+    /// Empty result carrying only the file counts (empty query / prefilter miss)
+    pub(crate) fn empty(total_files: usize, filtered_file_count: usize) -> Self {
+        Self {
+            total_files,
+            filtered_file_count,
+            ..Default::default()
+        }
+    }
+
+    pub(crate) fn collect(
+        per_file_results: Vec<(usize, &'a FileItem, Vec<GrepMatch>)>,
+        files_to_search_len: usize,
+        options: &GrepSearchOptions,
+        total_files: usize,
+        filtered_file_count: usize,
+        budget_exceeded: bool,
+    ) -> Self {
+        let page_limit = options.page_limit;
+
+        // Each match stores a `file_index` pointing into `result_files` so that
+        // consumers (FFI JSON, Lua) can look up file metadata without duplicating
+        // it across every match from the same file
+        let mut result_files: Vec<&'a FileItem> = Vec::new();
+        let mut all_matches: Vec<GrepMatch> = Vec::new();
+        // files_consumed tracks how far into files_to_search we have advanced,
+        // counting every file whose results were emitted (with or without matches).
+        // We use the batch_idx of the last consumed file + 1, which is correct
+        // because per_file_results only contains files that had matches, and
+        // files between them that had no matches were still searched and can be
+        // safely skipped on the next page
+        let mut files_consumed: usize = 0;
+
+        for (batch_idx, file, file_matches) in per_file_results {
+            // batch_idx is the 0-based position in files_to_search.
+            // Advance files_consumed to include this file and all no-match files before it.
+            files_consumed = batch_idx + 1;
+
+            let file_result_idx = result_files.len();
+            result_files.push(file);
+
+            for mut m in file_matches {
+                m.file_index = file_result_idx;
+                if options.trim_whitespace {
+                    m.trim_leading_whitespace();
+                }
+                all_matches.push(m);
+            }
+
+            // page_limit is a soft cap: we always finish the current file before
+            // stopping, so no matches are dropped. A page may return up to
+            // page_limit + max_matches_per_file - 1 matches in the worst case
+            if all_matches.len() >= page_limit {
+                break;
+            }
+        }
+
+        // If no file had any match, we searched the entire slice.
+        if result_files.is_empty() {
+            files_consumed = files_to_search_len;
+        }
+
+        let has_more = budget_exceeded
+            || (all_matches.len() >= page_limit && files_consumed < files_to_search_len);
+
+        let next_file_offset = if has_more {
+            options.file_offset + files_consumed
+        } else {
+            0
+        };
+
+        Self {
+            matches: all_matches,
+            files_with_matches: result_files.len(),
+            files: result_files,
+            total_files_searched: files_consumed,
+            total_files,
+            filtered_file_count,
+            next_file_offset,
+            regex_fallback_error: None,
+            literal_fallback: false,
+        }
+    }
+}

crates/fff-core/src/ignore.rs

@@ -0,0 +1,136 @@
+use std::path::Path;
+
+/// Directories excluded when walking a non-git root. Entries are `cfg`-gated
+/// so a single iteration covers standard + platform-specific overrides.
+pub(crate) const IGNORED_DIRS: &[&str] = &[
+    // various dev tools that can be meet in the developer app
+    "node_modules",
+    "__pycache__",
+    "venv",
+    ".venv",
+    "target/debug",
+    "target/release",
+    "target/rust-analyzer",
+    "target/criterion",
+    // Language package caches in non-git roots.
+    "go/pkg/mod",
+    ".cargo/registry",
+    ".rustup/toolchains",
+    ".gradle/caches",
+    ".m2/repository",
+    ".npm/_cacache",
+    ".pub-cache",
+    #[cfg(not(target_os = "windows"))]
+    ".local/state", // this contains tons of logs which generate too much watcher noise
+    #[cfg(target_os = "macos")]
+    "Library/Application Support",
+    #[cfg(target_os = "macos")]
+    "Library/Caches",
+    #[cfg(target_os = "macos")]
+    "Library/Containers", // sandboxed apps data
+    #[cfg(target_os = "macos")]
+    "Library/Group Containers", // random application data and networking
+    #[cfg(target_os = "macos")]
+    "Library/pnpm",
+    #[cfg(target_os = "macos")]
+    "Library/Metadata",
+    #[cfg(target_os = "macos")]
+    "Library/Developer/CoreSimulator",
+    #[cfg(target_os = "macos")]
+    "Library/Android",
+    #[cfg(target_os = "macos")]
+    "Library/Logs",
+    #[cfg(target_os = "macos")]
+    "Library/Daemon Containers",
+    #[cfg(target_os = "macos")]
+    "Library/Trial",
+    #[cfg(target_os = "macos")]
+    "Library/Preferences",
+    #[cfg(target_os = "macos")]
+    "Library/Messages",
+    #[cfg(target_os = "macos")]
+    "Library/IdentityServices",
+    #[cfg(target_os = "windows")]
+    "bin/Debug",
+    #[cfg(target_os = "windows")]
+    "bin/Release",
+    #[cfg(target_os = "windows")]
+    "Program Files",
+    #[cfg(target_os = "windows")]
+    "Program Files (x86)",
+    #[cfg(target_os = "windows")]
+    "AppData/Local",
+    #[cfg(target_os = "windows")]
+    "AppData/Roaming",
+];
+
+#[cfg(all(not(feature = "zlob"), feature = "ripgrep"))]
+pub(crate) fn non_git_repo_overrides(base_path: &Path) -> Option<ignore::overrides::Override> {
+    use ignore::overrides::OverrideBuilder;
+
+    let mut builder = OverrideBuilder::new(base_path);
+    for dir in IGNORED_DIRS {
+        let pattern = format!("!**/{dir}/");
+        if let Err(e) = builder.add(&pattern) {
+            tracing::warn!("failed to add ignore pattern {pattern}: {e}");
+        }
+    }
+
+    builder.build().ok()
+}
+
+pub(crate) fn is_non_code_directory(path: &Path) -> bool {
+    let path_str = path.as_os_str().to_str().unwrap_or("");
+    IGNORED_DIRS.iter().any(|&dir| {
+        // Entries are gitignore patterns for the walkers; here they are matched
+        // as substrings, so a leading `*` wildcard has to come off first.
+        let dir = dir.strip_prefix('*').unwrap_or(dir);
+
+        #[cfg(target_os = "windows")]
+        let dir = dir.replace('/', std::path::MAIN_SEPARATOR_STR);
+        #[cfg(target_os = "windows")]
+        return path_str.contains(dir.as_str());
+
+        #[cfg(not(target_os = "windows"))]
+        path_str.contains(dir)
+    })
+}
+
+#[cfg(all(test, target_os = "macos"))]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn home_machine_state_is_excluded_but_source_trees_are_not() {
+        // Representative machine state from a home index.
+        for rel in [
+            "Library/pnpm/store/v3/files/00/abcdef",
+            "Library/Preferences/com.apple.finder.plist",
+            "Library/Messages/prewarm.db-shm",
+            "Library/IdentityServices/TetraDB-identityservicesd.db-wal",
+            "Library/Developer/CoreSimulator/Devices/X/data/f",
+            "go/pkg/mod/github.com/x/y@v1/main.go",
+            ".cargo/registry/src/index.crates.io-1/serde-1.0/src/lib.rs",
+            "Library/Android/sdk/platforms/android-34/data/x",
+            ".local/state/nvim/fff+123+456.log",
+        ] {
+            assert!(
+                is_non_code_directory(Path::new(rel)),
+                "{rel} must not reach the index"
+            );
+        }
+
+        // Source trees under $HOME stay searchable.
+        for rel in [
+            "dev/chromium/third_party/blink/renderer/core/dom/node.cc",
+            "dev/fff/crates/fff-core/src/lib.rs",
+            "Documents/notes/todo.md",
+            "dev/myproj/pkg/mod/thing.go",
+        ] {
+            assert!(
+                !is_non_code_directory(Path::new(rel)),
+                "{rel} must stay searchable"
+            );
+        }
+    }
+}

crates/fff-core/src/index/bigram_query.rs

@@ -0,0 +1,998 @@
+use crate::index::bigram_filter::BigramFilter;
+use regex_syntax::hir::{Class, Hir, HirKind};
+use smallvec::SmallVec;
+use std::borrow::Cow;
+
+const MAX_CLASS_EXPAND: usize = 16;
+
+// stack inlined array padded with 0 and tracked length
+#[derive(Clone, Copy, PartialEq, Eq)]
+struct InlineArray {
+    bytes: [u8; MAX_CLASS_EXPAND],
+    len: usize,
+}
+
+impl InlineArray {
+    const fn new() -> Self {
+        Self {
+            bytes: [0; MAX_CLASS_EXPAND],
+            len: 0,
+        }
+    }
+
+    fn from_byte(b: u8) -> Self {
+        let mut set = Self::new();
+        set.push(b);
+        set
+    }
+
+    /// Append a byte; no-op if already full (callers guard against this).
+    fn push(&mut self, b: u8) {
+        if self.len < MAX_CLASS_EXPAND {
+            self.bytes[self.len] = b;
+            self.len += 1;
+        }
+    }
+}
+
+impl std::ops::Deref for InlineArray {
+    type Target = [u8];
+
+    fn deref(&self) -> &[u8] {
+        &self.bytes[..self.len]
+    }
+}
+
+#[inline]
+fn consec_key(a: u8, b: u8) -> Option<u16> {
+    let al = a.to_ascii_lowercase();
+    let bl = b.to_ascii_lowercase();
+    if (32..=126).contains(&al) && (32..=126).contains(&bl) {
+        Some((al as u16) << 8 | bl as u16)
+    } else {
+        None
+    }
+}
+
+#[derive(Debug, Clone)]
+pub enum BigramQuery {
+    Any,
+    /// A consecutive bigram key to look up in the main index.
+    Consec(u16),
+    /// A skip-1 bigram key to look up in the skip sub-index.
+    Skip1(u16),
+    /// All children must match (intersect posting lists).
+    And(Vec<BigramQuery>),
+    /// At least one child must match (union posting lists).
+    Or(Vec<BigramQuery>),
+}
+
+/// SIMD-friendly bitwise OR of two equal-length bitsets.
+#[inline]
+fn bitset_or(a: &mut [u64], b: &[u64]) {
+    a.iter_mut().zip(b.iter()).for_each(|(x, y)| *x |= *y);
+}
+
+/// SIMD-friendly bitwise AND of two equal-length bitsets.
+#[inline]
+fn bitset_and(a: &mut [u64], b: &[u64]) {
+    a.iter_mut().zip(b.iter()).for_each(|(x, y)| *x &= *y);
+}
+
+impl BigramQuery {
+    pub fn is_any(&self) -> bool {
+        matches!(self, BigramQuery::Any)
+    }
+
+    pub(crate) fn evaluate(&self, index: &BigramFilter) -> Option<Vec<u64>> {
+        self.evaluate_cow(index).map(Cow::into_owned)
+    }
+
+    fn evaluate_cow<'a>(&self, index: &'a BigramFilter) -> Option<Cow<'a, [u64]>> {
+        match self {
+            BigramQuery::Any => None,
+
+            BigramQuery::Consec(key) => {
+                let col = index.lookup()[*key as usize];
+                if col == u16::MAX {
+                    return None;
+                }
+                let words = index.words();
+                let offset = col as usize * words;
+                let data = index.dense_data();
+                if offset + words > data.len() {
+                    return None;
+                }
+                Some(Cow::Borrowed(&data[offset..offset + words]))
+            }
+
+            BigramQuery::Skip1(key) => {
+                let skip = index.skip_index()?;
+                let col = skip.lookup()[*key as usize];
+                if col == u16::MAX {
+                    return None;
+                }
+                let words = skip.words();
+                let offset = col as usize * words;
+                let data = skip.dense_data();
+                if offset + words > data.len() {
+                    return None;
+                }
+                Some(Cow::Borrowed(&data[offset..offset + words]))
+            }
+
+            BigramQuery::And(children) => {
+                let mut result: Option<Vec<u64>> = None;
+                for child in children {
+                    if let Some(child_bits) = child.evaluate_cow(index) {
+                        result = Some(match result {
+                            None => child_bits.into_owned(),
+                            Some(mut r) => {
+                                bitset_and(&mut r, &child_bits);
+                                r
+                            }
+                        });
+                    }
+                }
+                result.map(Cow::Owned)
+            }
+
+            BigramQuery::Or(children) => {
+                if children.is_empty() {
+                    return None;
+                }
+                let mut result: Option<Vec<u64>> = None;
+                for child in children {
+                    // Any branch can't be filtered -> whole OR can't be filtered
+                    let child_bits = child.evaluate_cow(index)?;
+                    result = Some(match result {
+                        None => child_bits.into_owned(),
+                        Some(mut r) => {
+                            bitset_or(&mut r, &child_bits);
+                            r
+                        }
+                    });
+                }
+                result.map(Cow::Owned)
+            }
+        }
+    }
+}
+
+struct HirInfo {
+    query: BigramQuery,
+    first: Option<InlineArray>,
+    last: Option<InlineArray>,
+    can_be_empty: bool,
+}
+
+impl HirInfo {
+    fn empty() -> Self {
+        Self {
+            query: BigramQuery::Any,
+            first: None,
+            last: None,
+            can_be_empty: true,
+        }
+    }
+}
+
+/// Prefilter fuzzy query. The algorithm is the following:
+/// we allow max_typos = min(len/3,2) every typo destroys at most 2 consecutive bigrams
+/// So out of N bigrams at least N - 2 * max_typos have to present in the matching fil
+pub(crate) fn fuzzy_to_bigram_query(query: &str, num_probes: usize) -> BigramQuery {
+    let lower: Vec<u8> = query.bytes().map(|b| b.to_ascii_lowercase()).collect();
+
+    if lower.len() < 2 {
+        return BigramQuery::Any;
+    }
+
+    let max_typos = (lower.len() / 3).min(2);
+
+    // Extract all consecutive bigram keys.
+    let bigram_keys: Vec<u16> = lower
+        .windows(2)
+        .filter_map(|w| consec_key(w[0], w[1]))
+        .collect();
+
+    if bigram_keys.is_empty() {
+        return BigramQuery::Any;
+    }
+
+    // the simplest case, just check that every bigram is present either consec or not
+    if max_typos == 0 {
+        return simplify_and(
+            bigram_keys
+                .iter()
+                .map(|&k| BigramQuery::Consec(k))
+                .collect(),
+        );
+    }
+
+    // Pick evenly-spaced probe bigrams.
+    let n = num_probes.min(bigram_keys.len());
+    if n <= max_typos {
+        // Too few probes to require anything useful.
+        return simplify_or(
+            bigram_keys
+                .iter()
+                .map(|&k| BigramQuery::Consec(k))
+                .collect(),
+        );
+    }
+
+    let probes: Vec<u16> = if n == bigram_keys.len() {
+        bigram_keys
+    } else {
+        (0..n)
+            .map(|i| {
+                let idx = i * (bigram_keys.len() - 1) / (n - 1);
+                bigram_keys[idx]
+            })
+            .collect()
+    };
+
+    let required = n - max_typos;
+
+    // If required == n, just AND all probes.
+    if required >= n {
+        return simplify_and(probes.iter().map(|&k| BigramQuery::Consec(k)).collect());
+    }
+
+    // Generate all C(n, required) subsets as OR(AND(subset), ...)
+    let mut branches = Vec::new();
+    let mut combo = vec![0u16; required];
+    combine(&probes, required, 0, 0, &mut combo, &mut branches);
+
+    simplify_or(branches)
+}
+
+/// Build C(n, k) combination branches in-place on a fixed-size slice.
+fn combine(
+    items: &[u16],
+    k: usize,
+    start: usize,
+    depth: usize,
+    combo: &mut [u16],
+    branches: &mut Vec<BigramQuery>,
+) {
+    if depth == k {
+        branches.push(simplify_and(
+            combo.iter().map(|&key| BigramQuery::Consec(key)).collect(),
+        ));
+        return;
+    }
+    let remaining = k - depth;
+    for i in start..=items.len() - remaining {
+        combo[depth] = items[i];
+        combine(items, k, i + 1, depth + 1, combo, branches);
+    }
+}
+
+pub(crate) fn regex_to_bigram_query(pattern: &str) -> BigramQuery {
+    let mut parser = regex_syntax::ParserBuilder::new()
+        .unicode(false)
+        .utf8(false)
+        .build();
+
+    let hir = match parser.parse(pattern) {
+        Ok(h) => h,
+        Err(_) => return BigramQuery::Any,
+    };
+
+    decompose(&hir).query
+}
+
+fn decompose(hir: &Hir) -> HirInfo {
+    let can_be_empty = hir.properties().minimum_len().is_none_or(|n| n == 0);
+
+    match hir.kind() {
+        HirKind::Empty => HirInfo::empty(),
+
+        HirKind::Literal(lit) => decompose_literal(lit.0.as_ref()),
+
+        HirKind::Class(class) => {
+            let bytes = expand_class(class);
+            match bytes {
+                Some(b) if !b.is_empty() => HirInfo {
+                    query: BigramQuery::Any,
+                    first: Some(b),
+                    last: Some(b),
+                    can_be_empty,
+                },
+                _ => HirInfo {
+                    query: BigramQuery::Any,
+                    first: None,
+                    last: None,
+                    can_be_empty,
+                },
+            }
+        }
+
+        HirKind::Look(_) => HirInfo::empty(),
+
+        HirKind::Repetition(rep) => {
+            let inner = decompose(&rep.sub);
+            if rep.min == 0 {
+                HirInfo {
+                    query: BigramQuery::Any,
+                    first: inner.first,
+                    last: inner.last,
+                    can_be_empty: true,
+                }
+            } else {
+                // min >= 1: inner bigrams guaranteed
+                let mut qs = Vec::new();
+                if !inner.query.is_any() {
+                    qs.push(inner.query.clone());
+                }
+                // min >= 2: cross-boundary between consecutive occurrences
+                if rep.min >= 2 {
+                    push_cross_consec(&mut qs, inner.last.as_deref(), inner.first.as_deref());
+                }
+                HirInfo {
+                    query: simplify_and(qs),
+                    first: inner.first,
+                    last: inner.last,
+                    can_be_empty,
+                }
+            }
+        }
+
+        HirKind::Capture(cap) => decompose(&cap.sub),
+
+        HirKind::Concat(parts) => decompose_concat(parts),
+
+        HirKind::Alternation(alts) => decompose_alternation(alts),
+    }
+}
+
+/// Extract bigrams from a literal byte sequence.
+fn decompose_literal(bytes: &[u8]) -> HirInfo {
+    if bytes.is_empty() {
+        return HirInfo::empty();
+    }
+
+    let lower: SmallVec<[u8; 64]> = bytes.iter().map(|b| b.to_ascii_lowercase()).collect();
+
+    if lower.len() == 1 {
+        let b = lower[0];
+        let first = if (32..=126).contains(&b) {
+            Some(InlineArray::from_byte(b))
+        } else {
+            None
+        };
+        return HirInfo {
+            query: BigramQuery::Any,
+            first,
+            last: first,
+            can_be_empty: false,
+        };
+    }
+
+    let mut qs: Vec<BigramQuery> = Vec::new();
+
+    // Consecutive bigrams
+    for w in lower.windows(2) {
+        if let Some(k) = consec_key(w[0], w[1]) {
+            qs.push(BigramQuery::Consec(k));
+        }
+    }
+
+    // Skip-1 bigrams from the literal itself
+    if lower.len() >= 3 {
+        for i in 0..lower.len() - 2 {
+            if let Some(k) = consec_key(lower[i], lower[i + 2]) {
+                qs.push(BigramQuery::Skip1(k));
+            }
+        }
+    }
+
+    let first_byte = lower[0];
+    let last_byte = *lower.last().unwrap();
+
+    HirInfo {
+        query: simplify_and(qs),
+        first: if (32..=126).contains(&first_byte) {
+            Some(InlineArray::from_byte(first_byte))
+        } else {
+            None
+        },
+        last: if (32..=126).contains(&last_byte) {
+            Some(InlineArray::from_byte(last_byte))
+        } else {
+            None
+        },
+        can_be_empty: false,
+    }
+}
+
+fn decompose_concat(parts: &[Hir]) -> HirInfo {
+    if parts.is_empty() {
+        return HirInfo::empty();
+    }
+
+    let infos: Vec<HirInfo> = parts.iter().map(decompose).collect();
+    let mut qs: Vec<BigramQuery> = Vec::new();
+
+    for info in &infos {
+        if !info.query.is_any() {
+            qs.push(info.query.clone());
+        }
+    }
+
+    //  Dense cross-boundary between adjacent mandatory parts
+    for pair in infos.windows(2) {
+        if !pair[0].can_be_empty && !pair[1].can_be_empty {
+            push_cross_consec(&mut qs, pair[0].last.as_deref(), pair[1].first.as_deref());
+        }
+    }
+
+    //  Sparse-1 cross-boundary: across a single 1 byte wide middle part
+    if parts.len() >= 3 {
+        for i in 0..parts.len() - 2 {
+            let left = &infos[i];
+            let mid = &parts[i + 1];
+            let right = &infos[i + 2];
+
+            let min_len = mid.properties().minimum_len();
+            let max_len = mid.properties().maximum_len();
+            let is_1byte = min_len == Some(1) && max_len == Some(1);
+
+            if is_1byte && !left.can_be_empty && !right.can_be_empty {
+                push_cross_skip1(&mut qs, left.last.as_deref(), right.first.as_deref());
+            }
+        }
+    }
+
+    let first = collect_first(&infos);
+    let last = collect_last(&infos);
+    let can_be_empty = infos.iter().all(|i| i.can_be_empty);
+
+    HirInfo {
+        query: simplify_and(qs),
+        first,
+        last,
+        can_be_empty,
+    }
+}
+
+fn decompose_alternation(alts: &[Hir]) -> HirInfo {
+    if alts.is_empty() {
+        return HirInfo::empty();
+    }
+
+    let infos: Vec<HirInfo> = alts.iter().map(decompose).collect();
+    let query = simplify_or(infos.iter().map(|i| i.query.clone()).collect());
+    let first = merge_byte_sets(infos.iter().map(|i| &i.first));
+    let last = merge_byte_sets(infos.iter().map(|i| &i.last));
+    let can_be_empty = infos.iter().any(|i| i.can_be_empty);
+
+    HirInfo {
+        query,
+        first,
+        last,
+        can_be_empty,
+    }
+}
+
+fn expand_class(class: &Class) -> Option<InlineArray> {
+    let mut bytes = InlineArray::new();
+    match class {
+        Class::Bytes(bc) => {
+            for range in bc.ranges() {
+                let count = (range.end() as usize) - (range.start() as usize) + 1;
+                if bytes.len() + count > MAX_CLASS_EXPAND {
+                    return None;
+                }
+
+                for b in range.start()..=range.end() {
+                    if (32..=126).contains(&b) {
+                        let lower = b.to_ascii_lowercase();
+                        if !bytes.contains(&lower) {
+                            bytes.push(lower);
+                        }
+                    }
+                }
+            }
+        }
+        Class::Unicode(uc) => {
+            for range in uc.ranges() {
+                let start = range.start() as u32;
+                let end = range.end() as u32;
+                if start > 127 {
+                    continue;
+                }
+                let ascii_end = end.min(126) as u8;
+                let ascii_start = start.max(32) as u8;
+                if ascii_start > ascii_end {
+                    continue;
+                }
+                let count = (ascii_end - ascii_start) as usize + 1;
+                if bytes.len() + count > MAX_CLASS_EXPAND {
+                    return None;
+                }
+                for b in ascii_start..=ascii_end {
+                    let lower = b.to_ascii_lowercase();
+                    if !bytes.contains(&lower) {
+                        bytes.push(lower);
+                    }
+                }
+            }
+        }
+    }
+    if bytes.is_empty() { None } else { Some(bytes) }
+}
+
+/// Push consecutive cross-product bigrams into `qs`.
+fn push_cross_consec(qs: &mut Vec<BigramQuery>, last: Option<&[u8]>, first: Option<&[u8]>) {
+    if let Some(q) = cross_product(last, first, false) {
+        qs.push(q);
+    }
+}
+
+/// Push skip-1 cross-product bigrams into `qs`.
+fn push_cross_skip1(qs: &mut Vec<BigramQuery>, last: Option<&[u8]>, first: Option<&[u8]>) {
+    if let Some(q) = cross_product(last, first, true) {
+        qs.push(q);
+    }
+}
+
+fn cross_product(last: Option<&[u8]>, first: Option<&[u8]>, skip: bool) -> Option<BigramQuery> {
+    let last = last?;
+    let first = first?;
+    let n = last.len() * first.len();
+    if n == 0 || n > MAX_CLASS_EXPAND * MAX_CLASS_EXPAND {
+        return None;
+    }
+
+    let mut bigrams: Vec<BigramQuery> = Vec::with_capacity(n);
+    for &l in last {
+        for &f in first {
+            if let Some(k) = consec_key(l, f) {
+                let node = if skip {
+                    BigramQuery::Skip1(k)
+                } else {
+                    BigramQuery::Consec(k)
+                };
+                bigrams.push(node);
+            }
+        }
+    }
+
+    match bigrams.len() {
+        0 => None,
+        1 => Some(bigrams.into_iter().next().unwrap()),
+        _ => Some(simplify_or(bigrams)),
+    }
+}
+
+fn collect_first(infos: &[HirInfo]) -> Option<InlineArray> {
+    let mut result = InlineArray::new();
+    for info in infos {
+        if let Some(ref bytes) = info.first {
+            for &b in bytes.iter() {
+                if !result.contains(&b) {
+                    if result.len() >= MAX_CLASS_EXPAND {
+                        return None;
+                    }
+                    result.push(b);
+                }
+            }
+        } else if !info.can_be_empty {
+            return None;
+        }
+        if !info.can_be_empty {
+            break;
+        }
+    }
+    if result.is_empty() {
+        None
+    } else {
+        Some(result)
+    }
+}
+
+fn collect_last(infos: &[HirInfo]) -> Option<InlineArray> {
+    let mut result = InlineArray::new();
+    for info in infos.iter().rev() {
+        if let Some(ref bytes) = info.last {
+            for &b in bytes.iter() {
+                if !result.contains(&b) {
+                    if result.len() >= MAX_CLASS_EXPAND {
+                        return None;
+                    }
+                    result.push(b);
+                }
+            }
+        } else if !info.can_be_empty {
+            return None;
+        }
+        if !info.can_be_empty {
+            break;
+        }
+    }
+    if result.is_empty() {
+        None
+    } else {
+        Some(result)
+    }
+}
+
+fn merge_byte_sets<'a>(iter: impl Iterator<Item = &'a Option<InlineArray>>) -> Option<InlineArray> {
+    let mut result = InlineArray::new();
+    for opt in iter {
+        let bytes = opt.as_ref()?;
+
+        for &b in bytes.iter() {
+            if !result.contains(&b) {
+                if result.len() >= MAX_CLASS_EXPAND {
+                    return None;
+                }
+                result.push(b);
+            }
+        }
+    }
+    if result.is_empty() {
+        None
+    } else {
+        Some(result)
+    }
+}
+
+fn simplify_and(children: Vec<BigramQuery>) -> BigramQuery {
+    let mut flat: Vec<BigramQuery> = Vec::new();
+    for child in children {
+        match child {
+            BigramQuery::Any => {}
+            BigramQuery::And(inner) => flat.extend(inner),
+            other => flat.push(other),
+        }
+    }
+    match flat.len() {
+        0 => BigramQuery::Any,
+        1 => flat.into_iter().next().unwrap(),
+        _ => BigramQuery::And(flat),
+    }
+}
+
+fn simplify_or(children: Vec<BigramQuery>) -> BigramQuery {
+    if children.iter().any(|c| c.is_any()) {
+        return BigramQuery::Any;
+    }
+    let mut flat: Vec<BigramQuery> = Vec::new();
+    for child in children {
+        match child {
+            BigramQuery::Or(inner) => flat.extend(inner),
+            other => flat.push(other),
+        }
+    }
+    match flat.len() {
+        0 => BigramQuery::Any,
+        1 => flat.into_iter().next().unwrap(),
+        _ => BigramQuery::Or(flat),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::index::bigram_filter::BigramIndexBuilder;
+
+    /// Build a tiny index from the given file contents for testing.
+    fn build_test_index(files: &[&[u8]]) -> BigramFilter {
+        let n = files.len();
+        let consec_builder = BigramIndexBuilder::new(n);
+        let skip_builder = BigramIndexBuilder::new(n);
+        for (i, content) in files.iter().enumerate() {
+            consec_builder.add_file_content(&skip_builder, i, content);
+        }
+        let mut idx = consec_builder.compress(Some(0));
+        idx.set_skip_index(skip_builder.compress(Some(0)));
+        idx
+    }
+
+    #[test]
+    fn literal_pattern() {
+        let idx = build_test_index(&[
+            b"hello world",     // 0: contains "hello"
+            b"goodbye world",   // 1: no "hello"
+            b"say hello there", // 2: contains "hello"
+        ]);
+
+        let q = regex_to_bigram_query("hello");
+        assert!(!q.is_any());
+
+        let candidates = q.evaluate(&idx).unwrap();
+        assert!(BigramFilter::is_candidate(&candidates, 0));
+        assert!(!BigramFilter::is_candidate(&candidates, 1));
+        assert!(BigramFilter::is_candidate(&candidates, 2));
+    }
+
+    #[test]
+    fn alternation() {
+        let idx = build_test_index(&[
+            b"has foo in it", // 0
+            b"has bar in it", // 1
+            b"has xyz in it", // 2
+        ]);
+
+        let q = regex_to_bigram_query("foo|bar");
+        assert!(!q.is_any());
+
+        let candidates = q.evaluate(&idx).unwrap();
+        assert!(BigramFilter::is_candidate(&candidates, 0));
+        assert!(BigramFilter::is_candidate(&candidates, 1));
+        // xyz doesn't contain foo or bar bigrams
+        assert!(!BigramFilter::is_candidate(&candidates, 2));
+    }
+
+    #[test]
+    fn wildcard_concat() {
+        let idx = build_test_index(&[
+            b"foo something bar", // 0
+            b"foo only",          // 1: has foo but not bar
+            b"only bar",          // 2: has bar but not foo
+        ]);
+
+        let q = regex_to_bigram_query("foo.*bar");
+        assert!(!q.is_any());
+
+        let candidates = q.evaluate(&idx).unwrap();
+        assert!(BigramFilter::is_candidate(&candidates, 0));
+        // file 1 and 2 should be filtered (missing bigrams from "bar" / "foo")
+        assert!(!BigramFilter::is_candidate(&candidates, 1));
+        assert!(!BigramFilter::is_candidate(&candidates, 2));
+    }
+
+    #[test]
+    fn sparse1_across_dot() {
+        // "a.b" should produce a skip-1 bigram (a,b)
+        let idx = build_test_index(&[
+            b"axb", // 0: has sparse-1 (a,b)
+            b"ayb", // 1: has sparse-1 (a,b)
+            b"xyz", // 2: no (a,b) at all
+        ]);
+
+        let q = regex_to_bigram_query("a.b");
+        assert!(!q.is_any());
+
+        let candidates = q.evaluate(&idx).unwrap();
+        assert!(BigramFilter::is_candidate(&candidates, 0));
+        assert!(BigramFilter::is_candidate(&candidates, 1));
+        assert!(!BigramFilter::is_candidate(&candidates, 2));
+    }
+
+    #[test]
+    fn sparse1_across_digit() {
+        // "foo\dbar" -> sparse-1 (o,b) across \d
+        let idx = build_test_index(&[
+            b"foo3bar baz", // 0: has all bigrams
+            b"foobar baz",  // 1: has consecutive (o,b) but pattern needs sparse-1
+            b"xyz only",    // 2: no relevant bigrams
+        ]);
+
+        let q = regex_to_bigram_query(r"foo\dbar");
+        assert!(!q.is_any());
+
+        let candidates = q.evaluate(&idx).unwrap();
+        assert!(BigramFilter::is_candidate(&candidates, 0));
+        // file 1 may or may not match depending on what bigrams are in the index
+        // (it has all the literal bigrams and also o,b as both consec and skip-1)
+        // The important thing is file 2 is excluded:
+        assert!(!BigramFilter::is_candidate(&candidates, 2));
+    }
+
+    #[test]
+    fn pure_wildcard_is_any() {
+        let q = regex_to_bigram_query(".*");
+        assert!(q.is_any());
+    }
+
+    #[test]
+    fn single_char_is_any() {
+        let q = regex_to_bigram_query("a");
+        assert!(q.is_any());
+    }
+
+    #[test]
+    fn invalid_regex_is_any() {
+        let q = regex_to_bigram_query("[invalid");
+        assert!(q.is_any());
+    }
+
+    #[test]
+    fn optional_group_excluded() {
+        let q = regex_to_bigram_query("foo(bar)?baz");
+        assert!(!q.is_any());
+
+        let idx = build_test_index(&[
+            b"foobaz content",    // 0: has foo+baz bigrams (bar absent)
+            b"foobarbaz content", // 1: has everything
+            b"xyz only",          // 2: nothing
+        ]);
+
+        let candidates = q.evaluate(&idx).unwrap();
+        assert!(BigramFilter::is_candidate(&candidates, 0));
+        assert!(BigramFilter::is_candidate(&candidates, 1));
+        assert!(!BigramFilter::is_candidate(&candidates, 2));
+    }
+
+    #[test]
+    fn repetition_min2_cross_boundary() {
+        // (ab){2,} -> bigram "ab" + cross-boundary "b","a"
+        let q = regex_to_bigram_query("(ab){2,}");
+        assert!(!q.is_any());
+
+        let idx = build_test_index(&[
+            b"ababab", // 0: has "ab" and "b"->"a"
+            b"abonly", // 1: has "ab" but not "b"->"a"
+            b"xyz",    // 2: nothing
+        ]);
+
+        let candidates = q.evaluate(&idx).unwrap();
+        assert!(BigramFilter::is_candidate(&candidates, 0));
+        assert!(!BigramFilter::is_candidate(&candidates, 2));
+    }
+
+    #[test]
+    fn two_dots_no_sparse1() {
+        let q = regex_to_bigram_query("a..b");
+        // Single-char literals with 2 unknown bytes between -> Any
+        assert!(q.is_any());
+    }
+
+    #[test]
+    fn character_class_cross_boundary() {
+        // [abc]de -> cross-boundary OR(ad,bd,cd) + bigram de
+        // All three class variants must appear in the corpus so the OR
+        // branches are tracked in the index (untracked bigrams make the
+        // OR conservatively return None, which is correct but untestable).
+        let idx = build_test_index(&[
+            b"ade content", // 0: has ad
+            b"bde content", // 1: has bd
+            b"cde content", // 2: has cd
+            b"xde content", // 3: has de but not ad/bd/cd
+        ]);
+
+        let q = regex_to_bigram_query("[abc]de");
+        assert!(!q.is_any());
+
+        let candidates = q.evaluate(&idx).unwrap();
+        assert!(BigramFilter::is_candidate(&candidates, 0));
+        assert!(BigramFilter::is_candidate(&candidates, 1));
+        assert!(BigramFilter::is_candidate(&candidates, 2));
+        // file 3 doesn't have ad/bd/cd so should be filtered
+        assert!(!BigramFilter::is_candidate(&candidates, 3));
+    }
+
+    fn has_consec(q: &BigramQuery, a: u8, b: u8) -> bool {
+        let Some(key) = consec_key(a, b) else {
+            return false;
+        };
+        match q {
+            BigramQuery::Consec(k) => *k == key,
+            BigramQuery::And(cs) | BigramQuery::Or(cs) => cs.iter().any(|c| has_consec(c, a, b)),
+            _ => false,
+        }
+    }
+
+    fn has_skip1(q: &BigramQuery, a: u8, b: u8) -> bool {
+        let Some(key) = consec_key(a, b) else {
+            return false;
+        };
+        match q {
+            BigramQuery::Skip1(k) => *k == key,
+            BigramQuery::And(cs) | BigramQuery::Or(cs) => cs.iter().any(|c| has_skip1(c, a, b)),
+            _ => false,
+        }
+    }
+
+    /// Bigram expectation: `("ab", is_skip1)`.
+    /// The 2-char str is the byte pair; C = consecutive, S = skip-1.
+    type Bg = (&'static str, bool);
+    const C: bool = false;
+    const S: bool = true;
+
+    /// Top 15+ commonly used regex patterns from
+    /// https://digitalfortress.tech/tips/top-15-commonly-used-regex/
+    /// plus typical grep patterns used by agentic tools.
+    ///
+    /// Each entry: `(regex, Option<&[Bg]>)`.
+    ///   - `None`        -> pure classes / unsupported syntax, Any is acceptable.
+    ///   - `Some(&[..])` -> must be non-Any, and every listed bigram must appear.
+    #[test]
+    fn common_regex_patterns() {
+        #[rustfmt::skip]
+        let cases: &[(&str, Option<&[Bg]>)] = &[
+            (r"^\d+$",                                                      None), // 1.  whole numbers
+            (r"^\d*\.\d+$",                                                 None), // 2.  decimals
+            (r"^\d*(\.\d+)?$",                                              None), // 3.  whole + decimal
+            (r"^-?\d*(\.\d+)?$",                                            None), // 4.  neg/pos decimal
+            (r"[-]?[0-9]+[,.]?[0-9]*([/][0-9]+[,.]?[0-9]*)*",             None), // 5.  fractions
+            (r"^[a-zA-Z0-9]*$",                                             None), // 6.  alphanumeric
+            (r"^[a-zA-Z0-9 ]*$",                                            None), // 7.  alphanum + space
+            (r"^([a-zA-Z0-9._%-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6})*$",      None), // 8.  email
+            (r"^([a-z0-9_\.\+-]+)@([\da-z\.-]+)\.([a-z\.]{2,6})$",         None), // 9.  email v2
+            (r"(?=(.*[0-9]))(?=.*[!@#$%^&*()\[\]{}\-_+=~`|:;<>,./?\x5c])(?=.*[a-z])(?=(.*[A-Z]))(?=(.*)).{8,}", None), // 10. complex pw
+            (r"(?=(.*[0-9]))((?=.*[A-Za-z0-9])(?=.*[A-Z])(?=.*[a-z]))^.{8,}$", None), // 11. moderate pw
+            (r"^[a-z0-9_-]{3,16}$",                                        None), // 12. username
+            (r"(https?://)?(www\.)?[-a-zA-Z0-9@:%._\+~#=]{2,256}\.[a-z]{2,6}\b([-a-zA-Z0-9@:%_\+.~#?&//=]*)", None), // 14. URL optional
+            (r"^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$", None), // 15. IPv4
+            (r"(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))", None), // 16. IPv6
+            (r"[12]\d{3}-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01])",          None), // 17. date
+            (r"^(0?[1-9]|1[0-2]):[0-5][0-9]$",                            None), // 18. time 12h
+            (r"((1[0-2]|0?[1-9]):([0-5][0-9]) ?([AaPp][Mm]))",            None), // 19. time AM/PM
+            (r"^(0[0-9]|1[0-9]|2[0-3]):[0-5][0-9]$",                     None), // 20. time 24h
+            (r"^([0-9]|0[0-9]|1[0-9]|2[0-3]):[0-5][0-9]$",               None), // 21. time 24h v2
+            (r"(?:[01]\d|2[0123]):(?:[012345]\d):(?:[012345]\d)",          None), // 22. time+sec
+            (r"</?[\w\s]*>|<.+[\W]>",                                      None), // 23. HTML tag
+            (r"\bon\w+=\S+(?=.*>)",                                        None), // 24. inline JS
+            (r"^[a-z0-9]+(?:-[a-z0-9]+)*$",                               None), // 25. slug
+            (r"(\b\w+\b)(?=.*\b\1\b)",                                    None), // 26. dup words
+            (r"^[\w,\s-]+\.[A-Za-z]{3}$",                                 None), // 27. filename
+            (r"^[A-PR-WY][1-9]\d\s?\d{4}[1-9]$",                         None), // 28. HK ID
+
+            // 13. URL with required protocol
+            (r"https?://(www\.)?[-a-zA-Z0-9@:%._\+~#=]{2,256}\.[a-z]{2,6}\b([-a-zA-Z0-9@:%_\+.~#?&//=]*)", Some(&[
+                ("ht", C), ("tt", C), ("tp", C),    // from "http"
+                ("ht", S), ("tp", S),               // from "http" skip-1
+                (":/", C), ("//", C),               // from "://"
+            ])),
+
+            // 29. fn\s+\w+
+            (r"fn\s+\w+", Some(&[
+                ("fn", C),                          // from "fn"
+                ("n ", C),                          // cross-boundary: 'n' -> \s starts ' '
+            ])),
+
+            // 30. use\s+crate::
+            (r"use\s+crate::", Some(&[
+                ("us", C), ("se", C), ("ue", S),    // from "use"
+                ("cr", C), ("ra", C), ("at", C),    // from "crate"
+                ("te", C), ("::", C),
+                ("ca", S), ("rt", S), ("ae", S),    // "crate" skip-1
+            ])),
+
+            // 31. unwrap\(\)|expect\(
+            (r"unwrap\(\)|expect\(", Some(&[
+                ("nw", C), ("wr", C), ("ra", C),    // "unwrap("
+                ("ap", C), ("p(", C),
+                ("xp", C), ("pe", C), ("ec", C),    // "expect("
+                ("ct", C), ("t(", C),
+            ])),
+
+            // 32. TODO|FIXME|HACK
+            (r"TODO|FIXME|HACK", Some(&[
+                ("to", C), ("od", C), ("do", C),    // "TODO"
+                ("fi", C), ("ix", C), ("xm", C),    // "FIXME"
+                ("me", C),
+                ("ha", C), ("ac", C), ("ck", C),    // "HACK"
+                ("hc", S), ("ak", S),               // "HACK" skip-1
+            ])),
+        ];
+
+        for (i, &(pattern, expected)) in cases.iter().enumerate() {
+            let q = regex_to_bigram_query(pattern);
+
+            if let Some(bigrams) = expected {
+                assert!(
+                    !q.is_any(),
+                    "#{i} {pattern:?}: expected bigrams but got Any"
+                );
+
+                for &(pair, skip) in bigrams {
+                    let b = pair.as_bytes();
+                    debug_assert_eq!(b.len(), 2, "bigram must be 2 chars: {pair:?}");
+                    let found = if skip {
+                        has_skip1(&q, b[0], b[1])
+                    } else {
+                        has_consec(&q, b[0], b[1])
+                    };
+                    let kind = if skip { "skip-1" } else { "consec" };
+                    assert!(found, "#{i} {pattern:?}: missing {kind} bigram {pair:?}");
+                }
+            }
+        }
+    }
+}

crates/fff-core/src/index/candidates.rs

@@ -0,0 +1,118 @@
+use super::{BigramFilter, BigramOverlay, extract_bigrams};
+use super::{fuzzy_to_bigram_query, regex_to_bigram_query};
+
+/// Number of evenly-spaced probe bigrams used by the fuzzy candidate query.
+const FUZZY_PROBE_COUNT: usize = 7;
+
+#[inline]
+fn set_bit(candidates: &mut [u64], file_idx: usize) {
+    let word = file_idx / 64;
+    if word < candidates.len() {
+        candidates[word] |= 1u64 << (file_idx % 64);
+    }
+}
+
+#[inline]
+fn clear_tombstones(candidates: &mut [u64], overlay: &BigramOverlay) {
+    for (r, t) in candidates.iter_mut().zip(overlay.tombstones().iter()) {
+        *r &= !t;
+    }
+}
+
+/// Number of base files covered by the bigram bitset; files past this
+/// boundary (overflow, max 1024) are always scanned.
+#[inline]
+pub(crate) fn bigram_boundary(overlay: Option<&BigramOverlay>, files_len: usize) -> usize {
+    overlay.map(|o| o.base_file_count()).unwrap_or(files_len)
+}
+
+/// Candidate bitset for literal patterns, OR-ed across all of them: a file is
+/// a candidate when it contains the bigrams of ANY pattern. Overlay-modified
+/// files are re-checked against each pattern's bigrams.
+pub(crate) fn literal_candidates(
+    index: Option<&BigramFilter>,
+    overlay: Option<&BigramOverlay>,
+    patterns: &[&str],
+) -> Option<Vec<u64>> {
+    let index = ready_index(index)?;
+
+    let mut combined: Option<Vec<u64>> = None;
+    for pattern in patterns {
+        if let Some(candidates) = index.query(pattern.as_bytes()) {
+            combined = Some(match combined {
+                None => candidates,
+                Some(mut acc) => {
+                    acc.iter_mut()
+                        .zip(candidates.iter())
+                        .for_each(|(a, b)| *a |= *b);
+                    acc
+                }
+            });
+        }
+    }
+
+    let mut candidates = combined?;
+    if let Some(overlay) = overlay {
+        clear_tombstones(&mut candidates, overlay);
+        for pattern in patterns {
+            let pattern_bigrams = extract_bigrams(pattern.as_bytes());
+            for file_idx in overlay.query_modified(&pattern_bigrams) {
+                set_bit(&mut candidates, file_idx);
+            }
+        }
+    }
+    Some(candidates)
+}
+
+/// Candidate bitset for a regex pattern: the regex HIR is decomposed into an
+/// AND/OR bigram query tree (supports alternation, optional groups, character
+/// classes, and sparse-1 bigrams across single-byte wildcards). Since modified
+/// file contents can't be re-checked against a regex cheaply, all
+/// overlay-modified files are conservatively added.
+pub(crate) fn regex_candidates(
+    index: Option<&BigramFilter>,
+    overlay: Option<&BigramOverlay>,
+    pattern: &str,
+) -> Option<Vec<u64>> {
+    let index = ready_index(index)?;
+
+    let bq = regex_to_bigram_query(pattern);
+    if bq.is_any() {
+        return None;
+    }
+    let candidates = bq.evaluate(index)?;
+    Some(add_all_modified(candidates, overlay))
+}
+
+/// Candidate bitset for a fuzzy pattern: evenly-spaced probe bigrams with a
+/// typo allowance (widely-spaced probes are far more selective than sliding
+/// windows of adjacent bigrams). All overlay-modified files are added.
+pub(crate) fn fuzzy_candidates(
+    index: Option<&BigramFilter>,
+    overlay: Option<&BigramOverlay>,
+    pattern: &str,
+) -> Option<Vec<u64>> {
+    let index = ready_index(index)?;
+
+    let bq = fuzzy_to_bigram_query(pattern, FUZZY_PROBE_COUNT);
+    if bq.is_any() {
+        return None;
+    }
+    let candidates = bq.evaluate(index)?;
+    Some(add_all_modified(candidates, overlay))
+}
+
+#[inline]
+fn ready_index(index: Option<&BigramFilter>) -> Option<&BigramFilter> {
+    index.filter(|idx| idx.is_ready())
+}
+
+fn add_all_modified(mut candidates: Vec<u64>, overlay: Option<&BigramOverlay>) -> Vec<u64> {
+    if let Some(overlay) = overlay {
+        clear_tombstones(&mut candidates, overlay);
+        for file_idx in overlay.modified_indices() {
+            set_bit(&mut candidates, file_idx);
+        }
+    }
+    candidates
+}

crates/fff-core/src/index/constraints.rs

@@ -0,0 +1,950 @@
+//! Constraint-based prefiltering for search queries.
+
+use fff_query_parser::{Constraint, GitStatusFilter};
+use smallvec::SmallVec;
+
+use crate::git::is_modified_status;
+use crate::simd_path::ArenaPtr;
+use crate::simd_string_utils::memmem::find_case_insensitive_short;
+
+const PAR_THRESHOLD: usize = 10_000;
+
+pub(crate) trait Constrainable {
+    fn write_file_name(&self, arena: ArenaPtr, out: &mut String);
+    fn git_status(&self) -> Option<git2::Status>;
+    fn write_relative_path(&self, arena: ArenaPtr, out: &mut String);
+    fn is_overflow(&self) -> bool;
+}
+
+/// Stored/canonical paths use `/`; also accept `\` so a Windows user typing
+/// a native separator in a query still matches.
+#[inline]
+fn is_path_sep(b: u8) -> bool {
+    b == b'/' || b == b'\\'
+}
+
+#[inline]
+fn path_slice_eq(a: &[u8], b: &[u8]) -> bool {
+    if a.len() != b.len() {
+        return false;
+    }
+    a.iter().zip(b).all(|(x, y)| {
+        if is_path_sep(*x) && is_path_sep(*y) {
+            true
+        } else {
+            x.eq_ignore_ascii_case(y)
+        }
+    })
+}
+
+/// Path ends with suffix at a path-separator boundary (case-insensitive).
+#[inline]
+pub fn path_ends_with_suffix(path: &str, suffix: &str) -> bool {
+    let path_bytes = path.as_bytes();
+    let suffix_bytes = suffix.as_bytes();
+    if path_bytes.len() < suffix_bytes.len() {
+        return false;
+    }
+
+    let start = path.len() - suffix.len();
+
+    // Multi-byte UTF-8 may put `start` inside a char.
+    if !path.is_char_boundary(start) {
+        return false;
+    }
+
+    if !path_slice_eq(&path_bytes[start..], suffix_bytes) {
+        return false;
+    }
+
+    // Exact or preceded by a separator. Scan backward past any multi-byte
+    // continuation bytes to find the preceding ASCII byte.
+    if start == 0 {
+        return true;
+    }
+    let mut i = start;
+    while i > 0 {
+        i -= 1;
+        if path_bytes[i] < 128 {
+            return is_path_sep(path_bytes[i]);
+        }
+    }
+    false
+}
+
+#[inline]
+pub fn file_has_extension(file_name: &str, ext: &str) -> bool {
+    let name_bytes = file_name.as_bytes();
+    let ext_bytes = ext.as_bytes();
+    if name_bytes.len() <= ext_bytes.len() + 1 {
+        return false;
+    }
+    let start = name_bytes.len() - ext_bytes.len() - 1;
+    if start > 0 && !file_name.is_char_boundary(start) {
+        return false;
+    }
+    name_bytes.get(start) == Some(&b'.') && name_bytes[start + 1..].eq_ignore_ascii_case(ext_bytes)
+}
+
+/// Matches multi-segment queries like `libswscale/aarch64`.
+#[inline]
+pub fn path_contains_segment(path: &str, segment: &str) -> bool {
+    let path_bytes = path.as_bytes();
+    let segment_bytes = segment.as_bytes();
+    let segment_len = segment_bytes.len();
+
+    if path_bytes.len() > segment_len
+        && is_path_sep(path_bytes[segment_len])
+        && path.is_char_boundary(segment_len)
+        && path_slice_eq(&path_bytes[..segment_len], segment_bytes)
+    {
+        return true;
+    }
+
+    if path_bytes.len() < segment_len + 2 {
+        return false;
+    }
+
+    for i in 0..path_bytes.len().saturating_sub(segment_len + 1) {
+        if is_path_sep(path_bytes[i]) {
+            let start = i + 1;
+            let end = start + segment_len;
+            if end < path_bytes.len()
+                && is_path_sep(path_bytes[end])
+                && path.is_char_boundary(start)
+                && path.is_char_boundary(end)
+                && path_slice_eq(&path_bytes[start..end], segment_bytes)
+            {
+                return true;
+            }
+        }
+    }
+    false
+}
+
+/// Returns `None` if no constraints are present, `Some(filtered)` otherwise.
+///
+/// Constraint semantics:
+/// - All `Extension` constraints OR together (file matches if ANY extension hits).
+///   They're split out up front so the per-item loop reads the OR predicate as a
+///   single short-circuit check, not as N AND-merged sub-constraints.
+/// - Every other constraint kind ANDs (file matches only if ALL hold). They're
+///   evaluated in order with short-circuit on first failure.
+pub(crate) fn apply_constraints<'a, T: Constrainable + Sync>(
+    items: &'a [T],
+    constraints: &[Constraint<'_>],
+    base_arena: ArenaPtr,
+    overflow_arena: ArenaPtr,
+) -> Option<Vec<&'a T>> {
+    if constraints.is_empty() {
+        return None;
+    }
+    let plan = ConstraintPlan::build(constraints, items, base_arena, overflow_arena);
+    Some(plan.run(items, base_arena, overflow_arena))
+}
+
+#[cfg(feature = "zlob")]
+pub(crate) type GlobPattern = zlob::ZlobPattern;
+#[cfg(all(not(feature = "zlob"), feature = "ripgrep"))]
+pub(crate) type GlobPattern = globset::GlobMatcher;
+
+/// How `Constraint::Glob` is evaluated for each item.
+enum GlobStrategy {
+    /// No Glob constraint present.
+    None,
+    /// Pure-glob workload (no Extension filter to reject items first).
+    /// Batch all paths through zlob/globset once; per-item check is a Vec<bool> lookup.
+    Prepass(Vec<Vec<bool>>),
+    /// Mixed workload (Extension filter present). Compile patterns up front, then
+    /// only run them on items that survive the cheap Extension OR check.
+    /// `None` slot = compile failure -> never matches; preserves index alignment.
+    Inline(Vec<Option<GlobPattern>>),
+}
+
+/// Bundles preprocessed constraints for the per-item evaluator.
+pub(crate) struct ConstraintPlan<'q, 'c> {
+    /// OR semantics — file passes if ANY extension matches. Empty = no ext filter.
+    extensions: SmallVec<[&'q str; 8]>,
+    /// AND semantics — file passes only if ALL match.
+    rest: SmallVec<[&'c Constraint<'q>; 8]>,
+    glob: GlobStrategy,
+}
+
+pub(crate) struct ConstraintsBuffers {
+    fname: String,
+    path: String,
+}
+
+impl ConstraintsBuffers {
+    pub(crate) fn new() -> Self {
+        Self {
+            fname: String::with_capacity(64),
+            path: String::with_capacity(64),
+        }
+    }
+}
+
+impl<'q, 'c> ConstraintPlan<'q, 'c> {
+    pub(crate) fn build<T: Constrainable>(
+        constraints: &'c [Constraint<'q>],
+        items: &[T],
+        base_arena: ArenaPtr,
+        overflow_arena: ArenaPtr,
+    ) -> Self {
+        let mut extensions = SmallVec::new();
+        let mut rest: SmallVec<[&'c Constraint<'q>; 8]> = SmallVec::new();
+        for c in constraints {
+            match c {
+                Constraint::Extension(ext) => extensions.push(*ext),
+                _ => rest.push(c),
+            }
+        }
+        let has_pre_filter = !extensions.is_empty() || rest.iter().any(|&c| !is_glob_node(c));
+        let glob = build_glob_strategy(&rest, has_pre_filter, items, base_arena, overflow_arena);
+
+        Self {
+            extensions,
+            rest,
+            glob,
+        }
+    }
+
+    fn run<'a, T: Constrainable + Sync>(
+        &self,
+        items: &'a [T],
+        base_arean: ArenaPtr,
+        overflow_arena: ArenaPtr,
+    ) -> Vec<&'a T> {
+        if items.len() >= PAR_THRESHOLD {
+            use rayon::prelude::*;
+            items
+                .par_iter()
+                .enumerate()
+                .map_init(ConstraintsBuffers::new, |scratch, (i, item)| {
+                    self.matches(item, i, base_arean, overflow_arena, scratch)
+                        .then_some(item)
+                })
+                .flatten()
+                .collect()
+        } else {
+            let mut scratch = ConstraintsBuffers::new();
+            items
+                .iter()
+                .enumerate()
+                .filter_map(|(i, item)| {
+                    self.matches(item, i, base_arean, overflow_arena, &mut scratch)
+                        .then_some(item)
+                })
+                .collect()
+        }
+    }
+
+    #[inline]
+    pub(crate) fn matches<T: Constrainable>(
+        &self,
+        item: &T,
+        index: usize,
+        base_arena: ArenaPtr,
+        overflow_arena: ArenaPtr,
+        scratch: &mut ConstraintsBuffers,
+    ) -> bool {
+        let arena = if item.is_overflow() {
+            overflow_arena
+        } else {
+            base_arena
+        };
+
+        if !self.passes_extensions(item, arena, scratch) {
+            return false;
+        }
+
+        let mut glob_idx = 0;
+        self.rest.iter().all(|c| {
+            evaluate(
+                item,
+                index,
+                c,
+                &self.glob,
+                &mut glob_idx,
+                false,
+                arena,
+                scratch,
+            )
+        })
+    }
+
+    #[inline]
+    fn passes_extensions<T: Constrainable>(
+        &self,
+        item: &T,
+        arena: ArenaPtr,
+        scratch: &mut ConstraintsBuffers,
+    ) -> bool {
+        if self.extensions.is_empty() {
+            return true;
+        }
+        item.write_file_name(arena, &mut scratch.fname);
+        self.extensions
+            .iter()
+            .any(|ext| file_has_extension(&scratch.fname, ext))
+    }
+}
+
+#[inline]
+#[allow(clippy::too_many_arguments)]
+fn evaluate<T: Constrainable>(
+    item: &T,
+    index: usize,
+    constraint: &Constraint<'_>,
+    glob: &GlobStrategy,
+    glob_idx: &mut usize,
+    negate: bool,
+    arena: ArenaPtr,
+    scratch: &mut ConstraintsBuffers,
+) -> bool {
+    let raw = match constraint {
+        Constraint::Glob(_) => {
+            let m = match glob {
+                GlobStrategy::None => true,
+                GlobStrategy::Prepass(masks) => masks
+                    .get(*glob_idx)
+                    .and_then(|mask| mask.get(index).copied())
+                    .unwrap_or(false),
+                GlobStrategy::Inline(patterns) => {
+                    item.write_relative_path(arena, &mut scratch.path);
+                    patterns
+                        .get(*glob_idx)
+                        .and_then(|p| p.as_ref())
+                        .map(|p| compiled_matches(p, &scratch.path))
+                        .unwrap_or(false)
+                }
+            };
+            *glob_idx += 1;
+            m
+        }
+        // Reachable only via `Not(Extension(_))` — bare extensions are split out
+        // up front and handled in `passes_extensions`.
+        Constraint::Extension(ext) => {
+            item.write_file_name(arena, &mut scratch.fname);
+            file_has_extension(&scratch.fname, ext)
+        }
+        Constraint::PathSegment(segment) => {
+            item.write_relative_path(arena, &mut scratch.path);
+            path_contains_segment(&scratch.path, segment)
+        }
+        Constraint::FilePath(suffix) => {
+            item.write_relative_path(arena, &mut scratch.path);
+            path_ends_with_suffix(&scratch.path, suffix)
+        }
+        Constraint::Text(text) => {
+            // Only meaningful under negation (used as exclude filter).
+            item.write_relative_path(arena, &mut scratch.path);
+            find_case_insensitive_short(scratch.path.as_bytes(), text.as_bytes()).is_some()
+        }
+        Constraint::GitStatus(filter) => matches_git_status(item.git_status(), filter),
+        Constraint::Not(inner) => {
+            return evaluate(item, index, inner, glob, glob_idx, !negate, arena, scratch);
+        }
+        // Pass-throughs — handled at higher levels.
+        Constraint::Parts(_) | Constraint::Exclude(_) | Constraint::FileType(_) => true,
+    };
+    if negate { !raw } else { raw }
+}
+
+#[inline]
+fn matches_git_status(status: Option<git2::Status>, filter: &GitStatusFilter) -> bool {
+    match (status, filter) {
+        (Some(s), GitStatusFilter::Modified) => is_modified_status(s),
+        (Some(s), GitStatusFilter::Untracked) => s.contains(git2::Status::WT_NEW),
+        (Some(s), GitStatusFilter::Staged) => s.intersects(
+            git2::Status::INDEX_NEW
+                | git2::Status::INDEX_MODIFIED
+                | git2::Status::INDEX_DELETED
+                | git2::Status::INDEX_RENAMED
+                | git2::Status::INDEX_TYPECHANGE,
+        ),
+        (Some(s), GitStatusFilter::Unmodified) => s.is_empty(),
+        (None, GitStatusFilter::Unmodified) => true,
+        (None, _) => false,
+    }
+}
+
+#[inline]
+#[cfg(feature = "zlob")]
+pub(crate) fn compiled_matches(p: &GlobPattern, path: &str) -> bool {
+    p.matches_default(path)
+}
+
+#[inline]
+#[cfg(all(not(feature = "zlob"), feature = "ripgrep"))]
+pub(crate) fn compiled_matches(p: &GlobPattern, path: &str) -> bool {
+    p.is_match(path)
+}
+
+/// Append indices (into `rels`) of paths matching `p`, in input order.
+/// zlob backend: ONE FFI call for the whole batch.
+#[cfg(feature = "zlob")]
+pub(crate) fn glob_matches_into(p: &GlobPattern, rels: &[&str], out: &mut Vec<usize>) {
+    match p.match_indices(rels, p.flags()) {
+        Ok(ix) => out.extend_from_slice(ix.as_slice()),
+        Err(e) => {
+            tracing::warn!(?e, "zlob batch match failed, falling back to per-path");
+            out.extend((0..rels.len()).filter(|&i| p.matches_default(rels[i])));
+        }
+    }
+}
+
+#[cfg(all(not(feature = "zlob"), feature = "ripgrep"))]
+pub(crate) fn glob_matches_into(p: &GlobPattern, rels: &[&str], out: &mut Vec<usize>) {
+    out.extend((0..rels.len()).filter(|&i| p.is_match(rels[i])));
+}
+
+/// Decide between batch prepass and inline compiled patterns.
+///
+/// `has_pre_filter` = true when something cheaper than glob can reject items first
+/// (extensions OR non-glob constraints in `rest`). In that case inline pays glob
+/// cost only on survivors and beats prepass on every workload we benched. Pure-glob
+/// (no pre-filter) takes prepass — single batched zlob call beats N inline matches.
+fn build_glob_strategy<T: Constrainable>(
+    rest: &[&Constraint<'_>],
+    has_pre_filter: bool,
+    items: &[T],
+    arena: ArenaPtr,
+    overflow_arena: ArenaPtr,
+) -> GlobStrategy {
+    if !contains_glob(rest) {
+        return GlobStrategy::None;
+    }
+    if has_pre_filter {
+        return GlobStrategy::Inline(compile_globs(rest));
+    }
+    let buf = PathBuffer::collect(items, arena, overflow_arena);
+    let path_refs = buf.as_strs();
+    GlobStrategy::Prepass(precompute_masks(rest, &path_refs))
+}
+
+/// `Glob` or `Not(Glob)` — the constraint kinds whose evaluation goes through
+/// the GlobStrategy. Everything else can pre-reject items before glob runs.
+fn is_glob_node(c: &Constraint<'_>) -> bool {
+    match c {
+        Constraint::Glob(_) => true,
+        Constraint::Not(inner) => is_glob_node(inner),
+        _ => false,
+    }
+}
+
+fn contains_glob(rest: &[&Constraint<'_>]) -> bool {
+    rest.iter().any(|c| is_glob_node(c))
+}
+
+/// Contiguous byte buffer holding every item's `relative_path`. Single allocation
+/// instead of N `String`s. On Windows the in-place pass folds `\\` -> `/` so the
+/// glob library sees a canonical separator.
+struct PathBuffer {
+    bytes: Vec<u8>,
+    offsets: Vec<(usize, usize)>,
+}
+
+impl PathBuffer {
+    fn collect<T: Constrainable>(items: &[T], arena: ArenaPtr, overflow_arena: ArenaPtr) -> Self {
+        let mut bytes = Vec::<u8>::new();
+        let mut offsets = Vec::with_capacity(items.len());
+        let mut tmp = String::with_capacity(64);
+        for item in items {
+            let item_arena = if item.is_overflow() {
+                overflow_arena
+            } else {
+                arena
+            };
+            let start = bytes.len();
+            item.write_relative_path(item_arena, &mut tmp);
+            bytes.extend_from_slice(tmp.as_bytes());
+            offsets.push((start, bytes.len() - start));
+        }
+        Self { bytes, offsets }
+    }
+
+    fn as_strs(&self) -> Vec<&str> {
+        self.offsets
+            .iter()
+            .map(|&(off, len)| unsafe {
+                std::str::from_utf8_unchecked(&self.bytes[off..off + len])
+            })
+            .collect()
+    }
+}
+
+fn precompute_masks(rest: &[&Constraint<'_>], paths: &[&str]) -> Vec<Vec<bool>> {
+    let mut out = Vec::new();
+    for c in rest {
+        walk_globs(c, &mut |pattern| {
+            out.push(match_glob_pattern(pattern, paths))
+        });
+    }
+    out
+}
+
+fn compile_globs(rest: &[&Constraint<'_>]) -> Vec<Option<GlobPattern>> {
+    let mut out = Vec::new();
+    for c in rest {
+        walk_globs(c, &mut |pattern| out.push(compile_one(pattern)));
+    }
+    out
+}
+
+/// Visit every Glob (including ones nested under Not) in constraint walk order.
+/// Order matters: `glob_idx` in the per-item evaluator increments by one per Glob node.
+fn walk_globs<F: FnMut(&str)>(c: &Constraint<'_>, f: &mut F) {
+    match c {
+        Constraint::Glob(p) => f(p),
+        Constraint::Not(inner) => walk_globs(inner, f),
+        _ => {}
+    }
+}
+
+#[cfg(feature = "zlob")]
+pub(crate) fn compile_one(pattern: &str) -> Option<GlobPattern> {
+    zlob::ZlobPattern::compile(pattern, zlob::ZlobFlags::RECOMMENDED).ok()
+}
+
+#[cfg(all(not(feature = "zlob"), feature = "ripgrep"))]
+pub(crate) fn compile_one(pattern: &str) -> Option<GlobPattern> {
+    globset::Glob::new(pattern)
+        .ok()
+        .map(|g| g.compile_matcher())
+}
+
+/// Build a `paths.len()`-sized bitmap. Vec<bool> beats AHashSet ~2× in the per-item
+/// filter loop — no hashing, plain array indexing, sequential prefetcher-friendly.
+#[cfg(feature = "zlob")]
+fn match_glob_pattern(pattern: &str, paths: &[&str]) -> Vec<bool> {
+    let mut mask = vec![false; paths.len()];
+    let Ok(hits) = zlob::zlob_match_paths_indices(pattern, paths, zlob::ZlobFlags::RECOMMENDED)
+    else {
+        return mask;
+    };
+    for i in hits.to_iter() {
+        if i < mask.len() {
+            mask[i] = true;
+        }
+    }
+    mask
+}
+
+#[cfg(all(not(feature = "zlob"), feature = "ripgrep"))]
+fn match_glob_pattern(pattern: &str, paths: &[&str]) -> Vec<bool> {
+    let mut mask = vec![false; paths.len()];
+    let Ok(glob) = globset::Glob::new(pattern) else {
+        return mask;
+    };
+    let matcher = glob.compile_matcher();
+    if paths.len() >= PAR_THRESHOLD {
+        use rayon::prelude::*;
+        mask.par_iter_mut()
+            .zip(paths.par_iter())
+            .for_each(|(slot, p)| *slot = matcher.is_match(p));
+    } else {
+        for (slot, p) in mask.iter_mut().zip(paths.iter()) {
+            *slot = matcher.is_match(p);
+        }
+    }
+    mask
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[derive(Clone)]
+    struct TestItem {
+        relative_path: &'static str,
+        file_name: &'static str,
+    }
+
+    impl Constrainable for TestItem {
+        fn write_file_name(&self, _arena: ArenaPtr, out: &mut String) {
+            out.clear();
+            out.push_str(self.file_name);
+        }
+
+        fn write_relative_path(&self, _arena: ArenaPtr, out: &mut String) {
+            out.clear();
+            out.push_str(self.relative_path);
+        }
+
+        fn git_status(&self) -> Option<git2::Status> {
+            None
+        }
+
+        fn is_overflow(&self) -> bool {
+            false
+        }
+    }
+
+    #[test]
+    fn test_file_has_extension() {
+        assert!(file_has_extension("file.rs", "rs"));
+        assert!(file_has_extension("file.RS", "rs")); // case-insensitive
+        assert!(file_has_extension("file.test.rs", "rs"));
+        assert!(file_has_extension("a.rs", "rs"));
+
+        assert!(!file_has_extension("file.tsx", "rs"));
+        assert!(!file_has_extension("rs", "rs")); // too short
+        assert!(!file_has_extension(".rs", "rs")); // just extension
+        assert!(!file_has_extension("file.rsx", "rs")); // different extension
+        assert!(!file_has_extension("filers", "rs")); // no dot
+    }
+
+    #[test]
+    fn test_path_contains_segment() {
+        // Segment at start
+        assert!(path_contains_segment("src/lib.rs", "src"));
+        assert!(path_contains_segment("SRC/lib.rs", "src")); // case-insensitive
+
+        // Segment in middle
+        assert!(path_contains_segment("app/src/lib.rs", "src"));
+        assert!(path_contains_segment("app/SRC/lib.rs", "src"));
+
+        // Multiple levels
+        assert!(path_contains_segment("core/workflow/src/main.rs", "src"));
+        assert!(path_contains_segment(
+            "core/workflow/src/main.rs",
+            "workflow"
+        ));
+        assert!(path_contains_segment("core/workflow/src/main.rs", "core"));
+
+        // Should not match partial segments
+        assert!(!path_contains_segment("source/lib.rs", "src"));
+        assert!(!path_contains_segment("mysrc/lib.rs", "src"));
+
+        // Should not match filename
+        assert!(!path_contains_segment("lib/src", "src"));
+
+        // Multi-segment constraints
+        assert!(path_contains_segment(
+            "libswscale/aarch64/input.S",
+            "libswscale/aarch64"
+        ));
+        assert!(path_contains_segment(
+            "foo/libswscale/aarch64/input.S",
+            "libswscale/aarch64"
+        ));
+        assert!(path_contains_segment(
+            "foo/LibSwscale/AArch64/input.S",
+            "libswscale/aarch64"
+        )); // case-insensitive
+        assert!(!path_contains_segment(
+            "xlibswscale/aarch64/input.S",
+            "libswscale/aarch64"
+        )); // partial match at start
+        assert!(!path_contains_segment(
+            "foo/libswscale/aarch64x/input.S",
+            "libswscale/aarch64"
+        )); // partial match at end
+        assert!(path_contains_segment(
+            "crates/fff-core/src/grep.rs",
+            "fff-core/src"
+        ));
+
+        // Edge cases
+        assert!(!path_contains_segment("", "src"));
+        assert!(!path_contains_segment("src", "src")); // no trailing slash
+    }
+
+    #[cfg(windows)]
+    #[test]
+    fn test_path_contains_segment_accepts_backslash() {
+        assert!(path_contains_segment("src\\lib.rs", "src"));
+        assert!(path_contains_segment(
+            "app\\modules\\src\\services\\x.lua",
+            "src"
+        ));
+        assert!(path_contains_segment("app\\SRC\\x.lua", "src"));
+
+        assert!(path_contains_segment(
+            "foo\\libswscale\\aarch64\\input.S",
+            "libswscale/aarch64"
+        ));
+        assert!(path_contains_segment(
+            "crates\\fff-core\\src\\grep.rs",
+            "fff-core/src"
+        ));
+
+        assert!(!path_contains_segment("mysrc\\lib.rs", "src"));
+        assert!(!path_contains_segment(
+            "xlibswscale\\aarch64\\in.S",
+            "libswscale/aarch64"
+        ));
+    }
+
+    #[test]
+    fn test_path_ends_with_suffix() {
+        // Exact match
+        assert!(path_ends_with_suffix(
+            "libswscale/input.c",
+            "libswscale/input.c"
+        ));
+
+        // Suffix match at / boundary
+        assert!(path_ends_with_suffix(
+            "foo/libswscale/input.c",
+            "libswscale/input.c"
+        ));
+
+        // Deep nesting
+        assert!(path_ends_with_suffix(
+            "a/b/c/libswscale/input.c",
+            "libswscale/input.c"
+        ));
+
+        // No boundary — partial directory name
+        assert!(!path_ends_with_suffix(
+            "xlibswscale/input.c",
+            "libswscale/input.c"
+        ));
+
+        // Case insensitive
+        assert!(path_ends_with_suffix(
+            "foo/LibSwscale/Input.C",
+            "libswscale/input.c"
+        ));
+
+        // Single file name
+        assert!(path_ends_with_suffix("input.c", "input.c"));
+        assert!(!path_ends_with_suffix("xinput.c", "input.c"));
+
+        // Suffix longer than path
+        assert!(!path_ends_with_suffix("input.c", "foo/input.c"));
+
+        // Simple path
+        assert!(path_ends_with_suffix("src/main.rs", "src/main.rs"));
+        assert!(path_ends_with_suffix("crates/src/main.rs", "src/main.rs"));
+    }
+
+    #[cfg(windows)]
+    #[test]
+    fn test_path_ends_with_suffix_accepts_backslash() {
+        assert!(path_ends_with_suffix(
+            "app\\modules\\src\\services\\handler.lua",
+            "services/handler.lua"
+        ));
+        assert!(path_ends_with_suffix(
+            "foo\\libswscale\\input.c",
+            "libswscale/input.c"
+        ));
+        assert!(!path_ends_with_suffix(
+            "xlibswscale\\input.c",
+            "libswscale/input.c"
+        ));
+    }
+
+    #[test]
+    fn test_path_ends_with_suffix_does_not_panic_on_unicode_suffix() {
+        assert!(!path_ends_with_suffix("유니코드_파일_테스트.csv", "트.c"));
+        assert!(path_ends_with_suffix(
+            "data/유니코드_파일_테스트.csv",
+            "유니코드_파일_테스트.csv"
+        ));
+    }
+
+    #[test]
+    fn test_path_ends_with_suffix_unicode_apostrophe_mismatch() {
+        assert!(!path_ends_with_suffix(
+            "dir/\u{2019}bar/file.txt",
+            "'bar/file.txt"
+        ));
+    }
+
+    #[test]
+    fn test_path_ends_with_suffix_unicode_space_mismatch() {
+        assert!(!path_ends_with_suffix(
+            "dir/\u{202f}am/file.txt",
+            " am/file.txt"
+        ));
+    }
+
+    #[test]
+    fn test_path_contains_segment_does_not_panic_on_unicode_segment() {
+        assert!(!path_contains_segment("문서/notes.txt", "문x"));
+        assert!(path_contains_segment("프로젝트/문서/notes.txt", "문서"));
+    }
+
+    #[test]
+    fn test_path_contains_segment_unicode_no_panic() {
+        assert!(!path_contains_segment(
+            "Library/Cloud/Project\u{2019}s Folder/books.ttl",
+            "Project's Folder"
+        ));
+    }
+
+    #[test]
+    fn test_file_has_extension_unicode_no_panic() {
+        assert!(!file_has_extension("cat\u{00e9}.rs", "s"));
+    }
+
+    #[test]
+    fn test_file_has_extension_unicode_filename() {
+        assert!(file_has_extension("운영-가이드.md", "md"));
+        assert!(file_has_extension("테스트.csv", "csv"));
+        assert!(!file_has_extension("테스트.csv", "md"));
+    }
+
+    #[test]
+    fn test_apply_constraints_file_path_with_unicode_suffix() {
+        let arena_ptr = ArenaPtr(std::ptr::null());
+
+        let item = TestItem {
+            relative_path: "data/유니코드_파일_테스트.csv",
+            file_name: "유니코드_파일_테스트.csv",
+        };
+
+        let exact = [Constraint::FilePath("유니코드_파일_테스트.csv")];
+        let mismatch = [Constraint::FilePath("트.c")];
+
+        let exact_items = [item.clone()];
+        let exact_matches = apply_constraints(&exact_items, &exact, arena_ptr, arena_ptr)
+            .expect("constraints applied");
+        assert_eq!(exact_matches.len(), 1);
+
+        let mismatch_items = [item];
+        let mismatch_matches = apply_constraints(&mismatch_items, &mismatch, arena_ptr, arena_ptr)
+            .expect("constraints applied");
+        assert!(mismatch_matches.is_empty());
+    }
+
+    #[test]
+    fn test_unicode_path_no_panic_real_korean_cases() {
+        // Real Korean paths that caused panics
+        let path1 = "Downloads/(커리큘럼) hermes agent_정승현님 - 1차 커리큘럼 (강사님 작성).csv";
+        let path2 = "hermes-agent-lecture-materials/세부_커리큘럼_최종.csv";
+        let path3 = "projects/fastcampus-hermes-agent-curriculum/chapters/part-02-Hermes-설치-및-기본-사용/section-02-doctor로-설치-상태-검증/research/03-fix가-자동-수정하는-것과-못하는-것.md";
+
+        // These must not panic regardless of segment/suffix used
+        assert!(!path_contains_segment(path1, "작성"));
+        assert!(!path_ends_with_suffix(path1, "작성.csv"));
+        assert!(!path_contains_segment(path2, "최종"));
+        assert!(!path_ends_with_suffix(path2, "최종.csv"));
+        assert!(!path_contains_segment(path3, "수정"));
+        assert!(!path_ends_with_suffix(path3, "것.md"));
+
+        // Positive cases should still work
+        assert!(path_contains_segment(
+            path2,
+            "hermes-agent-lecture-materials"
+        ));
+        assert!(path_ends_with_suffix(
+            path1,
+            "(커리큘럼) hermes agent_정승현님 - 1차 커리큘럼 (강사님 작성).csv"
+        ));
+        assert!(path_ends_with_suffix(path2, "세부_커리큘럼_최종.csv"));
+    }
+
+    #[test]
+    fn test_negated_glob_excludes_matching_files() {
+        let arena_ptr = ArenaPtr(std::ptr::null());
+
+        let items = vec![
+            TestItem {
+                relative_path: "src/main.rs",
+                file_name: "main.rs",
+            },
+            TestItem {
+                relative_path: "src/lib.ts",
+                file_name: "lib.ts",
+            },
+            TestItem {
+                relative_path: "include/fff.h",
+                file_name: "fff.h",
+            },
+        ];
+
+        // Not(Glob("**/*.rs")) should exclude .rs files
+        let constraints = vec![Constraint::Not(Box::new(Constraint::Glob("**/*.rs")))];
+        let result = apply_constraints(&items, &constraints, arena_ptr, arena_ptr).unwrap();
+        let paths: Vec<&str> = result.iter().map(|i| i.relative_path).collect();
+        assert!(
+            !paths.contains(&"src/main.rs"),
+            "rs file should be excluded"
+        );
+        assert!(paths.contains(&"src/lib.ts"), "ts file should be included");
+        assert!(
+            paths.contains(&"include/fff.h"),
+            "h file should be included"
+        );
+    }
+
+    #[test]
+    fn test_inline_glob_path_matches_prepass() {
+        // Mixed (extensions + glob) takes the inline-compiled path.
+        // Pure glob takes the prepass bitmap path. Both must give identical results.
+        let arena_ptr = ArenaPtr(std::ptr::null());
+        let items = vec![
+            TestItem {
+                relative_path: "src/main.rs",
+                file_name: "main.rs",
+            },
+            TestItem {
+                relative_path: "src/lib.ts",
+                file_name: "lib.ts",
+            },
+            TestItem {
+                relative_path: "tests/foo.rs",
+                file_name: "foo.rs",
+            },
+            TestItem {
+                relative_path: "docs/readme.md",
+                file_name: "readme.md",
+            },
+        ];
+
+        let mixed = vec![Constraint::Extension("rs"), Constraint::Glob("src/**")];
+        let mixed_paths: Vec<&str> = apply_constraints(&items, &mixed, arena_ptr, arena_ptr)
+            .unwrap()
+            .iter()
+            .map(|i| i.relative_path)
+            .collect();
+        assert_eq!(mixed_paths, vec!["src/main.rs"]);
+
+        let pure_glob = vec![Constraint::Glob("src/**")];
+        let glob_paths: Vec<&str> = apply_constraints(&items, &pure_glob, arena_ptr, arena_ptr)
+            .unwrap()
+            .iter()
+            .map(|i| i.relative_path)
+            .collect();
+        assert!(glob_paths.contains(&"src/main.rs"));
+        assert!(glob_paths.contains(&"src/lib.ts"));
+        assert_eq!(glob_paths.len(), 2);
+    }
+
+    #[test]
+    fn test_inline_negated_glob_with_extension() {
+        // Mixed Not(Glob) on inline path — exercise the negate=true branch in
+        // glob_matches_inline through the Not->Glob recursion.
+        let arena_ptr = ArenaPtr(std::ptr::null());
+        let items = vec![
+            TestItem {
+                relative_path: "src/main.rs",
+                file_name: "main.rs",
+            },
+            TestItem {
+                relative_path: "vendor/foo.rs",
+                file_name: "foo.rs",
+            },
+            TestItem {
+                relative_path: "vendor/foo.ts",
+                file_name: "foo.ts",
+            },
+        ];
+
+        let constraints = vec![
+            Constraint::Extension("rs"),
+            Constraint::Not(Box::new(Constraint::Glob("vendor/**"))),
+        ];
+        let paths: Vec<&str> = apply_constraints(&items, &constraints, arena_ptr, arena_ptr)
+            .unwrap()
+            .iter()
+            .map(|i| i.relative_path)
+            .collect();
+        assert_eq!(paths, vec!["src/main.rs"]);
+    }
+}

crates/fff-core/src/index/mod.rs

@@ -0,0 +1,11 @@
+#[doc(hidden)] // for bench
+pub mod bigram_filter;
+pub(crate) use bigram_filter::*;
+
+mod bigram_query;
+pub use bigram_query::*;
+
+mod candidates;
+pub(crate) use candidates::*;
+
+pub mod constraints;

crates/fff-core/src/lib.rs

@@ -1,10 +1,13 @@
 //! # FFF Search — High-performance file finder core
 //!
-//! This crate provides the core search engine for [FFF (Fast File Finder)](https://github.com/dmtrKovalenko/fff.nvim).
+//! This crate provides the core search engine for [FFF (Fast File Finder)](https://github.com/dmtrKovalenko/fff).
 //! It includes filesystem indexing with real-time watching, fuzzy matching powered
 //! by [frizbee](https://docs.rs/neo_frizbee), frecency scoring backed by LMDB,
 //! and multi-mode grep search.
 //!
+//! > [!Important performance information]  
+//! > For the most optimized fff build use `zlob` feature. It requires zig v0.16.0 to be installed on the machine.
+//!
 //! ## Architecture
 //!
 //! - [`file_picker::FilePicker`] — Main entry point. Indexes a directory tree in a
@@ -17,14 +20,16 @@
 //! - [`grep`] — Live grep search supporting regex, plain-text, and fuzzy modes
 //!   with optional constraint filtering.
 //! - [`git`] — Git status caching and repository detection.
+//! - [`watch`] — Client-facing filesystem watch subscriptions: glob, exact
+//!   path, or directory subtree with normalized batch delivery
+//!   (see [`SharedFilePicker::watch`]).
 //!
 //! ## Shared State
 //!
-//! [`SharedPicker`], [`SharedFrecency`], and [`SharedQueryTracker`] are
-//! `Arc<RwLock<Option<T>>>` type aliases for thread-safe shared access. FFF
-//! is designed for long-running processes that keep the file index in global
-//! state, so these wrappers let background threads (scanner, watcher) share
-//! data with the calling code safely.
+//! [`SharedFilePicker`], [`SharedFrecency`], and [`SharedQueryTracker`] are
+//! newtype wrappers around `Arc<RwLock<Option<T>>>` for thread-safe shared
+//! access. They provide `read()` / `write()` methods with built-in error
+//! conversion and convenience helpers like `wait_for_scan()`.
 //!
 //! ## Quick Start
 //!
@@ -33,49 +38,50 @@
 //! use fff_search::frecency::FrecencyTracker;
 //! use fff_search::query_tracker::QueryTracker;
 //! use fff_search::{
-//!     FFFMode, FuzzySearchOptions, PaginationArgs, QueryParser,
-//!     SharedFrecency, SharedPicker, SharedQueryTracker,
+//!     FFFMode, FilePickerOptions, FuzzySearchOptions, PaginationArgs, QueryParser,
+//!     SharedFrecency, SharedFilePicker, SharedQueryTracker,
 //! };
 //!
-//! let shared_picker: SharedPicker = Default::default();
-//! let shared_frecency: SharedFrecency = Default::default();
-//! let shared_query_tracker: SharedQueryTracker = Default::default();
+//! let shared_picker = SharedFilePicker::default();
+//! let shared_frecency = SharedFrecency::default();
+//! let shared_query_tracker = SharedQueryTracker::default();
 //!
 //! let tmp = std::env::temp_dir().join("fff-doctest");
 //! std::fs::create_dir_all(&tmp).unwrap();
 //!
 //! // 1. Optionally initialize frecency and query tracker databases
-//! let frecency = FrecencyTracker::new(tmp.join("frecency"), false)?;
-//! *shared_frecency.write().unwrap() = Some(frecency);
+//! let frecency = FrecencyTracker::open(tmp.join("frecency"))?;
+//! shared_frecency.init(frecency)?;
 //!
-//! let query_tracker = QueryTracker::new(tmp.join("queries"), false)?;
-//! *shared_query_tracker.write().unwrap() = Some(query_tracker);
+//! let query_tracker = QueryTracker::open(tmp.join("queries"))?;
+//! shared_query_tracker.init(query_tracker)?;
 //!
 //! // 2. Init the file picker (spawns background scan + watcher)
 //! FilePicker::new_with_shared_state(
-//!     ".".into(),
-//!     /* warmup memap caches = */ false,
-//!     FFFMode::Ai, // use AI for ai agents, and Neovim for editors
 //!     shared_picker.clone(),
 //!     shared_frecency.clone(),
+//!     FilePickerOptions {
+//!         base_path: ".".into(),
+//!         mode: FFFMode::Ai,
+//!         ..Default::default()
+//!     },
 //! )?;
 //!
-//! // 3. Wait for scan (in real app you would like to add some tokio flavor here)
-//! FilePicker::wait_for_scan(&shared_picker);
+//! // 3. Wait for scan
+//! shared_picker.wait_for_scan(std::time::Duration::from_secs(10));
 //!
 //! // 4. Search: lock the picker and query tracker
-//! let picker_lock_guard = shared_picker.read().unwrap();
-//! let picker = picker_lock_guard.as_ref().unwrap();
-//! let query_tracker_lock_guard = shared_query_tracker.read().unwrap();
+//! let picker_guard = shared_picker.read()?;
+//! let picker = picker_guard.as_ref().unwrap();
+//! let qt_guard = shared_query_tracker.read()?;
 //!
-//! // 5. Parse the query and perform fuzzy search with frecency and combo-boost scoring
+//! // 5. Parse the query and perform fuzzy search
 //! let parser = QueryParser::default();
 //! let query = parser.parse("lib.rs");
 //!
-//! let results = FilePicker::fuzzy_search(
-//!     picker.get_files(),
+//! let results = picker.fuzzy_search(
 //!     &query,
-//!     query_tracker_lock_guard.as_ref(),
+//!     qt_guard.as_ref(),
 //!     FuzzySearchOptions {
 //!         max_threads: 0,
 //!         current_file: None,
@@ -85,70 +91,84 @@
 //! );
 //!
 //! assert!(results.total_matched > 0);
-//! assert!(results.items.first().unwrap().path.ends_with("lib.rs"));
+//! assert!(results.items.first().unwrap().relative_path(picker).ends_with("lib.rs"));
 //!
 //! let _ = std::fs::remove_dir_all(&tmp);
 //! # Ok::<(), Box<dyn std::error::Error>>(())
 //! ```
 
-mod background_watcher;
-mod constraints;
-mod db_healthcheck;
-mod error;
-mod score;
-mod sort_buffer;
+#[cfg(not(any(feature = "ripgrep", feature = "zlob")))]
+compile_error!(
+    "fff-search requires either the `ripgrep` (default) or `zlob` feature. \
+     Enable one, e.g. `--features ripgrep` or `--features zlob`."
+);
 
-/// Core file picker: filesystem indexing, background watching, and fuzzy search.
-///
+/// Primary entry points with thread-safe [`SharedFilePicker`](shared::FilePicker) instance
+pub mod shared;
+pub use shared::*;
+
+/// Core file picker single thread: filesystem indexing, background watching, and fuzzy search.
 /// See [`FilePicker`](file_picker::FilePicker) for the main entry point.
 pub mod file_picker;
+pub use file_picker::*;
 
-/// Frecency (frequency + recency) database for file access scoring.
-///
-/// Backed by LMDB for persistent, crash-safe storage.
-pub mod frecency;
+/// Database-backed persistence: frecency, query history, LMDB plumbing.
+pub mod dbs;
+pub use dbs::*;
 
 /// Git status caching and repository detection utilities.
 pub mod git;
 
 /// Live grep search with regex, plain-text, and fuzzy matching modes.
-///
-/// Supports constraint filtering (file extensions, path segments, globs)
-/// and parallel execution via rayon.
 pub mod grep;
+pub use grep::*;
 
-/// Tracing/logging initialization and panic hook setup.
+/// Tracing/logging initialization
 pub mod log;
 
-/// Path manipulation utilities: cross platform canonicalization, tilde expansion, and
-/// directory distance penalties for search scoring.
+/// Various path utils might be handy for you to work with fff paths
 pub mod path_utils;
 
-/// Search query history tracker for combo-boost scoring.
-///
-/// Records which files a user selects for each query, enabling the scorer
-/// to boost files that were previously chosen for similar searches.
-pub mod query_tracker;
-
 /// Core data types shared across the crate.
 pub mod types;
-
-use std::sync::{Arc, RwLock};
-
-/// Thread-safe shared handle to the [`FilePicker`] instance.
-pub type SharedPicker = Arc<RwLock<Option<FilePicker>>>;
-
-/// Thread-safe shared handle to the [`FrecencyTracker`] instance.
-pub type SharedFrecency = Arc<RwLock<Option<FrecencyTracker>>>;
-
-/// Thread-safe shared handle to the [`QueryTracker`] instance.
-pub type SharedQueryTracker = Arc<RwLock<Option<QueryTracker>>>;
-
-pub use db_healthcheck::{DbHealth, DbHealthChecker};
-pub use error::{Error, Result};
-pub use fff_query_parser::*;
-pub use file_picker::*;
-pub use frecency::*;
-pub use grep::*;
-pub use query_tracker::*;
 pub use types::*;
+
+pub mod constants;
+
+/// Watcher rescan request accounting.
+pub mod rescan_stats;
+pub use rescan_stats::{RESCAN_STATS_ENABLED, RescanReason, RescanStats};
+
+mod rescan_throttle;
+
+// ==================================
+// these are public only for benchmarks, no backward compatibility guaranteed
+#[doc(hidden)]
+pub use index::bigram_filter;
+#[doc(hidden)]
+pub mod simd_string_utils;
+// ==================================
+
+mod error;
+mod git_status_worker;
+mod ignore;
+mod scan;
+mod score;
+mod sort_buffer;
+
+pub(crate) mod index;
+pub(crate) mod parallelism;
+pub(crate) mod simd_path;
+pub(crate) mod stable_vec;
+pub(crate) mod walk;
+
+/// Filesystem watch subscriptions with glob filtering and batched delivery,
+/// plus the background OS watcher.
+#[path = "watcher/mod.rs"]
+pub mod watch;
+pub use watch::{WatchEvent, WatchEventKind, WatchId, WatchOptions};
+
+// fff error
+pub use error::{Error, Result};
+
+pub use fff_query_parser::*;

crates/fff-core/src/log.rs

@@ -1,93 +1,125 @@
-//! Shared logging utilities for FFF crates.
-//!
-//! Provides file-based tracing initialization and a panic hook that writes
-//! to both stderr and a fallback log file.
-
 use std::io;
-use std::path::Path;
+use std::path::{Path, PathBuf};
+use std::sync::OnceLock;
 use tracing_appender::non_blocking;
 use tracing_subscriber::fmt::format::FmtSpan;
 use tracing_subscriber::{EnvFilter, fmt, prelude::*};
 
-static TRACING_INITIALIZED: std::sync::OnceLock<tracing_appender::non_blocking::WorkerGuard> =
-    std::sync::OnceLock::new();
+// Set once on first init_tracing; doubles as the init-once gate.
+static LOG_FILE_PATH: OnceLock<PathBuf> = OnceLock::new();
+static CRASH_HOOKS: OnceLock<()> = OnceLock::new();
 
-static PANIC_HOOK_INSTALLED: std::sync::OnceLock<()> = std::sync::OnceLock::new();
+fn write_crash_report(header: &str, body: &str) {
+    let msg = format!(
+        "\n=== CRASH (this might NOT BE fff related) {} ===\n{}\n=== CRASH END {} ===\n",
+        header, body, header
+    );
+    let _ = std::io::Write::write_all(&mut std::io::stderr(), msg.as_bytes());
+    if let Some(path) = LOG_FILE_PATH.get() {
+        let _ = std::fs::OpenOptions::new()
+            .create(true)
+            .append(true)
+            .open(path)
+            .and_then(|mut f| std::io::Write::write_all(&mut f, msg.as_bytes()));
+    }
+}
 
-/// Install panic hook that writes to both stderr and a fallback file.
-/// This is called separately from init_tracing to ensure panics are always logged.
-pub fn install_panic_hook() {
-    PANIC_HOOK_INSTALLED.get_or_init(|| {
-        let default_panic = std::panic::take_hook();
+// SIGSEGV handler writes a banner to a pre-opened fd (open(2) inside a signal
+// handler is unsafe due to path-resolution allocs). Unix only.
+#[cfg(unix)]
+mod sigsegv {
+    use std::os::fd::IntoRawFd;
+    use std::path::Path;
+    use std::sync::atomic::{AtomicI32, Ordering};
 
-        std::panic::set_hook(Box::new(move |panic_info| {
-            let payload = panic_info.payload();
-            let message = if let Some(s) = payload.downcast_ref::<&str>() {
-                s.to_string()
-            } else if let Some(s) = payload.downcast_ref::<String>() {
-                s.clone()
-            } else {
-                "Unknown panic payload".to_string()
-            };
+    static LOG_FD: AtomicI32 = AtomicI32::new(-1);
 
-            let location = if let Some(location) = panic_info.location() {
-                format!(
-                    "{}:{}:{}",
-                    location.file(),
-                    location.line(),
-                    location.column()
-                )
-            } else {
-                "unknown location".to_string()
-            };
-
-            // Always log to tracing (if initialized)
-            tracing::error!(
-                panic.message = %message,
-                panic.location = %location,
-                "PANIC occurred in FFF"
-            );
-
-            // Always print to stderr
-            eprintln!("=== FFF PANIC ===");
-            eprintln!("Message: {}", message);
-            eprintln!("Location: {}", location);
-            eprintln!("=================");
-
-            // Try to write to fallback panic log file
-            if let Some(cache_dir) = dirs::cache_dir() {
-                let panic_log = cache_dir.join("fff_panic.log");
-                let timestamp = std::time::SystemTime::now()
-                    .duration_since(std::time::UNIX_EPOCH)
-                    .map(|d| d.as_secs())
-                    .unwrap_or(0);
-
-                let panic_entry = format!(
-                    "\n[{}] PANIC at {}\nMessage: {}\n",
-                    timestamp, location, message
-                );
-
-                let _ = std::fs::OpenOptions::new()
-                    .create(true)
-                    .append(true)
-                    .open(&panic_log)
-                    .and_then(|mut f| {
-                        use std::io::Write;
-                        f.write_all(panic_entry.as_bytes())
-                    });
-
-                eprintln!("Panic logged to: {}", panic_log.display());
+    // Must `create(true)` — this runs before init_tracing opens/creates the
+    // writer file, so an append-only open on a non-existent path silently
+    // fails, LOG_FD stays -1, and the SIGSEGV banner never reaches the log.
+    pub fn set_log_fd(path: &Path) {
+        if let Ok(file) = std::fs::OpenOptions::new()
+            .create(true)
+            .append(true)
+            .open(path)
+        {
+            let prev = LOG_FD.swap(file.into_raw_fd(), Ordering::Relaxed);
+            if prev >= 0 {
+                unsafe { libc::close(prev) };
             }
+        }
+    }
 
-            default_panic(panic_info);
-        }));
-    });
+    // Body must be async-signal-safe: write(2), atomic load, signal(2). Nothing else.
+    fn handler(_info: &libc::siginfo_t) {
+        const BANNER: &[u8] = b"\n=== CRASH SIGSEGV (fff) ===\n\
+            fff.nvim's rust extension hit a segfault and is about to die.\n\
+            Please file the bug at https://github.com/dmtrKovalenko/fff/issues with this banner attached.\n\
+            === CRASH END SIGSEGV ===\n";
+        unsafe {
+            libc::write(2, BANNER.as_ptr().cast(), BANNER.len());
+            let log_fd = LOG_FD.load(Ordering::Relaxed);
+            if log_fd >= 0 {
+                libc::write(log_fd, BANNER.as_ptr().cast(), BANNER.len());
+            }
+            // Reset to default so handler return → kernel kills us instead of
+            // re-running the faulting instruction in an infinite loop.
+            libc::signal(libc::SIGSEGV, libc::SIG_DFL);
+        }
+    }
+
+    pub fn install() {
+        // signal-hook-registry chains to LuaJIT's prior handler automatically.
+        unsafe {
+            let _ = signal_hook_registry::register_unchecked(libc::SIGSEGV, handler);
+        }
+    }
+}
+
+#[cfg(not(unix))]
+mod sigsegv {
+    use std::path::Path;
+    pub fn set_log_fd(_path: &Path) {}
+    pub fn install() {}
+}
+
+pub fn install_panic_hook() {
+    CRASH_HOOKS.get_or_init(install_crash_hooks);
+}
+
+fn install_crash_hooks() {
+    let default_panic = std::panic::take_hook();
+    std::panic::set_hook(Box::new(move |panic_info| {
+        let message = if let Some(s) = panic_info.payload().downcast_ref::<&str>() {
+            s.to_string()
+        } else if let Some(s) = panic_info.payload().downcast_ref::<String>() {
+            s.clone()
+        } else {
+            "Unknown panic payload".to_string()
+        };
+
+        let location = panic_info
+            .location()
+            .map(|l| format!("{}:{}:{}", l.file(), l.line(), l.column()))
+            .unwrap_or_else(|| "unknown location".to_string());
+
+        tracing::error!(
+            panic.message = %message,
+            panic.location = %location,
+            "PANIC occurred in FFF"
+        );
+
+        write_crash_report(
+            "RUST PANIC",
+            &format!("Message: {}\nLocation: {}", message, location),
+        );
+        default_panic(panic_info);
+    }));
+
+    sigsegv::install();
 }
 
 /// Parse a log level string into a `tracing::Level`.
-///
-/// Accepts "trace", "debug", "info", "warn", "error" (case-insensitive).
-/// Returns `tracing::Level::INFO` for unrecognised values.
 pub fn parse_log_level(level: Option<&str>) -> tracing::Level {
     match level.as_ref().map(|s| s.trim().to_lowercase()).as_deref() {
         Some("trace") => tracing::Level::TRACE,
@@ -99,66 +131,151 @@ pub fn parse_log_level(level: Option<&str>) -> tracing::Level {
     }
 }
 
-/// Initialize tracing with a single log file.
-///
-/// Creates the parent directory if it doesn't exist, truncates the log file,
-/// and sets up a non-blocking file appender with structured formatting.
-///
-/// # Arguments
-/// * `log_file_path` - Full path to the log file
-/// * `log_level` - Log level (trace, debug, info, warn, error)
-///
-/// # Returns
-/// * `Result<String, io::Error>` - Full path to the log file on success
-pub fn init_tracing(log_file_path: &str, log_level: Option<&str>) -> Result<String, io::Error> {
-    // Install panic hook first (does nothing if already installed)
-    install_panic_hook();
+/// Default retention: how many prior nvim sessions' log files to keep.
+const DEFAULT_RETAIN_RUNS: usize = 20;
 
-    let log_path = Path::new(log_file_path);
-    if let Some(parent) = log_path.parent() {
-        std::fs::create_dir_all(parent)?;
+pub fn generate_trace_id() -> String {
+    use std::sync::atomic::{AtomicU64, Ordering};
+    static TRACE_COUNTER: AtomicU64 = AtomicU64::new(0);
+
+    let nanos = std::time::SystemTime::now()
+        .duration_since(std::time::UNIX_EPOCH)
+        .map(|d| d.as_nanos() as u64)
+        .unwrap_or(0);
+
+    let pid = std::process::id() as u64;
+    let counter = TRACE_COUNTER.fetch_add(1, Ordering::Relaxed);
+
+    // very simple hash functions helps to distinguish trace ids visually
+    let id = nanos ^ (pid.wrapping_mul(0x9E37_79B9_7F4A_7C15)) ^ (counter << 32);
+    format!("{:016x}", id)
+}
+
+pub fn trace_span(trace_id: &str, label: &'static str) -> tracing::Span {
+    tracing::info_span!("fff.trace", trace_id = trace_id, label = label)
+}
+
+fn unix_secs() -> u64 {
+    std::time::SystemTime::now()
+        .duration_since(std::time::UNIX_EPOCH)
+        .map(|d| d.as_secs())
+        .unwrap_or(0)
+}
+
+fn session_path_from_hint(hint: &Path) -> PathBuf {
+    let stem = hint.file_stem().and_then(|s| s.to_str()).unwrap_or("fff");
+    let ext = hint.extension().and_then(|e| e.to_str()).unwrap_or("log");
+    let parent = hint.parent().unwrap_or_else(|| Path::new("."));
+    parent.join(format!(
+        "{stem}+{ts}+{pid}.{ext}",
+        ts = unix_secs(),
+        pid = std::process::id(),
+    ))
+}
+
+fn rotate_logs(dir: &Path, stem: &str, ext: &str, retain_runs: usize) {
+    let Ok(entries) = std::fs::read_dir(dir) else {
+        return;
+    };
+    let prefix = format!("{stem}+");
+    let suffix = format!(".{ext}");
+
+    let mut files: Vec<(std::time::SystemTime, PathBuf)> = entries
+        .filter_map(|res| {
+            let entry = res.ok()?;
+            let name = entry.file_name();
+            let name = name.to_str()?;
+            if !name.starts_with(&prefix) || !name.ends_with(&suffix) {
+                return None;
+            }
+
+            let mtime = entry.metadata().ok()?.modified().ok()?;
+            Some((mtime, entry.path()))
+        })
+        .collect();
+
+    if files.len() <= retain_runs {
+        return;
+    }
+    // Newest first, then drop everything past retain_runs.
+    files.sort_by_key(|(mtime, _)| std::cmp::Reverse(*mtime));
+    for (_, path) in files.into_iter().skip(retain_runs) {
+        let _ = std::fs::remove_file(path);
+    }
+}
+
+/// `log_file_path` is a path-shape hint. Each call writes a unique sibling
+/// `<stem>+<unix-secs>+<pid>.<ext>` so concurrent processes never collide.
+/// Returns the absolute path of the session file.
+pub fn init_tracing(
+    log_file_path: &str,
+    log_level: Option<&str>,
+    retain_runs: Option<usize>,
+) -> Result<String, io::Error> {
+    let hint = Path::new(log_file_path);
+    let session_dir = hint
+        .parent()
+        .unwrap_or_else(|| Path::new("."))
+        .to_path_buf();
+    std::fs::create_dir_all(&session_dir)?;
+
+    let session_path = session_path_from_hint(hint);
+
+    // First init wins; repeat callers no-op and return the original path.
+    if LOG_FILE_PATH.set(session_path.clone()).is_err() {
+        return Ok(LOG_FILE_PATH
+            .get()
+            .map(|p| p.to_string_lossy().into_owned())
+            .unwrap_or_default());
     }
 
-    let file_appender = std::fs::OpenOptions::new()
+    sigsegv::set_log_fd(&session_path);
+    install_panic_hook();
+
+    let stem = hint.file_stem().and_then(|s| s.to_str()).unwrap_or("fff");
+    let ext = hint.extension().and_then(|e| e.to_str()).unwrap_or("log");
+    rotate_logs(
+        &session_dir,
+        stem,
+        ext,
+        retain_runs.unwrap_or(DEFAULT_RETAIN_RUNS),
+    );
+
+    let writer_file = std::fs::OpenOptions::new()
         .create(true)
-        .write(true)
-        .truncate(true) // creates a new file on every setup
-        .open(log_path)?;
+        .append(true)
+        .open(&session_path)?;
 
-    let level = parse_log_level(log_level);
+    // we intinionally leark the guard we don't ever want to stop logging
+    let (non_blocking_appender, guard) = non_blocking(writer_file);
+    Box::leak(Box::new(guard));
 
-    TRACING_INITIALIZED.get_or_init(|| {
-        let (non_blocking_appender, guard) = non_blocking(file_appender);
+    let subscriber = tracing_subscriber::registry()
+        .with(
+            fmt::layer()
+                .with_writer(non_blocking_appender)
+                .with_target(true)
+                .with_thread_ids(false)
+                .with_thread_names(true)
+                .with_ansi(false)
+                .with_span_events(FmtSpan::NEW | FmtSpan::CLOSE),
+        )
+        .with(
+            EnvFilter::builder()
+                .with_default_directive(parse_log_level(log_level).into())
+                .from_env_lossy(),
+        );
 
-        let subscriber = tracing_subscriber::registry()
-            .with(
-                fmt::layer()
-                    .with_writer(non_blocking_appender)
-                    .with_target(true)
-                    .with_thread_ids(false)
-                    .with_thread_names(false)
-                    .with_file(true)
-                    .with_line_number(true)
-                    .with_ansi(false)
-                    .with_span_events(FmtSpan::NEW | FmtSpan::CLOSE),
-            )
-            .with(
-                EnvFilter::builder()
-                    .with_default_directive(level.into())
-                    .from_env_lossy(),
-            );
+    if let Err(e) = tracing::subscriber::set_global_default(subscriber) {
+        eprintln!("Failed to set tracing subscriber: {}", e);
+    } else {
+        tracing::info!(
+            "FFF tracing initialized: {} (pid={}, retain_runs={})",
+            session_path.display(),
+            std::process::id(),
+            retain_runs.unwrap_or(DEFAULT_RETAIN_RUNS),
+        );
+    }
 
-        if let Err(e) = tracing::subscriber::set_global_default(subscriber) {
-            eprintln!("Failed to set tracing subscriber: {}", e);
-        } else {
-            tracing::info!(
-                "FFF tracing initialized with log file: {}",
-                log_path.display()
-            );
-        }
-
-        guard
-    });
-
-    Ok(log_file_path.to_string())
+    Ok(session_path.to_string_lossy().into_owned())
 }

crates/fff-core/src/parallelism.rs

@@ -0,0 +1,84 @@
+//! Dedicated rayon pools. The global pool spans every logical core, which
+//! oversubscribes asymmetric chips (Apple P+E): E-cores are ~2× slower and
+//! `open()` contends on a per-VFS lock past P-core count, so a larger pool is
+//! slower on file-heavy work.
+
+use std::sync::LazyLock;
+
+/// Dedicated thread pool for background work (scan, warmup, bigram build).
+pub static BACKGROUND_THREAD_POOL: LazyLock<rayon::ThreadPool> = LazyLock::new(|| {
+    let total = std::thread::available_parallelism()
+        .map(|p| p.get())
+        .unwrap_or(4);
+
+    // Background work is mostly syscall-bound; halving parallelism leaves
+    // cores for search/UI at negligible throughput cost.
+    let bg_threads = (total / 2).max(2);
+    rayon::ThreadPoolBuilder::new()
+        .num_threads(bg_threads)
+        .thread_name(|i| format!("fff-bg-{i}"))
+        .start_handler(|_| {
+            // QoS pin keeps workers on P-cores; the kernel otherwise drifts
+            // them to ~2× slower E-cores.
+            #[cfg(target_os = "macos")]
+            unsafe {
+                let _ = libc::pthread_set_qos_class_self_np(
+                    libc::qos_class_t::QOS_CLASS_USER_INITIATED,
+                    0,
+                );
+            }
+        })
+        .build()
+        .expect("failed to create background rayon pool")
+});
+
+/// Physical performance-core count via sysctl, falling back to logical cores.
+/// On a 12P+4E M4 Max, grep runs 16t=6.2s vs 13t=4.9s — fewer threads win.
+#[cfg(target_os = "macos")]
+fn performance_core_count() -> usize {
+    let mut count: libc::c_int = 0;
+    let mut size = std::mem::size_of::<libc::c_int>();
+    let name = c"hw.perflevel0.physicalcpu";
+    let ok = unsafe {
+        libc::sysctlbyname(
+            name.as_ptr(),
+            &mut count as *mut _ as *mut libc::c_void,
+            &mut size,
+            std::ptr::null_mut(),
+            0,
+        )
+    };
+    if ok == 0 && count > 0 {
+        count as usize
+    } else {
+        std::thread::available_parallelism()
+            .map(|p| p.get())
+            .unwrap_or(4)
+    }
+}
+
+/// Pool for grep content search: P-core sized and QoS-pinned on macOS, full
+/// parallelism elsewhere. Avoids E-core drag and VFS-lock contention.
+pub static SEARCH_THREAD_POOL: LazyLock<rayon::ThreadPool> = LazyLock::new(|| {
+    #[cfg(target_os = "macos")]
+    let threads = performance_core_count();
+    #[cfg(not(target_os = "macos"))]
+    let threads = std::thread::available_parallelism()
+        .map(|p| p.get())
+        .unwrap_or(4);
+
+    rayon::ThreadPoolBuilder::new()
+        .num_threads(threads)
+        .thread_name(|i| format!("fff-search-{i}"))
+        .start_handler(|_| {
+            #[cfg(target_os = "macos")]
+            unsafe {
+                let _ = libc::pthread_set_qos_class_self_np(
+                    libc::qos_class_t::QOS_CLASS_USER_INITIATED,
+                    0,
+                );
+            }
+        })
+        .build()
+        .expect("failed to create search rayon pool")
+});

crates/fff-core/src/path_utils.rs

@@ -10,6 +10,56 @@ pub fn canonicalize(path: impl AsRef<Path>) -> std::io::Result<PathBuf> {
     std::fs::canonicalize(path)
 }
 
+/// The index stores relative paths with `/` on every platform. These helpers
+/// convert between that canonical form and the OS-native separator, and are
+/// no-ops on non-Windows where `/` is already native.
+
+/// Fold a relative path to the canonical `/` form (no-op off Windows).
+#[cfg(windows)]
+pub fn to_canonical_slashes(rel: &str) -> std::borrow::Cow<'_, str> {
+    if rel.contains('\\') {
+        std::borrow::Cow::Owned(rel.replace('\\', "/"))
+    } else {
+        std::borrow::Cow::Borrowed(rel)
+    }
+}
+
+#[cfg(not(windows))]
+#[inline]
+pub fn to_canonical_slashes(rel: &str) -> std::borrow::Cow<'_, str> {
+    std::borrow::Cow::Borrowed(rel)
+}
+
+/// Rewrite canonical `/` bytes to the OS-native separator in place (no-op off
+/// Windows). Used at OS/state boundaries (absolute-path reconstruction).
+#[cfg(windows)]
+#[inline]
+pub fn nativize_slashes_in_place(bytes: &mut [u8]) {
+    for b in bytes {
+        if *b == b'/' {
+            *b = b'\\';
+        }
+    }
+}
+
+#[cfg(not(windows))]
+#[inline]
+pub fn nativize_slashes_in_place(_bytes: &mut [u8]) {}
+
+/// Git requires a normalized forward-slashed paths on windows
+#[cfg(windows)]
+pub fn normalize(path: PathBuf) -> PathBuf {
+    let as_str = path.to_string_lossy();
+    let with_backslashes: String = as_str.replace('/', "\\");
+    let buf = PathBuf::from(with_backslashes);
+    dunce::canonicalize(&buf).unwrap_or(buf)
+}
+
+#[cfg(not(windows))]
+pub fn normalize(path: PathBuf) -> PathBuf {
+    path
+}
+
 #[cfg(windows)]
 pub fn expand_tilde(path: &str) -> PathBuf {
     return PathBuf::from(path);
@@ -26,53 +76,58 @@ pub fn expand_tilde(path: &str) -> PathBuf {
     PathBuf::from(path)
 }
 
-/// Calculate distance penalty based on directory proximity
-/// Returns a negative penalty score based on how far the candidate is from the current file
-pub fn calculate_distance_penalty(current_file: Option<&str>, candidate_path: &str) -> i32 {
-    let Some(ref current_path) = current_file else {
-        return 0; // No penalty if no current file
+/// Calculate distance penalty based on directory proximity.
+/// Returns a negative penalty score based on how far the candidate is from the current file.
+///
+/// `candidate_dir` is the directory portion of the candidate path (e.g. `"src/components/"`).
+/// It may have a trailing `/` which is stripped internally.
+///
+/// Zero-allocation: walks both directory part iterators in lockstep.
+pub fn calculate_distance_penalty(current_file: Option<&str>, candidate_dir: &str) -> i32 {
+    let Some(current_path) = current_file else {
+        return 0;
     };
 
-    let current_dir = if let Some(parent) = std::path::Path::new(current_path).parent() {
-        parent.to_string_lossy().to_string()
-    } else {
-        String::new()
-    };
+    let current_dir = Path::new(current_path).parent().unwrap_or(Path::new(""));
+    let candidate = Path::new(candidate_dir);
 
-    let candidate_dir = if let Some(parent) = std::path::Path::new(candidate_path).parent() {
-        parent.to_string_lossy().to_string()
-    } else {
-        String::new()
-    };
-
-    if current_dir == candidate_dir {
-        return 0; // Same directory, no penalty
+    if current_dir == candidate {
+        return 0;
     }
 
-    let current_parts: Vec<&str> = current_dir
-        .split(std::path::MAIN_SEPARATOR)
-        .filter(|s| !s.is_empty())
-        .collect();
-    let candidate_parts: Vec<&str> = candidate_dir
-        .split(std::path::MAIN_SEPARATOR)
-        .filter(|s| !s.is_empty())
-        .collect();
+    let mut current_parts = current_dir.components();
+    let mut candidate_parts = candidate.components();
 
-    let common_len = current_parts
-        .iter()
-        .zip(candidate_parts.iter())
-        .take_while(|(a, b)| a == b)
-        .count();
+    let mut common_len = 0usize;
+    let mut current_total = 0usize;
 
-    let current_depth_from_common = current_parts.len() - common_len;
-
-    if current_depth_from_common == 0 {
-        return 0; // Current file is at the common ancestor level
+    loop {
+        match (current_parts.next(), candidate_parts.next()) {
+            (Some(a), Some(b)) => {
+                current_total += 1;
+                if a == b {
+                    common_len += 1;
+                } else {
+                    current_total += current_parts.count();
+                    break;
+                }
+            }
+            (Some(_), None) => {
+                current_total += 1 + current_parts.count();
+                break;
+            }
+            (None, _) => {
+                break;
+            }
+        }
     }
 
-    let penalty = -(current_depth_from_common as i32);
+    let depth_from_common = current_total - common_len;
+    if depth_from_common == 0 {
+        return 0;
+    }
 
-    penalty.max(-20)
+    (-(depth_from_common as i32)).max(-20)
 }
 
 #[cfg(test)]
@@ -82,16 +137,11 @@ mod tests {
     #[test]
     #[cfg(not(target_family = "windows"))]
     fn test_calculate_distance_penalty() {
-        assert_eq!(
-            calculate_distance_penalty(None, "examples/user/test/mod.rs"),
-            0
-        );
+        // candidate_dir is now just the directory portion (with or without trailing /)
+        assert_eq!(calculate_distance_penalty(None, "examples/user/test/"), 0);
         // Same directory
         assert_eq!(
-            calculate_distance_penalty(
-                Some("examples/user/test/main.rs"),
-                "examples/user/test/mod.rs"
-            ),
+            calculate_distance_penalty(Some("examples/user/test/main.rs"), "examples/user/test/"),
             0
         );
         //
@@ -99,7 +149,7 @@ mod tests {
         assert_eq!(
             calculate_distance_penalty(
                 Some("examples/user/test/subdir/file.rs"),
-                "examples/user/test/mod.rs"
+                "examples/user/test/"
             ),
             -1
         );
@@ -108,7 +158,7 @@ mod tests {
         assert_eq!(
             calculate_distance_penalty(
                 Some("examples/user/test/dir1/file.rs"),
-                "examples/user/test/dir2/mod.rs"
+                "examples/user/test/dir2/"
             ),
             -1
         );
@@ -116,7 +166,7 @@ mod tests {
         assert_eq!(
             calculate_distance_penalty(
                 Some("examples/audio-announce/src/lib/audio-announce.rs"),
-                "examples/audio-announce/src/main.rs"
+                "examples/audio-announce/src/"
             ),
             -1
         );
@@ -124,27 +174,27 @@ mod tests {
         assert_eq!(
             calculate_distance_penalty(
                 Some("examples/audio-announce/src/audio-announce.rs"),
-                "examples/pixel/src/main.rs"
+                "examples/pixel/src/"
             ),
             -2
         );
 
-        // Root level files
-        assert_eq!(calculate_distance_penalty(Some("main.rs"), "lib.rs"), 0);
+        // Root level files (empty dir)
+        assert_eq!(calculate_distance_penalty(Some("main.rs"), ""), 0);
     }
 
     #[test]
     #[cfg(target_family = "windows")]
     fn distance_penalty_works_on_windows() {
         assert_eq!(
-            calculate_distance_penalty(None, "examples\\user\\test\\mod.rs"),
+            calculate_distance_penalty(None, "examples\\user\\test\\"),
             0
         );
         // Same directory
         assert_eq!(
             calculate_distance_penalty(
                 Some("examples\\user\\test\\main.rs"),
-                "examples\\user\\test\\mod.rs"
+                "examples\\user\\test\\"
             ),
             0
         );
@@ -153,7 +203,7 @@ mod tests {
         assert_eq!(
             calculate_distance_penalty(
                 Some("examples\\user\\test\\subdir\\file.rs"),
-                "examples\\user\\test\\mod.rs"
+                "examples\\user\\test\\"
             ),
             -1
         );

crates/fff-core/src/rescan_stats.rs

@@ -0,0 +1,215 @@
+#[cfg(rescan_stats)]
+use std::sync::atomic::{AtomicUsize, Ordering};
+
+/// Whether rescan accounting is compiled in.
+pub const RESCAN_STATS_ENABLED: bool = cfg!(rescan_stats);
+
+/// Cause recorded for a filesystem rescan request.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
+pub enum RescanReason {
+    /// Requested through the public API (refresh, directory change).
+    Explicit,
+    /// The kernel dropped events and asked us to re-read the subtree.
+    KernelEventLoss,
+    /// A `.gitignore`/`.ignore` changed, so the cached ignore rules are stale.
+    IgnoreFileChanged,
+    /// A single debounce batch touched more paths than we apply incrementally.
+    EventBatchOverflow,
+    /// The picker refused an incremental insert/update.
+    IndexUpdateRejected,
+    /// The post-scan overflow region ran out of slots.
+    OverflowCapacity,
+}
+
+impl RescanReason {
+    pub const ALL: [RescanReason; 6] = [
+        RescanReason::Explicit,
+        RescanReason::KernelEventLoss,
+        RescanReason::IgnoreFileChanged,
+        RescanReason::EventBatchOverflow,
+        RescanReason::IndexUpdateRejected,
+        RescanReason::OverflowCapacity,
+    ];
+
+    pub const fn as_str(self) -> &'static str {
+        match self {
+            RescanReason::Explicit => "explicit",
+            RescanReason::KernelEventLoss => "kernel_event_loss",
+            RescanReason::IgnoreFileChanged => "ignore_file_changed",
+            RescanReason::EventBatchOverflow => "event_batch_overflow",
+            RescanReason::IndexUpdateRejected => "index_update_rejected",
+            RescanReason::OverflowCapacity => "overflow_capacity",
+        }
+    }
+
+    const fn slot(self) -> usize {
+        match self {
+            RescanReason::Explicit => 0,
+            RescanReason::KernelEventLoss => 1,
+            RescanReason::IgnoreFileChanged => 2,
+            RescanReason::EventBatchOverflow => 3,
+            RescanReason::IndexUpdateRejected => 4,
+            RescanReason::OverflowCapacity => 5,
+        }
+    }
+}
+
+impl std::fmt::Display for RescanReason {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.write_str(self.as_str())
+    }
+}
+
+/// Snapshot of rescan requests grouped by reason.
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
+pub struct RescanStats {
+    pub total: usize,
+    /// Requests suppressed during the cooldown.
+    pub throttled: usize,
+    counts: [usize; RescanReason::ALL.len()],
+    throttled_counts: [usize; RescanReason::ALL.len()],
+}
+
+impl RescanStats {
+    pub fn count(&self, reason: RescanReason) -> usize {
+        self.counts[reason.slot()]
+    }
+
+    pub fn count_throttled(&self, reason: RescanReason) -> usize {
+        self.throttled_counts[reason.slot()]
+    }
+
+    /// Admitted requests originating from watcher fallbacks.
+    pub fn watcher_triggered(&self) -> usize {
+        self.total - self.count(RescanReason::Explicit)
+    }
+
+    /// Per-reason delta against an earlier snapshot.
+    pub fn since(&self, earlier: &RescanStats) -> RescanStats {
+        let mut counts = [0usize; RescanReason::ALL.len()];
+        let mut throttled_counts = [0usize; RescanReason::ALL.len()];
+        for slot in 0..RescanReason::ALL.len() {
+            counts[slot] = self.counts[slot].saturating_sub(earlier.counts[slot]);
+            throttled_counts[slot] =
+                self.throttled_counts[slot].saturating_sub(earlier.throttled_counts[slot]);
+        }
+
+        RescanStats {
+            total: self.total.saturating_sub(earlier.total),
+            throttled: self.throttled.saturating_sub(earlier.throttled),
+            counts,
+            throttled_counts,
+        }
+    }
+}
+
+impl std::fmt::Display for RescanStats {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{} rescan(s)", self.total)?;
+        let mut first = true;
+        for reason in RescanReason::ALL {
+            let count = self.count(reason);
+            if count == 0 {
+                continue;
+            }
+            f.write_str(if first { " [" } else { ", " })?;
+            write!(f, "{reason}={count}")?;
+            first = false;
+        }
+        if !first {
+            f.write_str("]")?;
+        }
+        if self.throttled > 0 {
+            write!(f, ", {} throttled", self.throttled)?;
+        }
+        Ok(())
+    }
+}
+
+#[cfg(rescan_stats)]
+#[derive(Default)]
+pub(crate) struct RescanCounters {
+    counters: [AtomicUsize; RescanReason::ALL.len()],
+    throttled: [AtomicUsize; RescanReason::ALL.len()],
+}
+
+#[cfg(rescan_stats)]
+impl RescanCounters {
+    pub(crate) fn record(&self, reason: RescanReason) {
+        self.counters[reason.slot()].fetch_add(1, Ordering::Relaxed);
+    }
+
+    pub(crate) fn record_throttled(&self, reason: RescanReason) {
+        self.throttled[reason.slot()].fetch_add(1, Ordering::Relaxed);
+    }
+
+    pub(crate) fn snapshot(&self) -> RescanStats {
+        let mut stats = RescanStats::default();
+        for reason in RescanReason::ALL {
+            let count = self.counters[reason.slot()].load(Ordering::Relaxed);
+            stats.counts[reason.slot()] = count;
+            stats.total += count;
+
+            let throttled = self.throttled[reason.slot()].load(Ordering::Relaxed);
+            stats.throttled_counts[reason.slot()] = throttled;
+            stats.throttled += throttled;
+        }
+        stats
+    }
+
+    pub(crate) fn reset(&self) {
+        for counter in self.counters.iter().chain(self.throttled.iter()) {
+            counter.store(0, Ordering::Relaxed);
+        }
+    }
+}
+
+// Release builds retain the API without counter storage.
+#[cfg(not(rescan_stats))]
+#[derive(Default)]
+pub(crate) struct RescanCounters;
+
+#[cfg(not(rescan_stats))]
+impl RescanCounters {
+    pub(crate) fn record(&self, _reason: RescanReason) {}
+
+    pub(crate) fn record_throttled(&self, _reason: RescanReason) {}
+
+    pub(crate) fn snapshot(&self) -> RescanStats {
+        RescanStats::default()
+    }
+
+    pub(crate) fn reset(&self) {}
+}
+
+#[cfg(all(test, rescan_stats))]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn counters_attribute_and_diff_per_reason() {
+        let counters = RescanCounters::default();
+        counters.record(RescanReason::Explicit);
+        let baseline = counters.snapshot();
+
+        counters.record(RescanReason::IgnoreFileChanged);
+        counters.record(RescanReason::IgnoreFileChanged);
+        counters.record(RescanReason::OverflowCapacity);
+
+        let stats = counters.snapshot();
+        assert_eq!(stats.total, 4);
+        assert_eq!(stats.watcher_triggered(), 3);
+
+        let delta = stats.since(&baseline);
+        assert_eq!(delta.total, 3);
+        assert_eq!(delta.count(RescanReason::Explicit), 0);
+        assert_eq!(delta.count(RescanReason::IgnoreFileChanged), 2);
+        assert_eq!(
+            delta.to_string(),
+            "3 rescan(s) [ignore_file_changed=2, overflow_capacity=1]"
+        );
+
+        counters.reset();
+        assert_eq!(counters.snapshot(), RescanStats::default());
+    }
+}

crates/fff-core/src/rescan_throttle.rs

@@ -0,0 +1,124 @@
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::time::Instant;
+
+use crate::constants::{
+    LARGE_INDEX_FILE_COUNT, RESCAN_MIN_INTERVAL, RESCAN_MIN_INTERVAL_LARGE_INDEX,
+};
+
+const NEVER: u64 = u64::MAX;
+
+// Drops watcher rescan requests inside the cooldown after the last scan.
+// A slightly stale index is fine: the next admitted event rescans everything.
+pub(crate) struct RescanThrottle {
+    epoch: Instant,
+    last_admitted: AtomicU64,
+}
+
+impl Default for RescanThrottle {
+    fn default() -> Self {
+        Self {
+            epoch: Instant::now(),
+            last_admitted: AtomicU64::new(NEVER),
+        }
+    }
+}
+
+impl RescanThrottle {
+    /// Returns `true` if a rescan may start now and records it as the last scan
+    pub(crate) fn admit(&self, live_files: usize, has_git_repo: bool) -> bool {
+        let min_interval = if !has_git_repo && live_files >= LARGE_INDEX_FILE_COUNT {
+            RESCAN_MIN_INTERVAL_LARGE_INDEX
+        } else {
+            RESCAN_MIN_INTERVAL
+        };
+
+        let min_ms = min_interval.as_millis() as u64;
+        let now = self.elapsed_ms();
+
+        loop {
+            let last = self.last_admitted.load(Ordering::Acquire);
+            if last != NEVER && now.saturating_sub(last) < min_ms {
+                return false;
+            }
+            // CAS so two concurrent requests cannot both start a walk.
+            if self
+                .last_admitted
+                .compare_exchange(last, now, Ordering::AcqRel, Ordering::Acquire)
+                .is_ok()
+            {
+                return true;
+            }
+        }
+    }
+
+    /// Records an explicit (unthrottled) scan so watcher requests right after
+    /// it are dropped: the index is already fresh.
+    pub(crate) fn note_explicit_scan(&self) {
+        self.last_admitted
+            .store(self.elapsed_ms(), Ordering::Release);
+    }
+
+    fn elapsed_ms(&self) -> u64 {
+        self.epoch.elapsed().as_millis() as u64
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::time::Duration;
+
+    fn throttle_at(ms_ago: u64) -> RescanThrottle {
+        let now = Instant::now();
+        RescanThrottle {
+            epoch: now
+                .checked_sub(Duration::from_millis(ms_ago))
+                .expect("monotonic clock older than the rewind"),
+            last_admitted: AtomicU64::new(0),
+        }
+    }
+
+    #[test]
+    fn first_request_is_always_admitted() {
+        let throttle = RescanThrottle::default();
+        assert!(throttle.admit(100, true));
+    }
+
+    #[test]
+    fn requests_inside_the_cooldown_are_dropped() {
+        let throttle = throttle_at(1_000);
+        assert!(!throttle.admit(100, false));
+        assert!(!throttle.admit(100, false));
+    }
+
+    #[test]
+    fn a_large_index_outside_a_git_repo_uses_the_slower_cadence() {
+        // A minute is past the normal cooldown but not the large-index one.
+        let throttle = throttle_at(60_000);
+        assert!(throttle.admit(100, false));
+
+        let throttle = throttle_at(60_000);
+        assert!(!throttle.admit(LARGE_INDEX_FILE_COUNT, false));
+    }
+
+    #[test]
+    fn a_git_repo_keeps_the_normal_cadence_at_any_size() {
+        let throttle = throttle_at(60_000);
+        assert!(throttle.admit(LARGE_INDEX_FILE_COUNT, true));
+    }
+
+    #[test]
+    fn cooldown_expiry_admits_again() {
+        let throttle = throttle_at(RESCAN_MIN_INTERVAL.as_millis() as u64 + 1);
+        assert!(throttle.admit(100, false));
+        // Admission rearms the cooldown.
+        assert!(!throttle.admit(100, false));
+    }
+
+    #[test]
+    fn explicit_scan_rearms_the_cooldown() {
+        let throttle = RescanThrottle::default();
+        throttle.note_explicit_scan();
+        assert!(!throttle.admit(100, false));
+    }
+}

crates/fff-core/src/scan.rs

@@ -0,0 +1,398 @@
+use std::path::PathBuf;
+use std::sync::Arc;
+use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+
+use tracing::{error, info};
+
+use crate::FileSync;
+use crate::error::Error;
+use crate::file_picker::FFFMode;
+use crate::index::{build_bigram_index, sniff_binary_for_non_indexable};
+use crate::parallelism::BACKGROUND_THREAD_POOL;
+use crate::shared::{SharedFilePicker, SharedFrecency};
+use crate::types::ContentCacheBudget;
+use crate::watch::BackgroundWatcher;
+
+#[derive(Clone, Default)]
+pub(crate) struct ScanSignals {
+    /// Set to `true` while any scan phase is running
+    pub(crate) scanning: Arc<AtomicBool>,
+    /// Set to `true` once the filesystem watcher has been installed
+    pub(crate) watcher_ready: Arc<AtomicBool>,
+    /// Indicates that that owning picker was requested to shut down
+    pub(crate) cancelled: Arc<AtomicBool>,
+    /// Used to resolve conflicts if multiple rescans were triggered in a queue
+    pub(crate) rescan_pending: Arc<AtomicBool>,
+    /// Set by `post_scan_snapshot`, cleared by `PostScanSnapshot::drop`.
+    /// DO NOT set or clear this manually — it is managed exclusively by the
+    /// PostScanSnapshot lifecycle.
+    pub(crate) post_scan_indexing_active: Arc<AtomicBool>,
+}
+
+/// Which optional phases a scan should run.
+#[derive(Clone, Copy, Default, Debug)]
+pub(crate) struct ScanConfig {
+    pub(crate) warmup: bool,
+    pub(crate) content_indexing: bool,
+    pub(crate) watch: bool,
+    pub(crate) auto_cache_budget: bool,
+    pub(crate) install_watcher: bool,
+    pub(crate) follow_symlinks: bool,
+    pub(crate) enable_fs_root_scanning: bool,
+    pub(crate) enable_home_dir_scanning: bool,
+}
+
+/// A fully-configured scan job ready to run on a background thread.
+///
+/// Build with [`ScanJob::from_picker`] (reads all state from the
+/// current `FilePicker`) or [`ScanJob::initial`] (for the bootstrap
+/// scan, before the picker is published to `SharedPicker`).
+pub(crate) struct ScanJob {
+    shared_picker: SharedFilePicker,
+    shared_frecency: SharedFrecency,
+    base_path: PathBuf,
+    mode: FFFMode,
+    signals: ScanSignals,
+    config: ScanConfig,
+    /// Walker-maintained counter backing `get_scan_progress` on the UI
+    /// side. Reset to 0 at scan start, incremented per-file by the
+    /// walker. Shared `Arc` so the UI polls the same atomic.
+    scanned_files_counter: Arc<AtomicUsize>,
+    trace_span: tracing::Span,
+}
+
+impl ScanJob {
+    pub fn new_rescan(
+        shared_picker: &SharedFilePicker,
+        shared_frecency: &SharedFrecency,
+    ) -> Result<Option<Self>, Error> {
+        let guard = shared_picker.read()?;
+        let picker = guard.as_ref().ok_or(Error::FilePickerMissing)?;
+
+        if picker.is_scan_active()
+            || picker
+                .signals
+                .post_scan_indexing_active
+                .load(Ordering::Acquire)
+        {
+            return Ok(None);
+        }
+
+        let mode = picker.mode();
+        let signals = picker.scan_signals();
+        let scanned_files_counter = picker.scanned_files_counter();
+        let base_path = picker.base_path().to_path_buf();
+        let trace_span = picker.trace_span();
+
+        let new_scan_config = ScanConfig {
+            warmup: picker.has_mmap_cache(),
+            content_indexing: picker.has_content_indexing(),
+            watch: picker.has_watcher(),
+            auto_cache_budget: !picker.has_explicit_cache_budget(),
+            install_watcher: false, // the watcher is independent of rescan, it is not restarting EVER
+            follow_symlinks: picker.follows_symlinks(),
+            enable_fs_root_scanning: picker.fs_root_scanning_enabled(),
+            enable_home_dir_scanning: picker.home_dir_scanning_enabled(),
+        };
+
+        drop(guard); // just a sanity check
+
+        Ok(Some(Self {
+            mode,
+            signals,
+            base_path,
+            scanned_files_counter,
+            config: new_scan_config,
+            shared_picker: shared_picker.clone(),
+            shared_frecency: shared_frecency.clone(),
+            trace_span,
+        }))
+    }
+
+    #[allow(clippy::too_many_arguments)]
+    pub fn new_initial(
+        shared_picker: SharedFilePicker,
+        shared_frecency: SharedFrecency,
+        base_path: PathBuf,
+        mode: FFFMode,
+        signals: ScanSignals,
+        scanned_files_counter: Arc<AtomicUsize>,
+        trace_span: tracing::Span,
+        config: ScanConfig,
+    ) -> Self {
+        Self {
+            shared_picker,
+            shared_frecency,
+            base_path,
+            mode,
+            signals,
+            scanned_files_counter,
+            config,
+            trace_span,
+        }
+    }
+
+    /// Run the job on `BACKGROUND_THREAD_POOL`. Returns immediately.
+    ///
+    /// Routed through the pool — and not a fresh `std::thread::spawn` — so the
+    /// orchestrator inherits rayon's QoS pin (USER_INITIATED). Without that
+    /// pin, an interactive nvim's USER_INTERACTIVE main thread spawns a child
+    /// at lower QoS, the walker's Zig worker pool inherits the demotion, and
+    /// the kernel drifts those workers onto E-cores. On chromium that turns a
+    /// ~800 ms walk into ~3 s.
+    pub fn spawn(self) {
+        self.signals.scanning.store(true, Ordering::Release);
+        let span = self.trace_span.clone();
+        BACKGROUND_THREAD_POOL.spawn(move || {
+            let _g = span.enter();
+            self.run();
+        });
+    }
+
+    fn run(self) {
+        let Self {
+            shared_picker,
+            shared_frecency,
+            base_path,
+            mode,
+            signals,
+            scanned_files_counter,
+            config,
+            trace_span: _,
+        } = self;
+
+        let _scanning = ScanningGuard::new(&signals);
+        scanned_files_counter.store(0, Ordering::Relaxed);
+
+        // 1. Walk the file system and collect the list of files
+        let git_workdir = FileSync::discover_git_workdir(&base_path);
+        let sync = match FileSync::walk_filesystem(
+            &base_path,
+            git_workdir.clone(),
+            &scanned_files_counter,
+            &shared_frecency,
+            mode,
+            config.follow_symlinks,
+        ) {
+            Ok(sync) => sync,
+            Err(e) => {
+                error!(?e, "scan walk failed");
+                return;
+            }
+        };
+
+        // 2. Populate the file list
+        let git_status_worker;
+        if let Ok(mut guard) = shared_picker.write()
+            && let Some(picker) = guard.as_mut()
+        {
+            if signals.cancelled.load(Ordering::Acquire) {
+                info!("scan cancelled between walk and commit, discarding");
+                return;
+            }
+
+            let live_count = sync.live_count;
+            picker.commit_new_sync(sync);
+            git_status_worker = Arc::clone(&picker.git_status_worker);
+
+            if config.auto_cache_budget && !picker.has_explicit_cache_budget() {
+                picker.set_cache_budget(ContentCacheBudget::new_for_repo(live_count));
+            }
+        } else {
+            error!("failed to install scan results into picker");
+            return;
+        }
+
+        // Spawn the git status worker once. BUG PINNNING. If the user initiated git in the folder
+        // which is a real use case we need to have a way to start the git worker background thread dynamically
+        if git_workdir.is_some() && !signals.cancelled.load(Ordering::Acquire) {
+            git_status_worker.spawn_once(shared_picker.weaken(), shared_frecency.clone());
+            git_status_worker.request_full_rescan(); // this runs anyway
+        }
+
+        // BUG pinning: take the snapshot *before* the storing the scan=true, otherwise there is a tiny
+        // race window when there scanned is set to true, but `post_scan_indexing_active` flag is `false`
+        let snapshot = if !signals.cancelled.load(Ordering::Acquire) {
+            shared_picker.read().ok().and_then(|guard| {
+                guard
+                    .as_ref()
+                    .and_then(|picker| unsafe { picker.post_scan_snapshot() })
+            })
+        } else {
+            None
+        };
+
+        signals.scanning.store(false, Ordering::Relaxed); // file are searchable
+
+        // in case we do a rescan, we have to resubscribe a watcher to the new set of directories
+        // all the already watched directories are not going to be resubscribed (this is internally deduped)
+        if !config.install_watcher && !signals.cancelled.load(Ordering::Acquire) {
+            rescubscribe_watcher_post_scan(&shared_picker);
+        }
+
+        // 3. Runs post scna in parallel with git status collection
+        if !signals.cancelled.load(Ordering::Acquire)
+            && let Some(snap) = snapshot.as_ref()
+        {
+            Self::run_post_scan(&shared_picker, &signals, &config, snap);
+        }
+
+        drop(snapshot); // SNAPSHOT SHOULD NOT BE USED AFTER THIS POINT
+
+        // 5. Install filesystem watcher (initial scan only).
+        if config.install_watcher && config.watch && !signals.cancelled.load(Ordering::Acquire) {
+            let shared_picker: &SharedFilePicker = &shared_picker;
+            let shared_frecency: &SharedFrecency = &shared_frecency;
+            let base_path: &std::path::Path = &base_path;
+
+            match BackgroundWatcher::new(
+                base_path.to_path_buf(),
+                git_workdir,
+                shared_picker.clone(),
+                shared_frecency.clone(),
+                mode,
+                config.enable_fs_root_scanning,
+                config.enable_home_dir_scanning,
+                git_status_worker,
+                tracing::Span::current(),
+            ) {
+                Ok(watcher) => {
+                    if let Ok(mut guard) = shared_picker.write()
+                        && let Some(picker) = guard.as_mut()
+                        && picker.base_path() == base_path
+                        && !signals.cancelled.load(Ordering::Acquire)
+                    {
+                        picker.background_watcher = Some(watcher);
+                        signals.watcher_ready.store(true, Ordering::Release);
+                    }
+                }
+                Err(e) => error!(?e, "failed to initialize background watcher"),
+            };
+        }
+
+        // 6. Drain any rescan that arrived while we were busy.
+        // if user initiated a new rescan we had no way to cancel current post scan, so do it again
+        if !signals.cancelled.load(Ordering::Acquire)
+            && signals.rescan_pending.swap(false, Ordering::AcqRel)
+        {
+            match Self::new_rescan(&shared_picker, &shared_frecency) {
+                Ok(Some(follow_up)) => {
+                    info!("Rescheduling deferred rescan after current scan finished");
+                    follow_up.spawn();
+                }
+                Ok(None) => {
+                    // this should be practically impossible because we do not have any
+                    // queue, but if somehow a new rescan was triggered JUST IN THIS MOMENT
+                    // just ignore it because the ongoing one is fresh enough
+                    tracing::warn!("Post scan was re-triggered, ignoring");
+                }
+                Err(e) => {
+                    error!(?e, "Failed to reschedule deferred rescan");
+                }
+            }
+        }
+    }
+
+    /// THIS IS VERY VERY IMPORTANT THAT ANYTHING INSIDE THIS FUNCTION TO NOT READ ANYTHING CLEARABLE OUTSIDE
+    /// this is a very silly off lock implementation that actually matters, and that's why it is crafted
+    /// to never read anything from the picker, it can only WRITE information using single instructions
+    ///
+    /// Things that are safe and immutable - file list, indexes of files, paths, and signals.
+    #[tracing::instrument(skip_all, fields(warmup = ?config.warmup, indexing = ?config.content_indexing))]
+    fn run_post_scan(
+        shared_picker: &SharedFilePicker,
+        signals: &ScanSignals,
+        config: &ScanConfig,
+        unsafe_snapshot: &crate::file_picker::PostScanUnsafeSnapshot,
+    ) {
+        let Some(arena) = unsafe_snapshot
+            .arena // we are never touching overlays so this arena is always correct
+            .as_ref()
+            .map(|s| s.as_arena_ptr())
+        else {
+            tracing::error!("Failed to run post scan: arena is invalid");
+            return;
+        };
+
+        let files: &[crate::types::FileItem] = &unsafe_snapshot.files[..unsafe_snapshot.base_count];
+        if signals.cancelled.load(Ordering::Acquire) {
+            return;
+        }
+
+        if config.content_indexing {
+            let indexable_count = unsafe_snapshot.indexable_count.min(files.len());
+            let (indexable_files, non_indexable_files) = files.split_at(indexable_count);
+            let index = build_bigram_index(indexable_files, &unsafe_snapshot.base_path, arena);
+
+            if let Ok(mut guard) = shared_picker.write()
+                && let Some(picker) = guard.as_mut()
+            {
+                picker.set_bigram_index(index);
+            }
+
+            // Bigram only sniffs files <= MAX_INDEXABLE_FILE_SIZE; large
+            // unknown-extension binaries slip past it and would otherwise be
+            // grep-able as text. Cheap header sniff catches those.
+            if !signals.cancelled.load(Ordering::Acquire) {
+                sniff_binary_for_non_indexable(
+                    non_indexable_files,
+                    &unsafe_snapshot.base_path,
+                    arena,
+                    &signals.cancelled,
+                );
+            }
+        } else {
+            // this potentially a long running as we are not parallelizing it but it's okay
+            sniff_binary_for_non_indexable(
+                files,
+                &unsafe_snapshot.base_path,
+                arena,
+                &signals.cancelled,
+            );
+        }
+
+        // TODO Skipped as potentially unsafe - figure this out later
+        // if config.warmup && !signals.cancelled.load(Ordering::Acquire) {
+        //     warmup_mmaps(files, budget, &unsafe_snapshot.base_path, arena);
+        // }
+    }
+}
+
+// Ensures early returns clear the scanning signal.
+struct ScanningGuard<'a> {
+    signals: &'a ScanSignals,
+}
+
+impl<'a> ScanningGuard<'a> {
+    fn new(signals: &'a ScanSignals) -> Self {
+        signals.scanning.store(true, Ordering::Relaxed);
+        Self { signals }
+    }
+}
+
+impl Drop for ScanningGuard<'_> {
+    fn drop(&mut self) {
+        self.signals.scanning.store(false, Ordering::Relaxed);
+    }
+}
+
+/// If the scan encounters new directories created we have to add them to the watch list
+/// this is fine because the watcher does deduplicate the entries and doesn't add a lot of
+/// garbage notify watchers / fs events streams
+#[tracing::instrument(skip_all)]
+fn rescubscribe_watcher_post_scan(shared_picker: &SharedFilePicker) {
+    let Ok(guard) = shared_picker.read() else {
+        return;
+    };
+    let Some(picker) = guard.as_ref() else {
+        return;
+    };
+    let Some(watcher) = picker.background_watcher.as_ref() else {
+        return;
+    };
+
+    picker.for_each_dir(|dir: &std::path::Path| {
+        watcher.request_watch_dir(dir.to_path_buf());
+        std::ops::ControlFlow::Continue(())
+    });
+}

crates/fff-core/src/shared.rs

@@ -0,0 +1,531 @@
+use std::path::{Path, PathBuf};
+use std::sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard, Weak};
+use std::time::{Duration, Instant};
+
+use crate::dbs::lmdb::{LmdbStore, spawn_lmdb_gc};
+use crate::error::Error;
+use crate::file_picker::FilePicker;
+use crate::frecency::FrecencyTracker;
+use crate::git::GitStatusCache;
+use crate::query_tracker::QueryTracker;
+use crate::rescan_stats::{RescanCounters, RescanReason, RescanStats};
+use crate::rescan_throttle::RescanThrottle;
+use crate::scan::ScanJob;
+use crate::watch::{WatchEvent, WatchId, WatchOptions, WatchRegistry};
+use git2::Repository;
+
+/// Poll `.git/index.lock` until it disappears (git write completed), giving up
+/// after [`GIT_LOCK_MAX_WAIT`]. Used by [`SharedPicker::refresh_git_status`]
+/// to avoid reading a half-updated index when the watcher fires mid-`git add`.
+///
+/// The wait is bounded and cheap: the lock file is typically cleared within
+/// a few milliseconds of the git command exiting.
+fn wait_for_git_index_lock_release(git_root: &Path) {
+    const GIT_LOCK_POLL: Duration = Duration::from_millis(10);
+    const GIT_LOCK_MAX_WAIT: Duration = Duration::from_millis(500);
+
+    let lock = git_root.join(".git").join("index.lock");
+    // Fast path: no lock present.
+    if !lock.exists() {
+        return;
+    }
+    let deadline = Instant::now() + GIT_LOCK_MAX_WAIT;
+    while lock.exists() && Instant::now() < deadline {
+        std::thread::sleep(GIT_LOCK_POLL);
+    }
+    if lock.exists() {
+        tracing::warn!(
+            "Proceeding with git status refresh despite lingering \
+             .git/index.lock at {} — will retry once it clears",
+            lock.display()
+        );
+    }
+}
+
+/// Poll `done` every 10ms until it returns `true`, or until `timeout` elapses.
+/// Returns `true` if the condition was met, `false` on timeout.
+fn poll_until(timeout: Duration, mut done: impl FnMut() -> bool) -> bool {
+    let start = Instant::now();
+    while !done() {
+        if start.elapsed() >= timeout {
+            return false;
+        }
+        std::thread::sleep(Duration::from_millis(10));
+    }
+    true
+}
+
+/// Thread-safe shared handle to the [`FilePicker`] instance.
+/// This accumulates only asynchronous non-blocking operations against the
+/// file picker: creating, triggering various rescans and so on.
+///
+/// For blocking access use internal picker via `.read()` or `.write()`
+///
+/// ```ignore
+/// let shared_picker = SharedFilePicker::default();
+///
+/// if let Some(picker) = shared_picker.read()?.as_ref() {
+///     let files = picker.fuzzy_search(&query, options);
+///     println!("Found {} files", files.len());
+/// } else {
+///     println!("Picker not initialized");
+/// }
+/// ```
+#[derive(Clone, Default)]
+pub struct SharedFilePicker(pub(crate) Arc<SharedPickerInner>);
+
+pub struct SharedPickerInner {
+    picker: parking_lot::RwLock<Option<FilePicker>>,
+    /// Watch subscriptions live outside the picker lock so delivery and
+    /// (un)subscribing never contend with searches.
+    watchers: Arc<WatchRegistry>,
+    rescans: RescanCounters,
+    rescan_throttle: RescanThrottle,
+}
+
+impl Default for SharedPickerInner {
+    fn default() -> Self {
+        Self {
+            picker: parking_lot::RwLock::new(None),
+            watchers: Arc::new(WatchRegistry::default()),
+            rescans: RescanCounters::default(),
+            rescan_throttle: RescanThrottle::default(),
+        }
+    }
+}
+
+/// Non-owning handle to a [`SharedPicker`].
+#[derive(Clone)]
+pub(crate) struct WeakFilePicker(Weak<SharedPickerInner>);
+
+impl WeakFilePicker {
+    /// Try to promote the weak handle back to a strong [`SharedPicker`].
+    ///
+    /// Returns `None` once every strong `SharedPicker` clone has been
+    /// dropped. Callers should treat that as "the picker is being
+    /// torn down" and exit their current iteration cleanly.
+    pub(crate) fn upgrade(&self) -> Option<SharedFilePicker> {
+        self.0.upgrade().map(SharedFilePicker)
+    }
+}
+
+impl std::fmt::Debug for SharedFilePicker {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_tuple("SharedPicker").field(&"..").finish()
+    }
+}
+
+impl SharedFilePicker {
+    pub fn read(&self) -> Result<parking_lot::RwLockReadGuard<'_, Option<FilePicker>>, Error> {
+        Ok(self.0.picker.read())
+    }
+
+    pub fn write(&self) -> Result<parking_lot::RwLockWriteGuard<'_, Option<FilePicker>>, Error> {
+        Ok(self.0.picker.write())
+    }
+
+    /// Signal the background scan to cancel. Non-blocking: post-scan
+    /// threads check this flag and bail out at their next cancellation point.
+    pub fn cancel(&self) {
+        if let Ok(guard) = self.read()
+            && let Some(picker) = guard.as_ref()
+        {
+            picker.cancel();
+        }
+    }
+
+    /// Produce a non-owning handle to the same inner picker.
+    /// Use it if you don't need to block internal threads from dropping while owning this ref
+    pub(crate) fn weaken(&self) -> WeakFilePicker {
+        WeakFilePicker(Arc::downgrade(&self.0))
+    }
+
+    /// Return `true` if this is an instance of the picker that requires a complicated post-scan
+    /// indexing/cache warmup job. The indexing is not crazy but it takes time.
+    pub fn need_complex_rebuild(&self) -> bool {
+        let guard = self.0.picker.read();
+        guard
+            .as_ref()
+            .is_some_and(|p| p.has_mmap_cache() || p.has_content_indexing())
+    }
+
+    /// Block until the background filesystem scan finishes.
+    /// Returns `true` if scan completed, `false` on timeout.
+    pub fn wait_for_scan(&self, timeout: Duration) -> bool {
+        let signal = {
+            let guard = self.0.picker.read();
+            match &*guard {
+                Some(picker) => Arc::clone(&picker.signals.scanning),
+                None => return true,
+            }
+        };
+
+        poll_until(timeout, || {
+            !signal.load(std::sync::atomic::Ordering::Acquire)
+        })
+    }
+
+    /// Block until the background file watcher is ready.
+    /// Returns `true` if watcher ready, `false` on timeout.
+    pub fn wait_for_watcher(&self, timeout: Duration) -> bool {
+        let watch_ready_signal = {
+            let guard = self.0.picker.read();
+            match &*guard {
+                Some(picker) => Arc::clone(&picker.signals.watcher_ready),
+                None => return true,
+            }
+        };
+
+        poll_until(timeout, || {
+            watch_ready_signal.load(std::sync::atomic::Ordering::Acquire)
+        })
+    }
+
+    /// Blocks until both the filesystem walk and post-scan indexing are done.
+    /// Returns true once scanning=false AND post_scan_indexing_active=false.
+    pub fn wait_for_indexing_complete(&self, timeout: Duration) -> bool {
+        let (scanning, post_scan_active) = {
+            let guard = self.0.picker.read();
+            match &*guard {
+                Some(picker) => (
+                    Arc::clone(&picker.signals.scanning),
+                    Arc::clone(&picker.signals.post_scan_indexing_active),
+                ),
+                None => return true,
+            }
+        };
+
+        poll_until(timeout, || {
+            !scanning.load(std::sync::atomic::Ordering::Acquire)
+                && !post_scan_active.load(std::sync::atomic::Ordering::Acquire)
+        })
+    }
+
+    /// Trigger a full filesystem rescan without blocking the caller.
+    /// Performs a safe async rescan. Guarantees only single active rescan per picker.
+    /// If many rescans requested the last one guaranteed to be finished.
+    pub fn trigger_full_rescan_async(&self, shared_frecency: &SharedFrecency) -> Result<(), Error> {
+        self.trigger_full_rescan_with_reason(shared_frecency, RescanReason::Explicit)
+            .map(|_| ())
+    }
+
+    /// Returns admitted and throttled rescan requests by reason.
+    /// Counters start at picker creation or the last reset.
+    pub fn rescan_stats(&self) -> RescanStats {
+        self.0.rescans.snapshot()
+    }
+
+    pub fn reset_rescan_stats(&self) {
+        self.0.rescans.reset();
+    }
+
+    /// Returns `Ok(true)` when a rescan was started (or queued behind an
+    /// active scan) and `Ok(false)` when the request was throttled — the
+    /// caller must then fall back to incremental event processing.
+    pub(crate) fn trigger_full_rescan_with_reason(
+        &self,
+        shared_frecency: &SharedFrecency,
+        reason: RescanReason,
+    ) -> Result<bool, Error> {
+        // for giant folders we have no other choice other than throttling rescans
+        // if user is running application in millions of files with a ton of rescan events
+        // we drop / throttle some of requests to avoid constant burst of IO
+        if reason == RescanReason::Explicit {
+            self.0.rescan_throttle.note_explicit_scan();
+        } else if !self.check_rescan_throttle(reason) {
+            return Ok(false);
+        }
+
+        self.0.rescans.record(reason);
+
+        match ScanJob::new_rescan(self, shared_frecency)? {
+            Some(job) => {
+                job.spawn();
+            }
+            None => {
+                // we can not abort the ongoing sync, but if the events
+                if let Ok(guard) = self.read()
+                    && let Some(picker) = guard.as_ref()
+                {
+                    picker
+                        .scan_signals()
+                        .rescan_pending
+                        .store(true, std::sync::atomic::Ordering::Release);
+                    tracing::info!(
+                        "Full rescan requested while another scan is active — \
+                         deferred via rescan_pending flag"
+                    );
+                }
+            }
+        }
+        Ok(true)
+    }
+
+    fn check_rescan_throttle(&self, reason: RescanReason) -> bool {
+        let (live_files, has_git) = self
+            .read()
+            .ok()
+            .and_then(|guard| {
+                guard
+                    .as_ref()
+                    .map(|picker| (picker.live_file_count(), picker.has_git_repo()))
+            })
+            .unwrap_or((0, false));
+
+        if self.0.rescan_throttle.admit(live_files, has_git) {
+            return true;
+        }
+
+        self.0.rescans.record_throttled(reason);
+        tracing::debug!(%reason, live_files, "Rescan throttled, skipping");
+        false
+    }
+
+    /// Subscribe to filesystem changes matching `pattern`.
+    ///
+    /// Patterns may be base-relative globs (./ works), exact paths inside the indexed
+    /// tree, or existing directories. An empty pattern watches the whole tree.
+    ///
+    /// Events are debounced and submitted in batches per 100-ms window at most 128 events.
+    /// Gitignored and other ignored files are never triggering watcher.
+    pub fn watch(
+        &self,
+        pattern: &str,
+        options: WatchOptions,
+        callback: impl Fn(WatchId, &[WatchEvent]) + Send + Sync + 'static,
+    ) -> Result<WatchId, Error> {
+        let (base_path, has_watcher, watcher_ready) = {
+            let guard = self.read()?;
+            let picker = guard.as_ref().ok_or(Error::FilePickerMissing)?;
+
+            (
+                picker.base_path().to_path_buf(),
+                picker.has_watcher(),
+                picker.is_watcher_ready(),
+            )
+        };
+
+        if !has_watcher {
+            return Err(Error::WatcherDisabled);
+        }
+        if !watcher_ready {
+            return Err(Error::WatcherNotReady);
+        }
+
+        self.0
+            .watchers
+            .subscribe(&base_path, pattern, options, Box::new(callback))
+    }
+
+    /// Remove a watch subscription. Returns `true` if the id was active.
+    pub fn unwatch(&self, id: WatchId) -> bool {
+        self.0.watchers.unsubscribe(id)
+    }
+
+    /// Return whether a watch subscription is active.
+    pub fn is_watch_active(&self, id: WatchId) -> bool {
+        self.0.watchers.contains(id)
+    }
+
+    /// Remove every subscription without waiting for an executing callback.
+    pub fn shutdown_watches(&self) {
+        self.0.watchers.shutdown();
+    }
+
+    /// Remove every subscription and wait for an executing callback.
+    /// When called by that callback, it does not wait on itself.
+    pub fn shutdown_watches_and_wait(&self) {
+        self.0.watchers.shutdown_and_wait();
+    }
+
+    pub(crate) fn rebase_watches(&self, base_path: &Path) {
+        self.0.watchers.rebase(base_path);
+    }
+
+    pub(crate) fn watch_registry(&self) -> &Arc<WatchRegistry> {
+        &self.0.watchers
+    }
+
+    /// Refresh git statuses for all indexed files
+    #[tracing::instrument(level = "info", skip_all)]
+    pub fn refresh_git_status(&self, shared_frecency: &SharedFrecency) -> Result<usize, Error> {
+        use tracing::debug;
+
+        let git_status = {
+            let git_root = {
+                let guard = self.read()?;
+                let Some(ref picker) = *guard else {
+                    return Err(Error::FilePickerMissing);
+                };
+                picker.git_root().map(|p| p.to_path_buf())
+            };
+
+            debug!(?git_root, "Refreshing git status for picker");
+
+            if let Some(ref root) = git_root {
+                wait_for_git_index_lock_release(root);
+            }
+
+            GitStatusCache::read_git_status(
+                git_root.as_deref(),
+                &mut crate::git::default_status_options(),
+            )
+        };
+
+        let mut guard = self.write()?;
+        let picker = guard.as_mut().ok_or(Error::FilePickerMissing)?;
+
+        let statuses_count = if let Some(git_status) = git_status {
+            let count = git_status.statuses_len();
+            picker.update_git_statuses(git_status, shared_frecency)?;
+            count
+        } else {
+            0
+        };
+
+        Ok(statuses_count)
+    }
+
+    /// Recompute and apply git status for a specific set of paths.
+    pub fn update_git_status_for_paths(
+        &self,
+        paths: &[PathBuf],
+        shared_frecency: &SharedFrecency,
+    ) -> Result<(), Error> {
+        if paths.is_empty() {
+            return Ok(());
+        }
+
+        let git_root = {
+            let guard = self.read()?;
+            let Some(ref picker) = *guard else {
+                return Err(Error::FilePickerMissing);
+            };
+            picker.git_root().map(|p| p.to_path_buf())
+        };
+        let Some(git_root) = git_root else {
+            return Ok(());
+        };
+
+        wait_for_git_index_lock_release(&git_root);
+
+        let repo = Repository::open(&git_root)?;
+        let status = GitStatusCache::git_status_for_paths(&repo, paths)?;
+
+        let mut guard = self.write()?;
+        let picker = guard.as_mut().ok_or(Error::FilePickerMissing)?;
+        picker.update_git_statuses(status, shared_frecency)
+    }
+}
+
+/// Thread-safe shared handle to an LMDB-backed store. A disabled (`noop`)
+/// instance silently ignores writes. See the [`SharedFrecency`] and
+/// [`SharedQueryTracker`] aliases.
+///
+/// `LmdbStore` is intentionally crate-private, so the store type is sealed:
+/// only `FrecencyTracker` / `QueryTracker` can ever instantiate this.
+#[allow(private_bounds)]
+pub struct SharedDb<T: LmdbStore> {
+    inner: Arc<RwLock<Option<T>>>,
+    enabled: bool,
+}
+
+// Hand-written to avoid a spurious `T: Clone` bound — `Arc` is always `Clone`.
+impl<T: LmdbStore> Clone for SharedDb<T> {
+    fn clone(&self) -> Self {
+        Self {
+            inner: self.inner.clone(),
+            enabled: self.enabled,
+        }
+    }
+}
+
+impl<T: LmdbStore> Default for SharedDb<T> {
+    fn default() -> Self {
+        Self {
+            inner: Arc::new(RwLock::new(None)),
+            enabled: true,
+        }
+    }
+}
+
+impl<T: LmdbStore> std::fmt::Debug for SharedDb<T> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_tuple("SharedDb").field(&T::LABEL).finish()
+    }
+}
+
+#[allow(private_bounds)]
+impl<T: LmdbStore> SharedDb<T> {
+    /// Creates a disabled instance that silently ignores all writes.
+    pub fn noop() -> Self {
+        Self {
+            inner: Arc::new(RwLock::new(None)),
+            enabled: false,
+        }
+    }
+
+    pub fn read(&self) -> Result<RwLockReadGuard<'_, Option<T>>, Error> {
+        self.inner.read().map_err(|_| Error::AcquireFrecencyLock)
+    }
+
+    pub fn write(&self) -> Result<RwLockWriteGuard<'_, Option<T>>, Error> {
+        self.inner.write().map_err(|_| Error::AcquireFrecencyLock)
+    }
+
+    /// Initialize the store + spawn GC in the background. No-op when disabled.
+    pub fn init(&self, tracker: T) -> Result<(), Error> {
+        if !self.enabled {
+            return Ok(());
+        }
+
+        {
+            let mut guard = self.write()?;
+            *guard = Some(tracker);
+        }
+
+        // GC holds a read guard on this lock, so destroy / re-init wait won't race
+        spawn_lmdb_gc(self.inner.clone());
+        Ok(())
+    }
+
+    /// Drop the in-memory tracker and delete the on-disk database directory.
+    ///
+    /// Returns `Ok(Some(path))` with the deleted path, or `Ok(None)` if no tracker was initialized.
+    pub fn destroy(&self) -> Result<Option<PathBuf>, Error> {
+        let mut guard = self.write()?;
+        let Some(tracker) = guard.take() else {
+            return Ok(None);
+        };
+
+        let closing_event = match tracker.shared_env().destroy() {
+            Ok(closing) => closing,
+            Err(e) => {
+                *guard = Some(tracker);
+                return Err(e);
+            }
+        };
+
+        let db_path = tracker.env().path().to_path_buf();
+        // Drop closes the LMDB env and unmaps the files
+        drop(tracker);
+        drop(guard);
+
+        // Deleting before mdb_env_close finishes would race the unmap.
+        if let Some(event) = closing_event {
+            event.wait_timeout(Duration::from_secs(5));
+        }
+
+        std::fs::remove_dir_all(&db_path).map_err(|source| Error::RemoveDbDir {
+            path: db_path.clone(),
+            source,
+        })?;
+        Ok(Some(db_path))
+    }
+}
+
+/// Thread-safe shared handle to the [`FrecencyTracker`] instance.
+pub type SharedFrecency = SharedDb<FrecencyTracker>;
+
+/// Thread-safe shared handle to the [`QueryTracker`] instance.
+pub type SharedQueryTracker = SharedDb<QueryTracker>;

crates/fff-core/src/simd_path.rs

@@ -0,0 +1,624 @@
+use ahash::AHashMap;
+use smallvec::SmallVec;
+use std::borrow::Cow;
+
+/// SIMD chunk size in bytes (matches NEON/SSE2 register width).
+/// This must stay in sync with neo_frizbee's internal chunk size.
+pub(crate) const SIMD_CHUNK_BYTES: usize = 16;
+
+/// 4 chunks = 64 bytes inline, covers ~85% of paths without heap fallback.
+const INLINE_CHUNKS: usize = 4;
+
+pub(crate) type ChunkIndices = SmallVec<[u32; INLINE_CHUNKS]>;
+
+#[derive(Clone, Copy)]
+pub struct ArenaPtr(pub(crate) *const u8);
+
+// SAFETY: The arena is a read-only immutable part of file sync
+unsafe impl Send for ArenaPtr {}
+unsafe impl Sync for ArenaPtr {}
+
+impl ArenaPtr {
+    #[inline]
+    pub fn new(ptr: *const u8) -> Self {
+        Self(ptr)
+    }
+
+    #[inline]
+    pub fn null() -> Self {
+        Self(std::ptr::null())
+    }
+
+    #[inline]
+    pub fn as_ptr(self) -> *const u8 {
+        self.0
+    }
+}
+
+impl std::fmt::Debug for ArenaPtr {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "--arena-raw-pointer-0x({:?})", self.0)
+    }
+}
+
+#[repr(C, align(16))]
+#[derive(Clone, Copy)]
+pub(crate) struct SimdChunk(pub(crate) [u8; SIMD_CHUNK_BYTES]);
+
+impl Default for SimdChunk {
+    #[inline]
+    fn default() -> Self {
+        Self([0u8; SIMD_CHUNK_BYTES])
+    }
+}
+
+impl std::fmt::Debug for SimdChunk {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        // Show the actual bytes, trimming trailing zeros for readability
+        let end = self.0.iter().rposition(|&b| b != 0).map_or(0, |i| i + 1);
+        write!(f, "SimdChunk({:?})", &self.0[..end])
+    }
+}
+
+pub use crate::constants::PATH_BUF_SIZE;
+
+/// Chunk pointer capacity needed for the longest path the platform allows.
+pub(crate) const MAX_PATH_CHUNKS: usize = PATH_BUF_SIZE.div_ceil(SIMD_CHUNK_BYTES);
+
+/// Indices into a shared `SimdChunk` arena representing a file path.
+///
+/// All read methods require an explicit `arena_base` pointer from the owning
+/// `ChunkedPathStore`. The struct itself contains no raw pointers to the arena
+#[derive(Clone)]
+pub(crate) struct ChunkedString {
+    indices: ChunkIndices,
+    pub byte_len: u16,
+    /// Byte offset where the filename begins. 0 for root-level files.
+    pub filename_offset: u16,
+}
+
+impl ChunkedString {
+    pub fn empty() -> Self {
+        Self {
+            indices: SmallVec::new(),
+            byte_len: 0,
+            filename_offset: 0,
+        }
+    }
+
+    #[inline]
+    pub fn new(indices: ChunkIndices, byte_len: u16, filename_offset: u16) -> Self {
+        Self {
+            indices,
+            byte_len,
+            filename_offset,
+        }
+    }
+
+    #[cfg(test)]
+    pub fn chunk_count(&self) -> usize {
+        self.indices.len()
+    }
+
+    #[inline]
+    pub fn resolve_ptrs<'a>(&self, arena: ArenaPtr, buf: &'a mut [*const u8]) -> &'a [*const u8] {
+        let count = self.indices.len().min(buf.len());
+        let base = arena.as_ptr();
+        for (i, &idx) in self.indices[..count].iter().enumerate() {
+            buf[i] = unsafe { base.add(idx as usize * SIMD_CHUNK_BYTES) };
+        }
+        &buf[..count]
+    }
+
+    #[inline]
+    fn write_slice_to_vec(
+        indices: &[u32],
+        base: *const u8,
+        offset_in_chunk: usize,
+        len: usize,
+        vec: &mut Vec<u8>,
+    ) {
+        let mut written = 0usize;
+        for (i, &idx) in indices.iter().enumerate() {
+            let src = unsafe { base.add(idx as usize * SIMD_CHUNK_BYTES) };
+            let chunk_bytes = unsafe { core::slice::from_raw_parts(src, SIMD_CHUNK_BYTES) };
+            let start = if i == 0 { offset_in_chunk } else { 0 };
+            let end = SIMD_CHUNK_BYTES.min(start + (len - written));
+            vec.extend_from_slice(&chunk_bytes[start..end]);
+            written += end - start;
+        }
+    }
+
+    /// Return the filename portion as a `Cow<str>`.
+    ///
+    /// When the filename starts at a chunk boundary and fits in one chunk we
+    /// borrow directly from the arena (zero-copy). Otherwise we allocate.
+    /// Filenames are almost always <=16 bytes so the fast path dominates.
+    #[inline]
+    pub fn filename_cow<'a>(&self, arena: ArenaPtr) -> Cow<'a, str> {
+        let fname_offset = self.filename_offset as usize;
+        let fname_len = self.byte_len as usize - fname_offset;
+        if fname_len == 0 {
+            return Cow::Borrowed("");
+        }
+
+        let base = arena.as_ptr();
+        let start_chunk = fname_offset / SIMD_CHUNK_BYTES;
+        let offset_in_chunk = fname_offset % SIMD_CHUNK_BYTES;
+
+        if offset_in_chunk == 0 && fname_len <= SIMD_CHUNK_BYTES {
+            let ptr = unsafe { base.add(self.indices[start_chunk] as usize * SIMD_CHUNK_BYTES) };
+            let slice = unsafe { core::slice::from_raw_parts(ptr, fname_len) };
+            return Cow::Borrowed(unsafe { core::str::from_utf8_unchecked(slice) });
+        }
+
+        let mut out = String::with_capacity(fname_len);
+        let needed_chunks = chunks_needed(offset_in_chunk + fname_len);
+        Self::write_slice_to_vec(
+            &self.indices[start_chunk..start_chunk + needed_chunks],
+            base,
+            offset_in_chunk,
+            fname_len,
+            unsafe { out.as_mut_vec() },
+        );
+        Cow::Owned(out)
+    }
+
+    /// Truncates at `buf.len()` if exceeded -- use `[u8; PATH_BUF_SIZE]` to avoid.
+    #[inline]
+    pub fn read_to_buf<'a>(&self, arena: ArenaPtr, buf: &'a mut [u8]) -> &'a str {
+        let total = (self.byte_len as usize).min(buf.len());
+        let usable_chunks = total.div_ceil(SIMD_CHUNK_BYTES);
+        let chunks_to_copy = usable_chunks.min(self.indices.len());
+        let base = arena.as_ptr();
+
+        for (i, &idx) in self.indices[..chunks_to_copy].iter().enumerate() {
+            let src = unsafe { base.add(idx as usize * SIMD_CHUNK_BYTES) };
+            let dst_offset = i * SIMD_CHUNK_BYTES;
+            let take = SIMD_CHUNK_BYTES.min(total - dst_offset);
+
+            unsafe {
+                core::ptr::copy_nonoverlapping(src, buf.as_mut_ptr().add(dst_offset), take);
+            }
+        }
+
+        unsafe { core::str::from_utf8_unchecked(&buf[..total]) }
+    }
+
+    #[inline]
+    pub fn write_dir_to(&self, arena: ArenaPtr, out: &mut String) {
+        out.clear();
+
+        let dir_len = self.filename_offset as usize;
+        out.reserve(dir_len);
+        let dir_chunks = chunks_needed(dir_len).min(self.indices.len());
+        let base = arena.as_ptr();
+        let vec = unsafe { out.as_mut_vec() };
+        for (i, &idx) in self.indices[..dir_chunks].iter().enumerate() {
+            let src = unsafe { base.add(idx as usize * SIMD_CHUNK_BYTES) };
+            let take = SIMD_CHUNK_BYTES.min(dir_len - i * SIMD_CHUNK_BYTES);
+            vec.extend_from_slice(unsafe { core::slice::from_raw_parts(src, take) });
+        }
+    }
+
+    #[inline]
+    pub fn write_filename_to(&self, arena: ArenaPtr, out: &mut String) {
+        out.clear();
+
+        let fname_offset = self.filename_offset as usize;
+        let fname_len = self.byte_len as usize - fname_offset;
+        out.reserve(fname_len);
+        let start_chunk = fname_offset / SIMD_CHUNK_BYTES;
+        let offset_in_chunk = fname_offset % SIMD_CHUNK_BYTES;
+        let needed_chunks = chunks_needed(offset_in_chunk + fname_len);
+        Self::write_slice_to_vec(
+            &self.indices[start_chunk..start_chunk + needed_chunks],
+            arena.as_ptr(),
+            offset_in_chunk,
+            fname_len,
+            unsafe { out.as_mut_vec() },
+        );
+    }
+
+    #[inline]
+    pub fn write_to_string(&self, arena: ArenaPtr, out: &mut String) {
+        out.clear();
+
+        let total = self.byte_len as usize;
+        if total == 0 {
+            return;
+        }
+        out.reserve(total);
+        let base = arena.as_ptr();
+        let vec = unsafe { out.as_mut_vec() };
+        for (i, &idx) in self.indices.iter().enumerate() {
+            let src = unsafe { base.add(idx as usize * SIMD_CHUNK_BYTES) };
+            let take = SIMD_CHUNK_BYTES.min(total - i * SIMD_CHUNK_BYTES);
+            vec.extend_from_slice(unsafe { core::slice::from_raw_parts(src, take) });
+        }
+    }
+}
+
+impl std::fmt::Debug for ChunkedString {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("ChunkedString")
+            .field("indices", &self.indices.as_slice())
+            .field("chunks", &self.indices.len())
+            .field("byte_len", &self.byte_len)
+            .field("filename_offset", &self.filename_offset)
+            .finish()
+    }
+}
+
+#[inline]
+const fn chunks_needed(byte_len: usize) -> usize {
+    if byte_len == 0 {
+        0
+    } else {
+        byte_len.div_ceil(SIMD_CHUNK_BYTES)
+    }
+}
+
+#[derive(Clone, Debug)]
+pub(crate) struct ChunkedPathStore {
+    arena: Vec<SimdChunk>,
+}
+
+// SAFETY: arena is immutable after construction. Pointers derived from it are
+// only read during scoring (no mutation, no reallocation).
+unsafe impl Send for ChunkedPathStore {}
+unsafe impl Sync for ChunkedPathStore {}
+
+impl ChunkedPathStore {
+    pub fn heap_bytes(&self) -> usize {
+        self.arena.len() * SIMD_CHUNK_BYTES
+    }
+
+    #[cfg(test)]
+    fn unique_chunks(&self) -> usize {
+        self.arena.len()
+    }
+
+    #[inline]
+    pub fn as_arena_ptr(&self) -> ArenaPtr {
+        ArenaPtr::new(self.arena.as_ptr() as *const u8)
+    }
+}
+
+/// At runtime the builder should be split out from the store after `finish()`.
+#[derive(Clone, Debug)]
+pub(crate) struct ChunkedPathStoreBuilder {
+    arena: Vec<SimdChunk>,
+    chunk_dedup: AHashMap<[u8; SIMD_CHUNK_BYTES], u32>,
+}
+
+impl ChunkedPathStoreBuilder {
+    pub fn new(estimated_files: usize) -> Self {
+        let est_chunks = estimated_files * INLINE_CHUNKS; // we know that most of repos will fit
+        // most paths into 64 = 16 * INLINE_CHUNKS
+        Self {
+            arena: Vec::with_capacity(est_chunks),
+            chunk_dedup: AHashMap::with_capacity(est_chunks),
+        }
+    }
+
+    pub fn finish(self) -> ChunkedPathStore {
+        ChunkedPathStore { arena: self.arena }
+    }
+
+    pub fn as_arena_ptr(&self) -> ArenaPtr {
+        ArenaPtr::new(self.arena.as_ptr() as *const u8)
+    }
+
+    /// Like [`add_file_immediate`] but for directory paths where the entire
+    /// string is the "directory" portion (filename_offset == byte_len).
+    pub fn add_dir_immediate(&mut self, dir_rel_path: &str) -> ChunkedString {
+        self.add_file_immediate(dir_rel_path, dir_rel_path.len() as u16)
+    }
+
+    pub fn add_file_immediate(&mut self, rel_path: &str, filename_offset: u16) -> ChunkedString {
+        let path_bytes = rel_path.as_bytes();
+        let byte_len = rel_path.len();
+        let mut indices = ChunkIndices::with_capacity(chunks_needed(byte_len));
+
+        for chunk in path_bytes.chunks(SIMD_CHUNK_BYTES) {
+            let mut chunk_bytes = [0u8; SIMD_CHUNK_BYTES];
+            chunk_bytes[..chunk.len()].copy_from_slice(chunk);
+
+            let arena_idx = match self.chunk_dedup.get(&chunk_bytes) {
+                Some(&idx) => idx,
+                None => {
+                    let idx = self.arena.len() as u32;
+                    self.arena.push(SimdChunk(chunk_bytes));
+                    self.chunk_dedup.insert(chunk_bytes, idx);
+                    idx
+                }
+            };
+
+            indices.push(arena_idx);
+        }
+
+        ChunkedString::new(indices, byte_len as u16, filename_offset)
+    }
+}
+
+#[cfg(test)]
+pub(crate) fn build_chunked_path_store_from_strings(
+    rel_paths: &[String],
+    files: &[crate::types::FileItem],
+) -> (ChunkedPathStore, Vec<ChunkedString>) {
+    assert_eq!(rel_paths.len(), files.len());
+    let mut builder = ChunkedPathStoreBuilder::new(rel_paths.len());
+    let strings: Vec<ChunkedString> = rel_paths
+        .iter()
+        .zip(files.iter())
+        .map(|(rel_path, file)| builder.add_file_immediate(rel_path, file.path.filename_offset))
+        .collect();
+    (builder.finish(), strings)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn make_file_item(path: &str) -> crate::types::FileItem {
+        let filename_start = path
+            .rfind(std::path::is_separator)
+            .map(|i| i + 1)
+            .unwrap_or(0) as u16;
+        crate::types::FileItem::new_raw(filename_start, 0, 0, None, false)
+    }
+
+    fn build_test_store(
+        paths: &[&str],
+    ) -> (
+        ChunkedPathStore,
+        Vec<ChunkedString>,
+        Vec<crate::types::FileItem>,
+    ) {
+        let mut files: Vec<crate::types::FileItem> =
+            paths.iter().map(|p| make_file_item(p)).collect();
+        let path_strings: Vec<String> = paths.iter().map(|p| p.to_string()).collect();
+        let (store, strings) = build_chunked_path_store_from_strings(&path_strings, &files);
+        for (i, file) in files.iter_mut().enumerate() {
+            file.set_path(strings[i].clone());
+        }
+        (store, strings, files)
+    }
+
+    #[test]
+    fn test_chunked_store_empty() {
+        let (store, strings, _files) = build_test_store(&[]);
+        assert_eq!(strings.len(), 0);
+        assert_eq!(store.unique_chunks(), 0);
+    }
+
+    #[test]
+    fn test_chunked_store_basic() {
+        let (store, strings, _files) =
+            build_test_store(&["src/lib.rs", "src/main.rs", "Cargo.toml"]);
+        let arena = store.as_arena_ptr();
+
+        assert_eq!(strings.len(), 3);
+        assert!(store.unique_chunks() >= 2);
+
+        let mut buf = [0u8; 512];
+        assert_eq!(
+            strings[0].read_to_buf(arena, &mut buf).len(),
+            "src/lib.rs".len()
+        );
+        assert_eq!(
+            strings[2].read_to_buf(arena, &mut buf).len(),
+            "Cargo.toml".len()
+        );
+    }
+
+    #[test]
+    fn test_chunked_string_full_path() {
+        let (store, strings, _files) = build_test_store(&[
+            "src/components/Button.tsx",
+            "src/components/Button.test.tsx",
+        ]);
+        let arena = store.as_arena_ptr();
+        let cs = &strings[0];
+
+        let mut buf = [0u8; 512];
+        assert_eq!(cs.read_to_buf(arena, &mut buf), "src/components/Button.tsx");
+        assert_eq!(cs.byte_len, 25);
+        assert_eq!(cs.filename_offset, 15);
+
+        let cs = &strings[1];
+        let mut buf = [0u8; 512];
+        assert_eq!(
+            cs.read_to_buf(arena, &mut buf),
+            "src/components/Button.test.tsx"
+        );
+        assert_eq!(cs.byte_len, 30);
+        assert_eq!(cs.filename_offset, 15);
+    }
+
+    #[test]
+    fn test_chunked_string_dir_and_filename() {
+        let (store, strings, _files) = build_test_store(&["src/components/Button.tsx"]);
+        let arena = store.as_arena_ptr();
+        let cs = &strings[0];
+
+        let mut s = String::new();
+        cs.write_dir_to(arena, &mut s);
+        assert_eq!(s, "src/components/");
+        cs.write_filename_to(arena, &mut s);
+        assert_eq!(s, "Button.tsx");
+    }
+
+    #[test]
+    fn test_chunked_string_root_file() {
+        let (store, strings, _files) = build_test_store(&["Cargo.toml"]);
+        let arena = store.as_arena_ptr();
+        let cs = &strings[0];
+
+        let mut s = String::new();
+        cs.write_dir_to(arena, &mut s);
+        assert_eq!(s, "");
+        cs.write_filename_to(arena, &mut s);
+        assert_eq!(s, "Cargo.toml");
+        let mut buf = [0u8; 512];
+        assert_eq!(cs.read_to_buf(arena, &mut buf), "Cargo.toml");
+    }
+
+    #[test]
+    fn test_chunked_string_resolve_ptrs() {
+        let (store, strings, _files) = build_test_store(&["src/components/Button.tsx"]);
+        let arena = store.as_arena_ptr();
+        let cs = &strings[0];
+
+        let mut ptrs = [std::ptr::null::<u8>(); MAX_PATH_CHUNKS];
+        let resolved = cs.resolve_ptrs(arena, &mut ptrs);
+        assert_eq!(resolved.len(), 2); // 25 bytes = 2 chunks
+
+        // Verify we can read back the bytes
+        let mut reconstructed = Vec::new();
+        for (i, &ptr) in resolved.iter().enumerate() {
+            let chunk = unsafe { std::slice::from_raw_parts(ptr, SIMD_CHUNK_BYTES) };
+            let start = i * SIMD_CHUNK_BYTES;
+            let take = SIMD_CHUNK_BYTES.min(25 - start);
+            reconstructed.extend_from_slice(&chunk[..take]);
+        }
+        assert_eq!(
+            std::str::from_utf8(&reconstructed).unwrap(),
+            "src/components/Button.tsx"
+        );
+    }
+
+    #[test]
+    fn test_resolve_ptrs_path_exceeding_512_bytes() {
+        // Regression: a fixed 32-ptr buffer covered only 512 bytes while
+        // PATH_BUF_SIZE (libc::PATH_MAX) allows longer paths, panicking with
+        // "index out of bounds: the len is 32 but the index is 32"
+        let mut path = String::new();
+        while path.len() < 600 {
+            path.push_str("deeply_nested_directory_segment/");
+        }
+        path.push_str("needle_file.rs");
+        assert!(path.len() > 512 && path.len() < PATH_BUF_SIZE);
+
+        let (store, strings, _files) = build_test_store(&[path.as_str()]);
+        let arena = store.as_arena_ptr();
+        let cs = &strings[0];
+        assert!(cs.chunk_count() > 32, "path must span more than 32 chunks");
+
+        let mut ptrs = [std::ptr::null::<u8>(); MAX_PATH_CHUNKS];
+        let resolved = cs.resolve_ptrs(arena, &mut ptrs);
+
+        // Truncation is not acceptable either: it silently drops the tail of
+        // the path (including the filename here) from fuzzy matching.
+        assert_eq!(
+            resolved.len(),
+            cs.chunk_count(),
+            "resolve_ptrs must resolve every chunk of a PATH_MAX-legal path"
+        );
+
+        let total = cs.byte_len as usize;
+        let mut reconstructed = Vec::with_capacity(total);
+        for (i, &ptr) in resolved.iter().enumerate() {
+            let take = SIMD_CHUNK_BYTES.min(total - i * SIMD_CHUNK_BYTES);
+            reconstructed.extend_from_slice(unsafe { std::slice::from_raw_parts(ptr, take) });
+        }
+        assert_eq!(std::str::from_utf8(&reconstructed).unwrap(), path);
+    }
+
+    #[test]
+    fn test_filename_cow_mid_chunk() {
+        let (store, strings, _files) = build_test_store(&["src/components/Button.tsx"]);
+        let arena = store.as_arena_ptr();
+        let cs = &strings[0];
+
+        assert_eq!(cs.filename_offset, 15);
+        assert_eq!(cs.byte_len, 25);
+
+        let fname = cs.filename_cow(arena);
+        assert_eq!(&*fname, "Button.tsx");
+    }
+
+    #[test]
+    fn test_filename_cow_chunk_aligned() {
+        let path = "0123456789abcdef/file.txt";
+        let (store, strings, _files) = build_test_store(&[path]);
+        let arena = store.as_arena_ptr();
+        let cs = &strings[0];
+
+        assert_eq!(cs.filename_offset, 17);
+        let fname = cs.filename_cow(arena);
+        assert_eq!(&*fname, "file.txt");
+    }
+
+    #[test]
+    fn test_filename_cow_root_file() {
+        let (store, strings, _files) = build_test_store(&["Cargo.toml"]);
+        let arena = store.as_arena_ptr();
+        let cs = &strings[0];
+
+        assert_eq!(cs.filename_offset, 0);
+        let fname = cs.filename_cow(arena);
+        assert_eq!(&*fname, "Cargo.toml");
+    }
+
+    #[test]
+    fn test_chunked_string_long_path() {
+        let path = "very/deeply/nested/directory/structure/with/many/levels/file.txt";
+        let (store, strings, _files) = build_test_store(&[path]);
+        let arena = store.as_arena_ptr();
+        let cs = &strings[0];
+
+        let mut buf = [0u8; 512];
+        assert_eq!(cs.read_to_buf(arena, &mut buf), path);
+        assert!(
+            cs.chunk_count() <= 6,
+            "should fit inline in ChunkIndices (INLINE_CHUNKS={})",
+            INLINE_CHUNKS
+        );
+    }
+
+    #[test]
+    fn test_chunked_string_clone() {
+        let (store, strings, _files) = build_test_store(&["src/main.rs"]);
+        let arena = store.as_arena_ptr();
+        let cs = &strings[0];
+        let cs2 = cs.clone();
+
+        let mut buf1 = [0u8; 512];
+        let mut buf2 = [0u8; 512];
+        assert_eq!(
+            cs.read_to_buf(arena, &mut buf1),
+            cs2.read_to_buf(arena, &mut buf2)
+        );
+    }
+
+    #[test]
+    fn test_chunked_string_full_path_roundtrip() {
+        let paths = [
+            "src/components/Button.tsx",
+            "src/components/ui/DatePicker.tsx",
+            "very/deeply/nested/directory/structure/file.txt",
+            "Cargo.toml",
+            "a.rs",
+        ];
+        let (store, strings, _files) = build_test_store(&paths);
+        let arena = store.as_arena_ptr();
+
+        for (i, expected) in paths.iter().enumerate() {
+            let mut buf = [0u8; 512];
+            let got = strings[i].read_to_buf(arena, &mut buf);
+            assert_eq!(got, *expected, "full path roundtrip failed for file {i}");
+
+            let mut ds = String::new();
+            let mut fs = String::new();
+            strings[i].write_dir_to(arena, &mut ds);
+            strings[i].write_filename_to(arena, &mut fs);
+            assert_eq!(
+                format!("{ds}{fs}"),
+                *expected,
+                "dir+fname mismatch for file {i}"
+            );
+        }
+    }
+}

crates/fff-core/src/simd_string_utils/case.rs

@@ -0,0 +1,168 @@
+#[inline]
+pub fn ascii_swap_case(b: u8) -> u8 {
+    b ^ 0x20
+}
+
+#[inline]
+fn eq_lowered_scalar(h: *const u8, needle_lower: &[u8]) -> bool {
+    for (i, &n) in needle_lower.iter().enumerate() {
+        if unsafe { *h.add(i) }.to_ascii_lowercase() != n {
+            return false;
+        }
+    }
+    true
+}
+
+/// AVX2 only has a **signed** byte compare (`cmpgt`), but we need an
+/// **unsigned** range check (`'A' <= byte <= 'Z'`). XOR-ing every byte with
+/// `0x80` maps the unsigned range `[0, 255]` into the signed range
+/// `[-128, 127]` preserving order, so signed `cmpgt` becomes correct
+#[cfg(target_arch = "x86_64")]
+#[target_feature(enable = "avx2")]
+unsafe fn eq_lowered_avx2(h: *const u8, needle_lower: &[u8]) -> bool {
+    use core::arch::x86_64::*;
+
+    let len = needle_lower.len();
+    let mut i = 0usize;
+
+    let flip = _mm256_set1_epi8(0x80u8 as i8);
+    let a_minus_1 = _mm256_set1_epi8((b'A' - 1) as i8 ^ 0x80u8 as i8);
+    let z_plus_1 = _mm256_set1_epi8((b'Z' + 1) as i8 ^ 0x80u8 as i8);
+    let bit20 = _mm256_set1_epi8(0x20u8 as i8);
+
+    while i + 32 <= len {
+        let hv = unsafe { _mm256_loadu_si256(h.add(i) as *const __m256i) };
+        let nv = unsafe { _mm256_loadu_si256(needle_lower.as_ptr().add(i) as *const __m256i) };
+
+        // Signed-domain range check selects uppercase lanes, OR bit 5 folds them.
+        let x = _mm256_xor_si256(hv, flip);
+        let ge_a = _mm256_cmpgt_epi8(x, a_minus_1);
+        let le_z = _mm256_cmpgt_epi8(z_plus_1, x);
+        let upper = _mm256_and_si256(ge_a, le_z);
+        let folded = _mm256_or_si256(hv, _mm256_and_si256(upper, bit20));
+
+        let eq = _mm256_cmpeq_epi8(folded, nv);
+        if _mm256_movemask_epi8(eq) != -1i32 {
+            return false;
+        }
+
+        i += 32;
+    }
+
+    while i < len {
+        if unsafe { *h.add(i) }.to_ascii_lowercase() != needle_lower[i] {
+            return false;
+        }
+        i += 1;
+    }
+    true
+}
+
+/// Unsigned range checks (`vcge`/`vcle`) detect uppercase ASCII, bit 5 folds
+/// to lowercase, then equality is checked via udot: xors the folded haystack
+/// with the pre-lowered needle and dot-product the difference with itself
+/// any non-zero byte produces a non-zero u32 lane. udot is emitted via inline
+/// asm because `vdotq_u32` is still behind an unstable feature gate.
+#[cfg(target_arch = "aarch64")]
+#[target_feature(enable = "neon,dotprod")]
+unsafe fn eq_lowered_neon_dotprod(h: *const u8, needle_lower: &[u8]) -> bool {
+    use core::arch::aarch64::*;
+
+    let len = needle_lower.len();
+    let mut i = 0usize;
+
+    let a_val = vdupq_n_u8(b'A');
+    let z_val = vdupq_n_u8(b'Z');
+    let bit20 = vdupq_n_u8(0x20);
+
+    while i + 16 <= len {
+        let hv = unsafe { vld1q_u8(h.add(i)) };
+        let nv = unsafe { vld1q_u8(needle_lower.as_ptr().add(i)) };
+
+        let upper = vandq_u8(vcgeq_u8(hv, a_val), vcleq_u8(hv, z_val));
+        let folded = vorrq_u8(hv, vandq_u8(upper, bit20));
+        let xored = veorq_u8(folded, nv);
+
+        let dots: uint32x4_t;
+        let zero = vdupq_n_u32(0);
+        unsafe {
+            core::arch::asm!(
+                "udot {d:v}.4s, {a:v}.16b, {b:v}.16b",
+                d = inlateout(vreg) zero => dots,
+                a = in(vreg) xored,
+                b = in(vreg) xored,
+            );
+        }
+
+        if vmaxvq_u32(dots) != 0 {
+            return false;
+        }
+
+        i += 16;
+    }
+
+    while i < len {
+        if unsafe { *h.add(i) }.to_ascii_lowercase() != needle_lower[i] {
+            return false;
+        }
+        i += 1;
+    }
+    true
+}
+
+/// Case-insensitive equality of `needle_lower` against the haystack bytes
+/// starting at `h`. `needle_lower` must be pre-lowercased (ASCII).
+///
+/// # Safety
+/// `h` must be valid for reads of `needle_lower.len()` bytes.
+#[inline]
+pub(crate) unsafe fn eq_lowered_case(haystack: *const u8, needle_lower: &[u8]) -> bool {
+    #[cfg(target_arch = "x86_64")]
+    {
+        if needle_lower.len() >= 32 && std::is_x86_feature_detected!("avx2") {
+            return unsafe { eq_lowered_avx2(haystack, needle_lower) };
+        }
+    }
+    #[cfg(target_arch = "aarch64")]
+    {
+        if needle_lower.len() >= 16 && std::arch::is_aarch64_feature_detected!("dotprod") {
+            return unsafe { eq_lowered_neon_dotprod(haystack, needle_lower) };
+        }
+    }
+
+    eq_lowered_scalar(haystack, needle_lower)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn eq_lowered(haystack: &[u8], needle_lower: &[u8]) -> bool {
+        assert!(haystack.len() >= needle_lower.len());
+        unsafe { eq_lowered_case(haystack.as_ptr(), needle_lower) }
+    }
+
+    #[test]
+    fn swap_case_toggles_letters() {
+        assert_eq!(ascii_swap_case(b'n'), b'N');
+        assert_eq!(ascii_swap_case(b'N'), b'n');
+        assert_eq!(ascii_swap_case(b'z'), b'Z');
+    }
+
+    #[test]
+    fn eq_matches_std_semantics() {
+        assert!(eq_lowered(b"Hello", b"hello"));
+        assert!(eq_lowered(b"HELLO WORLD", b"hello"));
+        assert!(!eq_lowered(b"Hellp", b"hello"));
+        // Non-letters must not fold: '[' (0x5B) vs '{' (0x7B) differ only in bit 5.
+        assert!(!eq_lowered(b"A[", b"a{"));
+        assert!(eq_lowered(b"A{", b"a{"));
+        // Long inputs exercise the SIMD kernels.
+        let hay = b"INT STRUCT MUTEX *LOCK(STRUCT MUTEX *LOCK) { RETURN 0; }";
+        let needle: Vec<u8> = hay.iter().map(|b| b.to_ascii_lowercase()).collect();
+        assert!(eq_lowered(hay, &needle));
+        let mut bad = needle.clone();
+        *bad.last_mut().unwrap() = b'!';
+        assert!(!eq_lowered(hay, &bad));
+    }
+}

crates/fff-core/src/simd_string_utils/memmem.rs

@@ -0,0 +1,494 @@
+use super::case::{ascii_swap_case, eq_lowered_case};
+use smallvec::SmallVec;
+
+// Byte frequency table stolen from memchr
+const BYTE_FREQUENCIES: [u8; 256] = [
+    55, 52, 51, 50, 49, 48, 47, 46, 45, 103, 242, 66, 67, 229, 44, 43, // 0x00
+    42, 41, 40, 39, 38, 37, 36, 35, 34, 33, 56, 32, 31, 30, 29, 28, // 0x10
+    255, 148, 164, 149, 136, 160, 155, 173, 221, 222, 134, 122, 232, 202, 215, 224, // 0x20
+    208, 220, 204, 187, 183, 179, 177, 168, 178, 200, 226, 195, 154, 184, 174, 126, // 0x30
+    120, 191, 157, 194, 170, 189, 162, 161, 150, 193, 142, 137, 171, 176, 185,
+    167, // 0x40 A-O
+    186, 112, 175, 192, 188, 156, 140, 143, 123, 133, 128, 147, 138, 146, 114,
+    223, // 0x50 P-_
+    151, 249, 216, 238, 236, 253, 227, 218, 230, 247, 135, 180, 241, 233, 246,
+    244, // 0x60 a-o
+    231, 139, 245, 243, 251, 235, 201, 196, 240, 214, 152, 182, 205, 181, 127,
+    27, // 0x70 p-DEL
+    212, 211, 210, 213, 228, 197, 169, 159, 131, 172, 105, 80, 98, 96, 97, 81, // 0x80
+    207, 145, 116, 115, 144, 130, 153, 121, 107, 132, 109, 110, 124, 111, 82, 108, // 0x90
+    118, 141, 113, 129, 119, 125, 165, 117, 92, 106, 83, 72, 99, 93, 65, 79, // 0xa0
+    166, 237, 163, 199, 190, 225, 209, 203, 198, 217, 219, 206, 234, 248, 158, 239, // 0xb0
+    255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, // 0xc0
+    255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, // 0xd0
+    255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, // 0xe0
+    255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, // 0xf0
+];
+
+#[inline]
+fn rank(lower: u8) -> u8 {
+    if lower.is_ascii_lowercase() {
+        BYTE_FREQUENCIES[lower as usize].max(BYTE_FREQUENCIES[ascii_swap_case(lower) as usize])
+    } else {
+        BYTE_FREQUENCIES[lower as usize]
+    }
+}
+
+/// Pick two needle positions with the rarest bytes (case-insensitive)
+fn select_rare_pair(needle_lower: &[u8]) -> (usize, usize) {
+    debug_assert!(needle_lower.len() >= 2);
+
+    let mut best1 = (u8::MAX, 0usize); // (rank, position)
+    let mut best2 = (u8::MAX, 1usize);
+
+    for (i, &b) in needle_lower.iter().enumerate() {
+        let r = rank(b);
+        if r < best1.0 {
+            best2 = best1;
+            best1 = (r, i);
+        } else if r < best2.0 && i != best1.1 {
+            best2 = (r, i);
+        }
+    }
+
+    let i1 = best1.1.min(best2.1);
+    let i2 = best1.1.max(best2.1);
+    (i1, i2)
+}
+
+/// Extract a 16-bit bitmask from a NEON comparison result (each byte 0x00 or 0xFF)
+/// Bit *i* of the result corresponds to byte *i* of the input vector
+#[cfg(target_arch = "aarch64")]
+#[target_feature(enable = "neon")]
+#[inline]
+unsafe fn neon_movemask(v: core::arch::aarch64::uint8x16_t) -> u16 {
+    use core::arch::aarch64::*;
+
+    // AND each byte with its bit-position mask, then horizontally sum each half.
+    static BITS: [u8; 16] = [1, 2, 4, 8, 16, 32, 64, 128, 1, 2, 4, 8, 16, 32, 64, 128];
+    let bit_mask = unsafe { vld1q_u8(BITS.as_ptr()) };
+    let masked = vandq_u8(v, bit_mask);
+    let lo = vaddv_u8(vget_low_u8(masked));
+    let hi = vaddv_u8(vget_high_u8(masked));
+    (lo as u16) | ((hi as u16) << 8)
+}
+
+/// AVX2 packed-pair kernel: scan 32 haystack positions per iteration,
+/// checking two rare bytes (case-insensitive) simultaneously
+#[cfg(target_arch = "x86_64")]
+#[target_feature(enable = "avx2")]
+unsafe fn find_packed_pair_avx2(
+    haystack: &[u8],
+    needle_lower: &[u8],
+    i1: usize,
+    i2: usize,
+) -> Option<usize> {
+    use core::arch::x86_64::*;
+
+    let n = needle_lower.len();
+    let hlen = haystack.len();
+    let ptr = haystack.as_ptr();
+    let last_start = hlen - n; // last valid match-start position
+
+    let b1 = needle_lower[i1];
+    let b1_alt = if b1.is_ascii_lowercase() {
+        ascii_swap_case(b1)
+    } else {
+        b1
+    };
+    let b2 = needle_lower[i2];
+    let b2_alt = if b2.is_ascii_lowercase() {
+        ascii_swap_case(b2)
+    } else {
+        b2
+    };
+
+    let v1_lo = _mm256_set1_epi8(b1 as i8);
+    let v1_hi = _mm256_set1_epi8(b1_alt as i8);
+    let v2_lo = _mm256_set1_epi8(b2 as i8);
+    let v2_hi = _mm256_set1_epi8(b2_alt as i8);
+
+    // Loads come from ptr+offset+i1 and ptr+offset+i2, so we need offset + max(i1,i2) + 31 < hlen.
+    let max_idx = i1.max(i2);
+    let max_offset = hlen.saturating_sub(max_idx + 32);
+    let mut offset = 0usize;
+
+    while offset <= max_offset {
+        let chunk1 = unsafe { _mm256_loadu_si256(ptr.add(offset + i1) as *const __m256i) };
+        let chunk2 = unsafe { _mm256_loadu_si256(ptr.add(offset + i2) as *const __m256i) };
+
+        // Case-insensitive match: OR both case variants, then AND the two positions.
+        let eq1 = _mm256_or_si256(
+            _mm256_cmpeq_epi8(chunk1, v1_lo),
+            _mm256_cmpeq_epi8(chunk1, v1_hi),
+        );
+        let eq2 = _mm256_or_si256(
+            _mm256_cmpeq_epi8(chunk2, v2_lo),
+            _mm256_cmpeq_epi8(chunk2, v2_hi),
+        );
+
+        let mut mask = _mm256_movemask_epi8(_mm256_and_si256(eq1, eq2)) as u32;
+
+        // Candidates are visited in increasing position order, so the first
+        // verified candidate is the leftmost match
+        while mask != 0 {
+            let bit = mask.trailing_zeros() as usize;
+            let candidate = offset + bit;
+            if candidate > last_start {
+                return None;
+            }
+            if unsafe { eq_lowered_case(ptr.add(candidate), needle_lower) } {
+                return Some(candidate);
+            }
+            mask &= mask - 1;
+        }
+
+        offset += 32;
+    }
+
+    // handle remaining characters
+    if offset <= last_start {
+        let rare_pos = if rank(needle_lower[i1]) <= rank(needle_lower[i2]) {
+            i1
+        } else {
+            i2
+        };
+        let rare_byte = needle_lower[rare_pos];
+        let tail_start = offset + rare_pos;
+        let tail_end = last_start + rare_pos + 1;
+        if tail_start < tail_end {
+            let tail_space = &haystack[tail_start..tail_end];
+            if rare_byte.is_ascii_lowercase() {
+                for pos in memchr::memchr2_iter(rare_byte, ascii_swap_case(rare_byte), tail_space) {
+                    let candidate = offset + pos;
+                    if unsafe { eq_lowered_case(ptr.add(candidate), needle_lower) } {
+                        return Some(candidate);
+                    }
+                }
+            } else {
+                for pos in memchr::memchr_iter(rare_byte, tail_space) {
+                    let candidate = offset + pos;
+                    if unsafe { eq_lowered_case(ptr.add(candidate), needle_lower) } {
+                        return Some(candidate);
+                    }
+                }
+            }
+        }
+    }
+
+    None
+}
+
+#[cfg(target_arch = "aarch64")]
+#[target_feature(enable = "neon")]
+unsafe fn find_packed_pair_neon(
+    haystack: &[u8],
+    needle_lower: &[u8],
+    i1: usize,
+    i2: usize,
+) -> Option<usize> {
+    use core::arch::aarch64::*;
+
+    let n = needle_lower.len();
+    let hlen = haystack.len();
+    let ptr = haystack.as_ptr();
+    let last_start = hlen - n;
+
+    let b1 = needle_lower[i1];
+    let b1_alt = if b1.is_ascii_lowercase() {
+        ascii_swap_case(b1)
+    } else {
+        b1
+    };
+    let b2 = needle_lower[i2];
+    let b2_alt = if b2.is_ascii_lowercase() {
+        ascii_swap_case(b2)
+    } else {
+        b2
+    };
+
+    let v1_lo = vdupq_n_u8(b1);
+    let v1_hi = vdupq_n_u8(b1_alt);
+    let v2_lo = vdupq_n_u8(b2);
+    let v2_hi = vdupq_n_u8(b2_alt);
+
+    let max_idx = i1.max(i2);
+    let max_offset = hlen.saturating_sub(max_idx + 16);
+    let mut offset = 0usize;
+
+    while offset <= max_offset {
+        let chunk1 = unsafe { vld1q_u8(ptr.add(offset + i1)) };
+        let chunk2 = unsafe { vld1q_u8(ptr.add(offset + i2)) };
+
+        // Case-insensitive match: OR both case variants, then AND the two positions.
+        let eq1 = vorrq_u8(vceqq_u8(chunk1, v1_lo), vceqq_u8(chunk1, v1_hi));
+        let eq2 = vorrq_u8(vceqq_u8(chunk2, v2_lo), vceqq_u8(chunk2, v2_hi));
+
+        let mut mask = unsafe { neon_movemask(vandq_u8(eq1, eq2)) };
+
+        while mask != 0 {
+            let bit = mask.trailing_zeros() as usize;
+            let candidate = offset + bit;
+            if candidate > last_start {
+                return None;
+            }
+            if unsafe { eq_lowered_case(ptr.add(candidate), needle_lower) } {
+                return Some(candidate);
+            }
+            mask &= mask - 1;
+        }
+
+        offset += 16;
+    }
+
+    // Tail: remaining positions that couldn't fill a full vector.
+    if offset <= last_start {
+        let rare_pos = if rank(needle_lower[i1]) <= rank(needle_lower[i2]) {
+            i1
+        } else {
+            i2
+        };
+        let rare_byte = needle_lower[rare_pos];
+        let tail_start = offset + rare_pos;
+        let tail_end = last_start + rare_pos + 1;
+        if tail_start < tail_end {
+            let tail_space = &haystack[tail_start..tail_end];
+            if rare_byte.is_ascii_lowercase() {
+                for pos in memchr::memchr2_iter(rare_byte, ascii_swap_case(rare_byte), tail_space) {
+                    let candidate = offset + pos;
+                    if unsafe { eq_lowered_case(ptr.add(candidate), needle_lower) } {
+                        return Some(candidate);
+                    }
+                }
+            } else {
+                for pos in memchr::memchr_iter(rare_byte, tail_space) {
+                    let candidate = offset + pos;
+                    if unsafe { eq_lowered_case(ptr.add(candidate), needle_lower) } {
+                        return Some(candidate);
+                    }
+                }
+            }
+        }
+    }
+
+    None
+}
+
+fn find_first_byte_with_memchr(haystack: &[u8], needle_lower: &[u8]) -> Option<usize> {
+    let n = needle_lower.len();
+    debug_assert!(n >= 1 && n <= haystack.len());
+
+    let search_space = &haystack[..=haystack.len() - n];
+    let first = needle_lower[0];
+
+    if first.is_ascii_lowercase() {
+        let alt = ascii_swap_case(first);
+        for pos in memchr::memchr2_iter(first, alt, search_space) {
+            if unsafe { eq_lowered_case(haystack.as_ptr().add(pos), needle_lower) } {
+                return Some(pos);
+            }
+        }
+    } else {
+        for pos in memchr::memchr_iter(first, search_space) {
+            if unsafe { eq_lowered_case(haystack.as_ptr().add(pos), needle_lower) } {
+                return Some(pos);
+            }
+        }
+    }
+    None
+}
+
+/// ASCII case-insensitive substring search returning the leftmost match
+/// position. `needle_lower` must be pre-lowercased (ASCII).
+// pub because it is used in out of the crate benchmarks
+#[doc(hidden)] // it's pub only for benches
+pub fn find(haystack: &[u8], needle_lower: &[u8]) -> Option<usize> {
+    let n = needle_lower.len();
+    if n == 0 {
+        return Some(0);
+    }
+    if n > haystack.len() {
+        return None;
+    }
+
+    if n == 1 {
+        let first = needle_lower[0];
+        return if first.is_ascii_lowercase() {
+            memchr::memchr2(first, ascii_swap_case(first), haystack)
+        } else {
+            memchr::memchr(first, haystack)
+        };
+    }
+
+    #[cfg_attr(
+        not(any(target_arch = "x86_64", target_arch = "aarch64")),
+        allow(unused_variables)
+    )]
+    let (i1, i2) = select_rare_pair(needle_lower);
+
+    #[cfg(target_arch = "x86_64")]
+    {
+        if std::is_x86_feature_detected!("avx2") {
+            // Need enough haystack for at least one vector load.
+            let max_idx = i1.max(i2);
+            if haystack.len() >= max_idx + 32 {
+                return unsafe { find_packed_pair_avx2(haystack, needle_lower, i1, i2) };
+            }
+        }
+    }
+
+    #[cfg(target_arch = "aarch64")]
+    {
+        // Packed-pair wins when the first byte is common (memchr2 drowns in
+        // false positives), but a rare first byte (z, q, x, ...) makes
+        // memchr2's raw throughput dominate. Threshold 200 on the frequency
+        // table splits common letters (s=243, e=253) from rare ones (z=152).
+        let first_byte_rank = rank(needle_lower[0]);
+        let max_idx = i1.max(i2);
+        if first_byte_rank >= 200 && haystack.len() >= max_idx + 16 {
+            return unsafe { find_packed_pair_neon(haystack, needle_lower, i1, i2) };
+        }
+    }
+
+    // fallbacks to memchr based implementation cause we still have it and it supports more SIMD backends
+    // TODO convert all the supported backend by memchr and get rid of the fallback
+    find_first_byte_with_memchr(haystack, needle_lower)
+}
+
+/// A case insensitive find that works better with smaller strings, doesn't unwrap a complicated
+/// AVX backend we use for grep because only cpu flags check takes usually more time than find itself
+pub fn find_case_insensitive_short(haystack: &[u8], needle: &[u8]) -> Option<usize> {
+    debug_assert!(haystack.len() < 1024);
+    let mut needle_lower: SmallVec<[u8; 64]> = SmallVec::from_slice(needle);
+    needle_lower.make_ascii_lowercase();
+
+    find(haystack, &needle_lower)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn reference_find(haystack: &[u8], needle_lower: &[u8]) -> Option<usize> {
+        if needle_lower.is_empty() {
+            return Some(0);
+        }
+        if needle_lower.len() > haystack.len() {
+            return None;
+        }
+        haystack
+            .windows(needle_lower.len())
+            .position(|w| w.eq_ignore_ascii_case(needle_lower))
+    }
+
+    #[test]
+    fn basic_case_insensitive() {
+        assert_eq!(find(b"Hello World", b"hello"), Some(0));
+        assert_eq!(find(b"Hello World", b"world"), Some(6));
+        assert_eq!(find(b"NOMORE bugs", b"nomore"), Some(0));
+        assert_eq!(find(b"Hello World", b"xyz"), None);
+        assert!(find(b"Hello World", b"o w").is_some());
+    }
+
+    #[test]
+    fn edge_cases() {
+        assert_eq!(find(b"ab", b"ab"), Some(0));
+        assert_eq!(find(b"AB", b"ab"), Some(0));
+        assert_eq!(find(b"a", b"ab"), None);
+        assert_eq!(find(b"anything", b""), Some(0));
+        assert_eq!(find(b"", b"x"), None);
+        assert_eq!(find(b"xxA", b"a"), Some(2));
+        assert_eq!(find(b"xx:", b":"), Some(2));
+    }
+
+    #[test]
+    fn returns_leftmost_match() {
+        assert_eq!(find(b"foo FOO foo", b"foo"), Some(0));
+        let mut big = vec![b'.'; 300];
+        big[100..103].copy_from_slice(b"FoO");
+        big[200..203].copy_from_slice(b"foo");
+        assert_eq!(find(&big, b"foo"), Some(100));
+    }
+
+    #[test]
+    fn non_letter_bytes_do_not_case_fold() {
+        // '[' (0x5B) and '{' (0x7B) differ only in bit 5 but are not letters.
+        // A fold implemented as a bare `| 0x20` would falsely match these.
+        assert_eq!(find(b"A[", b"a{"), None);
+        assert_eq!(find(b"x@y", b"x`y"), None);
+        assert_eq!(find(b"a]b", b"a}b"), None);
+        assert_eq!(find(b"A{", b"a{"), Some(0));
+    }
+
+    #[test]
+    fn matches_reference_on_random_inputs() {
+        // Deterministic xorshift PRNG — no external deps.
+        let mut state = 0x9E3779B97F4A7C15u64;
+        let mut next = move || {
+            state ^= state << 13;
+            state ^= state >> 7;
+            state ^= state << 17;
+            state
+        };
+
+        // Alphabet with letters, both-case pairs, and 0x20-differing symbols.
+        let alphabet = b"aAbBzZ [{@`]}^~_0.\n";
+        for _ in 0..2000 {
+            let hlen = (next() % 200) as usize;
+            let nlen = (next() % 8) as usize;
+            let haystack: Vec<u8> = (0..hlen)
+                .map(|_| alphabet[(next() % alphabet.len() as u64) as usize])
+                .collect();
+            let needle: Vec<u8> = (0..nlen)
+                .map(|_| alphabet[(next() % alphabet.len() as u64) as usize].to_ascii_lowercase())
+                .collect();
+
+            assert_eq!(
+                find(&haystack, &needle),
+                reference_find(&haystack, &needle),
+                "mismatch for haystack={:?} needle={:?}",
+                haystack,
+                needle,
+            );
+        }
+    }
+
+    #[test]
+    fn long_haystack_simd_paths() {
+        let haystack =
+            b"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaTHIS_IS_A_LONG_NEEDLE_TESTbbbbbbbbbbbbbbbbbb";
+        assert_eq!(find(haystack, b"this_is_a_long_needle_test"), Some(32));
+        assert_eq!(find(haystack, b"this_is_a_long_needle_testz"), None);
+
+        // Needle >= 16 bytes exercises SIMD verify.
+        let haystack2 = b"int STRUCT MUTEX *LOCK(struct mutex *lock) { return 0; }";
+        assert_eq!(find(haystack2, b"struct mutex *lock"), Some(4));
+
+        let upper_hay = b"ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ";
+        assert_eq!(find(upper_hay, b"qrstuvwxyz0123456789a"), Some(16));
+        assert_eq!(find(upper_hay, b"qrstuvwxyz01234567899"), None);
+
+        // Needle at very end / very start.
+        let end_hay = b"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxfind_me";
+        assert_eq!(find(end_hay, b"find_me"), Some(end_hay.len() - 7));
+        assert_eq!(find(end_hay, b"xx"), Some(0));
+
+        // 1KB haystack with needle near the end.
+        let mut big = vec![b'z'; 1024];
+        big[1000..1010].copy_from_slice(b"hElLo_WoRl");
+        assert_eq!(find(&big, b"hello_wo"), Some(1000));
+        assert_eq!(find(&big, b"hello_world"), None);
+    }
+
+    #[test]
+    fn rare_pair_selection() {
+        let (i1, i2) = select_rare_pair(b"nomore");
+        let ranks: Vec<u8> = b"nomore".iter().map(|&b| rank(b)).collect();
+        let (r1, r2) = (ranks[i1], ranks[i2]);
+        for (i, &r) in ranks.iter().enumerate() {
+            if i != i1 && i != i2 {
+                assert!(r1 <= r || r2 <= r, "pair ({i1},{i2}) not optimal");
+            }
+        }
+    }
+}

crates/fff-core/src/simd_string_utils/mod.rs

@@ -0,0 +1,5 @@
+//! SIMD-accelerated string utilities: case flipping/folding and
+//! case-insensitive substring search.
+
+pub mod case;
+pub mod memmem;

crates/fff-core/src/sort_buffer.rs

@@ -1,43 +1,56 @@
-//! Thread-local sort buffer management for glidesort optimization
-//!
-//! This module provides thread-local buffers for glidesort's with_buffer API,
-//! eliminating allocations in the hot path of fuzzy search operations.
-
-use std::cell::RefCell;
+use parking_lot::Mutex;
 use std::mem::MaybeUninit;
 
-// glidesort requires a buffer to allocate, we use one reused buffer as it can grow pretty big
-// for a large projects, this effectively saves 12kb of allocation on every search in linux repo
-thread_local! {
-    static SORT_BUFFER: RefCell<Vec<u8>> = RefCell::new(Vec::with_capacity(1024));
+// this originally happen to be in TLS but there is a limit of TLS
+// + the storage itself is not free, so now we rely on the fact that most calls
+// are sequential in practice and allocate ONLY when we have a parallel access
+static SORT_BUFFER: Mutex<Vec<u8>> = Mutex::new(Vec::new());
+
+fn ensure_capacity(buf: &mut Vec<u8>, required: usize) {
+    if buf.capacity() < required {
+        let len = buf.len();
+        buf.reserve(required - len);
+    }
+}
+
+struct SharedSortBuf {
+    guard: parking_lot::MutexGuard<'static, Vec<u8>>,
+}
+
+impl SharedSortBuf {
+    fn as_slice_mut<T>(&mut self, len: usize) -> &mut [MaybeUninit<T>] {
+        let align = std::mem::align_of::<MaybeUninit<T>>();
+        let size = std::mem::size_of::<MaybeUninit<T>>();
+        let required = len.saturating_mul(size).saturating_add(align);
+        ensure_capacity(&mut self.guard, required);
+
+        // SAFETY: the Vec<u8> is only 1-byte aligned, so we over-allocate by
+        // `align` bytes and shift the pointer to satisfy T's alignment.
+        // Callers never read uninitialised data through the returned slice.
+        unsafe {
+            let ptr = self.guard.as_mut_ptr();
+            let offset = ptr.align_offset(align);
+            debug_assert!(offset != usize::MAX && offset + len * size <= self.guard.capacity());
+            std::slice::from_raw_parts_mut(ptr.add(offset) as *mut MaybeUninit<T>, len)
+        }
+    }
+}
+
+fn try_lock_shared_buf() -> Option<SharedSortBuf> {
+    SORT_BUFFER.try_lock().map(|guard| SharedSortBuf { guard })
 }
 
 pub fn sort_with_buffer<T, F>(slice: &mut [T], compare: F)
 where
     F: FnMut(&T, &T) -> std::cmp::Ordering,
 {
-    SORT_BUFFER.with(|buffer| {
-        let mut buffer = buffer.borrow_mut();
-
-        // Calculate required buffer size in u8 units
-        let size_of_t = std::mem::size_of::<MaybeUninit<T>>();
-        let size_of_usize = std::mem::size_of::<u8>();
-        let required_usizes = (slice.len() * size_of_t).div_ceil(size_of_usize);
-
-        // Ensure buffer has enough capacity
-        if buffer.len() < required_usizes {
-            buffer.resize(required_usizes, 0);
+    match try_lock_shared_buf() {
+        Some(mut buf) => {
+            let typed = buf.as_slice_mut::<T>(slice.len());
+            glidesort::sort_with_buffer_by(slice, typed, compare);
         }
-
-        // Cast u8 buffer to MaybeUninit<T> slice
-        // SAFETY: u8 provides sufficient alignment for most types, and we've ensured
-        // the buffer is large enough
-        let typed_buffer = unsafe {
-            std::slice::from_raw_parts_mut(buffer.as_mut_ptr() as *mut MaybeUninit<T>, slice.len())
-        };
-
-        glidesort::sort_with_buffer_by(slice, typed_buffer, compare);
-    });
+        None => glidesort::sort_by(slice, compare),
+    }
 }
 
 pub fn sort_by_key_with_buffer<T, K, F>(slice: &mut [T], key_fn: F)
@@ -45,28 +58,13 @@ where
     K: Ord,
     F: FnMut(&T) -> K,
 {
-    SORT_BUFFER.with(|buffer| {
-        let mut buffer = buffer.borrow_mut();
-
-        // Calculate required buffer size in u8 units
-        let size_of_t = std::mem::size_of::<MaybeUninit<T>>();
-        let size_of_usize = std::mem::size_of::<u8>();
-        let required_usizes = (slice.len() * size_of_t).div_ceil(size_of_usize);
-
-        // Ensure buffer has enough capacity
-        if buffer.len() < required_usizes {
-            buffer.resize(required_usizes, 0);
+    match try_lock_shared_buf() {
+        Some(mut buf) => {
+            let typed = buf.as_slice_mut::<T>(slice.len());
+            glidesort::sort_with_buffer_by_key(slice, typed, key_fn);
         }
-
-        // Cast u8 buffer to MaybeUninit<T> slice
-        // SAFETY: u8 provides sufficient alignment for most types, and we've ensured
-        // the buffer is large enough
-        let typed_buffer = unsafe {
-            std::slice::from_raw_parts_mut(buffer.as_mut_ptr() as *mut MaybeUninit<T>, slice.len())
-        };
-
-        glidesort::sort_with_buffer_by_key(slice, typed_buffer, key_fn);
-    });
+        None => glidesort::sort_by_key(slice, key_fn),
+    }
 }
 
 #[cfg(test)]
@@ -82,9 +80,9 @@ mod tests {
 
     #[test]
     fn test_sort_by_key_with_buffer() {
-        let mut data = vec![(2, "b"), (1, "a"), (3, "c")];
-        sort_by_key_with_buffer(&mut data, |item| item.0);
-        assert_eq!(data, vec![(1, "a"), (2, "b"), (3, "c")]);
+        let mut data = vec![(1, 50), (2, 20), (3, 80), (4, 10), (5, 90)];
+        sort_by_key_with_buffer(&mut data, |a| a.1);
+        assert_eq!(data, vec![(4, 10), (2, 20), (1, 50), (3, 80), (5, 90)]);
     }
 
     #[test]
@@ -94,19 +92,6 @@ mod tests {
         assert_eq!(data, vec![5, 4, 3, 2, 1]);
     }
 
-    #[test]
-    fn test_multiple_sorts_reuse_buffer() {
-        // This test verifies that multiple sorts on the same thread reuse the buffer
-        let mut data1 = vec![5, 2, 8, 1, 9];
-        sort_with_buffer(&mut data1, |a, b| a.cmp(b));
-
-        let mut data2 = vec![15, 12, 18, 11, 19];
-        sort_with_buffer(&mut data2, |a, b| a.cmp(b));
-
-        assert_eq!(data1, vec![1, 2, 5, 8, 9]);
-        assert_eq!(data2, vec![11, 12, 15, 18, 19]);
-    }
-
     #[test]
     fn test_empty_slice() {
         let mut data: Vec<i32> = vec![];
@@ -121,13 +106,6 @@ mod tests {
         assert_eq!(data, vec![42]);
     }
 
-    #[test]
-    fn test_already_sorted() {
-        let mut data = vec![1, 2, 3, 4, 5];
-        sort_with_buffer(&mut data, |a, b| a.cmp(b));
-        assert_eq!(data, vec![1, 2, 3, 4, 5]);
-    }
-
     #[test]
     fn test_with_duplicates() {
         let mut data = vec![3, 1, 4, 1, 5, 9, 2, 6, 5];
@@ -144,11 +122,10 @@ mod tests {
 
     #[test]
     fn test_simple_descending() {
-        // Simple test to verify highest scores come first
         let mut data = vec![100, 300, 200];
         sort_with_buffer(&mut data, |a, b| b.cmp(a));
-        assert_eq!(data[0], 300, "Highest should be first");
-        assert_eq!(data[1], 200, "Middle should be second");
-        assert_eq!(data[2], 100, "Lowest should be last");
+        assert_eq!(data[0], 300);
+        assert_eq!(data[1], 200);
+        assert_eq!(data[2], 100);
     }
 }

crates/fff-core/src/stable_vec.rs

@@ -0,0 +1,177 @@
+use std::alloc::{self, Layout};
+use std::ptr::NonNull;
+use std::sync::Arc;
+use std::sync::atomic::{AtomicUsize, Ordering};
+
+/// Vector that guarantees no re-alloc happening at runtime
+pub(crate) struct StableVec<T> {
+    inner: Arc<StableBuf<T>>,
+}
+
+struct StableBuf<T> {
+    ptr: NonNull<T>,
+    cap: usize,
+    /// Atomic because:
+    ///   1. `push(&self)` must mutate this through a shared `&StableBuf`,
+    ///      which requires interior mutability.
+    ///   2. Arc clones (e.g. post-scan snapshots) read `len` outside the
+    ///      picker lock, concurrent with an appending writer. Acquire/Release
+    ///      on len is what makes "observed len ⇒ element bytes initialized"
+    ///      actually hold.
+    ///
+    /// Arc wrapping only shares ownership of the buffer; it does NOT
+    /// synchronize access to fields inside the shared buffer.
+    len: AtomicUsize,
+}
+
+// SAFETY: StableBuf is a thread-safe container when T is send + sync
+// There is another application level constraint: mutations are safe
+// when they are atomic updates, not read + update.
+unsafe impl<T: Send> Send for StableBuf<T> {}
+unsafe impl<T: Sync> Sync for StableBuf<T> {}
+
+impl<T> Drop for StableBuf<T> {
+    fn drop(&mut self) {
+        let len = *self.len.get_mut();
+        unsafe {
+            std::ptr::drop_in_place(std::ptr::slice_from_raw_parts_mut(self.ptr.as_ptr(), len));
+            if self.cap > 0 {
+                let layout = Layout::array::<T>(self.cap).expect("layout");
+                alloc::dealloc(self.ptr.as_ptr().cast(), layout);
+            }
+        }
+    }
+}
+
+impl<T> StableVec<T> {
+    pub fn from_vec_with_reserve(mut vec: Vec<T>, extra: usize) -> Self {
+        vec.reserve(extra);
+        let cap = vec.capacity();
+        let len = vec.len();
+
+        let inner = if cap == 0 {
+            StableBuf {
+                ptr: NonNull::dangling(),
+                cap: 0,
+                len: AtomicUsize::new(0),
+            }
+        } else {
+            // Take ownership of the Vec's buffer without running element
+            // drops; we hand them off to the StableBuf.
+            let mut vec = std::mem::ManuallyDrop::new(vec);
+            let ptr = NonNull::new(vec.as_mut_ptr()).expect("non-null");
+            StableBuf {
+                ptr,
+                cap,
+                len: AtomicUsize::new(len),
+            }
+        };
+
+        Self {
+            inner: Arc::new(inner),
+        }
+    }
+
+    /// Append. Returns `false` if capacity is exhausted (item dropped).
+    ///
+    /// Safe to call via `&self` as long as the caller holds the outer
+    /// picker write lock (single-writer invariant).
+    #[inline]
+    pub fn push(&self, item: T) -> bool {
+        let cap = self.inner.cap;
+        let len = self.inner.len.load(Ordering::Acquire);
+        if len >= cap {
+            debug_assert!(
+                false,
+                "StableVec: push would exceed capacity ({len} at capacity {cap})"
+            );
+            tracing::error!(
+                len,
+                capacity = cap,
+                "StableVec: capacity exhausted — dropping item to prevent reallocation"
+            );
+            return false;
+        }
+        unsafe {
+            std::ptr::write(self.inner.ptr.as_ptr().add(len), item);
+        }
+        self.inner.len.store(len + 1, Ordering::Release);
+        true
+    }
+
+    // this method is specifically private because you probably need to use
+    // live_count if you are trying to access this method
+    #[inline]
+    pub fn len(&self) -> usize {
+        self.inner.len.load(Ordering::Acquire)
+    }
+
+    /// Mutable element access for in-place field updates. Never shifts.
+    ///
+    /// LATENT UB: produces `&mut T` aliasing Arc-shared memory; the
+    /// `&mut self` on StableVec does NOT imply unique access to the
+    /// `StableBuf` when sibling Arc clones exist. Safe in practice
+    /// because callers hold the picker write lock and writes target
+    /// disjoint fields, but strictly forbidden by the aliasing model.
+    #[inline]
+    pub fn get_mut(&mut self, index: usize) -> Option<&mut T> {
+        let len = self.inner.len.load(Ordering::Acquire);
+        if index >= len {
+            return None;
+        }
+        unsafe { Some(&mut *self.inner.ptr.as_ptr().add(index)) }
+    }
+
+    #[inline]
+    pub fn last(&self) -> Option<&T> {
+        let len = self.len();
+        if len == 0 {
+            None
+        } else {
+            unsafe { Some(&*self.inner.ptr.as_ptr().add(len - 1)) }
+        }
+    }
+
+    /// Iterate mutably for in-place field updates. Never shifts storage.
+    /// Same latent-UB caveat as [`get_mut`]: `&mut T` into Arc-shared memory.
+    #[inline]
+    pub fn iter_mut(&mut self) -> std::slice::IterMut<'_, T> {
+        let len = self.inner.len.load(Ordering::Acquire);
+        unsafe { std::slice::from_raw_parts_mut(self.inner.ptr.as_ptr(), len).iter_mut() }
+    }
+}
+
+impl<T> Clone for StableVec<T> {
+    #[inline]
+    fn clone(&self) -> Self {
+        Self {
+            inner: Arc::clone(&self.inner),
+        }
+    }
+}
+
+impl<T: std::fmt::Debug> std::fmt::Debug for StableVec<T> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_tuple("StableVec").field(&self.len()).finish()
+    }
+}
+
+impl<T> std::ops::Deref for StableVec<T> {
+    type Target = [T];
+    #[inline]
+    fn deref(&self) -> &[T] {
+        let len = self.len();
+        unsafe { std::slice::from_raw_parts(self.inner.ptr.as_ptr(), len) }
+    }
+}
+
+impl<T> std::ops::DerefMut for StableVec<T> {
+    /// LATENT UB: `&mut [T]` aliases Arc-shared memory. Kept for
+    /// Index/IndexMut ergonomics at call sites that write disjoint
+    /// fields under the picker write lock. See module-level doc.
+    #[inline]
+    fn deref_mut(&mut self) -> &mut [T] {
+        let len = self.inner.len.load(Ordering::Acquire);
+        unsafe { std::slice::from_raw_parts_mut(self.inner.ptr.as_ptr(), len) }
+    }
+}

crates/fff-core/src/walk/mod.rs

@@ -0,0 +1,148 @@
+//! Filesystem traversal backend. Selects one implementation at compile time:
+//! - `zlob`: zlob's native parallel walker (requires the Zig toolchain).
+//! - `ripgrep`: the `ignore` crate (ripgrep's walker), used by default.
+//!
+//! Both expose [`walk_collect_files`] with identical semantics so the rest of
+//! the crate stays backend-agnostic.
+
+use crate::types::FileItem;
+use std::path::Path;
+
+#[cfg(feature = "zlob")]
+mod zlob;
+#[cfg(feature = "zlob")]
+pub(crate) use zlob::walk_collect_files;
+
+#[cfg(all(not(feature = "zlob"), feature = "ripgrep"))]
+mod ripgrep;
+#[cfg(all(not(feature = "zlob"), feature = "ripgrep"))]
+pub(crate) use ripgrep::walk_collect_files;
+
+pub(crate) struct WalkOutput {
+    pub(crate) pairs: Vec<(FileItem, String)>,
+    /// Every non-ignored directory the walk visited, relative, ending with /
+    pub(crate) dirs: Vec<String>,
+    pub(crate) ignore_rules: Option<WalkIgnoreRules>,
+}
+
+pub(crate) struct WalkIgnoreRules {
+    #[cfg(feature = "zlob")]
+    inner: ::zlob::walk::WalkerOutcomeRules,
+    #[cfg(not(feature = "zlob"))]
+    _never: std::convert::Infallible,
+}
+
+// SAFETY: the underlying storage is immutable, heap-owned, and thread-safe to
+// read from concurrently (mirrors zlob's `IgnoreRules: Send + Sync`).
+unsafe impl Send for WalkIgnoreRules {}
+unsafe impl Sync for WalkIgnoreRules {}
+
+impl std::fmt::Debug for WalkIgnoreRules {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.write_str("WalkIgnoreRules")
+    }
+}
+
+// In ripgrep builds `WalkIgnoreRules` is never constructed (the `_never`
+// field is uninhabited), so its methods are legitimately dead there.
+#[cfg_attr(not(feature = "zlob"), allow(dead_code))]
+impl WalkIgnoreRules {
+    /// Returns `true` if the provided path is ignored by the collected rule set
+    ///
+    /// `relative_path` has to be relative to the walker's provided base path
+    pub(crate) fn is_ignored(&self, relative_path: &Path) -> bool {
+        #[cfg(feature = "zlob")]
+        {
+            self.inner
+                .rules()
+                .is_some_and(|rules| rules.is_ignored(relative_path))
+        }
+        #[cfg(not(feature = "zlob"))]
+        {
+            let _ = relative_path;
+            match self._never {}
+        }
+    }
+
+    // The old `is_ignored_untrusted` variant was folded away when zlob's
+    // ignore matcher moved to full ancestor enumeration — trailing-slash
+    // sniffing on the input is now sufficient for external queries.
+}
+
+#[cfg(test)]
+mod tests {
+    use super::walk_collect_files;
+    use std::fs;
+    use std::sync::Arc;
+    use std::sync::atomic::{AtomicUsize, Ordering};
+
+    // Backend-agnostic parity check: both the zlob and ripgrep walkers must
+    // respect .gitignore, skip hidden files in a git repo, and surface the
+    // expected file set with a correct synced count.
+    #[test]
+    fn collects_files_respecting_gitignore() {
+        let dir = tempfile::tempdir().unwrap();
+        let root = dir.path();
+        fs::create_dir(root.join(".git")).unwrap();
+        fs::create_dir(root.join("src")).unwrap();
+        fs::create_dir(root.join("target")).unwrap();
+        fs::write(root.join(".gitignore"), "target/\n*.log\n").unwrap();
+        fs::write(root.join("Cargo.toml"), "x").unwrap();
+        fs::write(root.join("debug.log"), "").unwrap();
+        fs::write(root.join("src/main.rs"), "fn main() {}").unwrap();
+        fs::write(root.join("target/out.bin"), "bin").unwrap();
+
+        let counter = Arc::new(AtomicUsize::new(0));
+        let out = walk_collect_files(root, true, false, 1, &counter).unwrap();
+
+        let mut names: Vec<String> = out.pairs.into_iter().map(|(_, rel)| rel).collect();
+        names.sort();
+
+        assert!(names.contains(&"Cargo.toml".to_string()));
+        assert!(names.iter().any(|n| n.ends_with("main.rs")));
+        // target/ and *.log are gitignored; .git/ is skipped.
+        assert!(!names.iter().any(|n| n.contains("target")));
+        assert!(!names.iter().any(|n| n.ends_with(".log")));
+        assert!(!names.iter().any(|n| n.contains(".git/")));
+        assert_eq!(counter.load(Ordering::Relaxed), names.len());
+    }
+
+    // Non-git roots prune known non-code directories (node_modules).
+    #[test]
+    fn prunes_non_code_dirs_for_non_git_root() {
+        let dir = tempfile::tempdir().unwrap();
+        let root = dir.path();
+        fs::create_dir(root.join("node_modules")).unwrap();
+        fs::write(root.join("node_modules/lib.js"), "x").unwrap();
+        fs::write(root.join("index.js"), "x").unwrap();
+
+        let counter = Arc::new(AtomicUsize::new(0));
+        let out = walk_collect_files(root, false, false, 1, &counter).unwrap();
+        let names: Vec<String> = out.pairs.into_iter().map(|(_, rel)| rel).collect();
+
+        assert!(names.iter().any(|n| n.ends_with("index.js")));
+        assert!(!names.iter().any(|n| n.contains("node_modules")));
+    }
+
+    // Only the zlob backend surfaces reusable ignore rules; they must match
+    // the same tree the walk respected.
+    #[cfg(feature = "zlob")]
+    #[test]
+    fn surfaces_reusable_ignore_rules() {
+        use std::path::Path;
+
+        let dir = tempfile::tempdir().unwrap();
+        let root = dir.path();
+        fs::create_dir(root.join(".git")).unwrap();
+        fs::write(root.join(".gitignore"), "target/\n*.log\n").unwrap();
+        fs::write(root.join("Cargo.toml"), "x").unwrap();
+
+        let counter = Arc::new(AtomicUsize::new(0));
+        let out = walk_collect_files(root, true, false, 1, &counter).unwrap();
+
+        let rules = out.ignore_rules.expect("zlob surfaces ignore rules");
+        assert!(rules.is_ignored(Path::new("target/")));
+        assert!(rules.is_ignored(Path::new("debug.log")));
+        assert!(!rules.is_ignored(Path::new("Cargo.toml")));
+    }
+}

crates/fff-core/src/walk/ripgrep.rs

@@ -0,0 +1,87 @@
+use crate::ignore::non_git_repo_overrides;
+use crate::types::FileItem;
+use crate::walk::WalkOutput;
+use crate::watch::is_git_file;
+use ignore::WalkBuilder;
+use std::path::Path;
+use std::sync::{
+    Arc,
+    atomic::{AtomicUsize, Ordering},
+};
+
+#[tracing::instrument(skip_all, name = "ripgrep walker", level = "info")]
+pub(crate) fn walk_collect_files(
+    base_path: &Path,
+    is_git_repo: bool,
+    follow_symlinks: bool,
+    threads: usize,
+    synced_files_count: &Arc<AtomicUsize>,
+) -> crate::Result<WalkOutput> {
+    let mut walk_builder = WalkBuilder::new(base_path);
+    walk_builder
+        // this is a very important guard for the user opening ~/ or other root non-git dir
+        .hidden(!is_git_repo)
+        .git_ignore(true)
+        .git_exclude(true)
+        .git_global(true)
+        .ignore(true)
+        .follow_links(follow_symlinks)
+        .threads(threads);
+
+    if !is_git_repo && let Some(overrides) = non_git_repo_overrides(base_path) {
+        walk_builder.overrides(overrides);
+    }
+
+    let walker = walk_builder.build_parallel();
+
+    // Single lock for both collections: every entry is either a file or a
+    // dir, so this keeps one mutex acquisition per entry.
+    let collected =
+        parking_lot::Mutex::new((Vec::<(FileItem, String)>::new(), Vec::<String>::new()));
+    walker.run(|| {
+        let collected = &collected;
+        let counter = Arc::clone(synced_files_count);
+        let base_path = base_path.to_path_buf();
+
+        Box::new(move |result| {
+            let Ok(entry) = result else {
+                return ignore::WalkState::Continue;
+            };
+
+            if entry.file_type().is_some_and(|ft| ft.is_file()) {
+                let path = entry.path();
+
+                // Ignore walkers sometimes surface files inside `.git/`
+                // when the base is itself a git repo — skip them.
+                if is_git_file(path) {
+                    return ignore::WalkState::Continue;
+                }
+
+                let metadata = entry.metadata().ok();
+                let (file_item, rel_path) =
+                    FileItem::new_from_walk(path, &base_path, None, metadata.as_ref());
+
+                collected.lock().0.push((file_item, rel_path));
+                counter.fetch_add(1, Ordering::Relaxed);
+            } else if entry.depth() > 0 && entry.file_type().is_some_and(|ft| ft.is_dir()) {
+                let path = entry.path();
+                if !is_git_file(path)
+                    && let Ok(rel) = path.strip_prefix(&base_path)
+                {
+                    let mut rel = crate::path_utils::to_canonical_slashes(&rel.to_string_lossy())
+                        .into_owned();
+                    rel.push('/');
+                    collected.lock().1.push(rel);
+                }
+            }
+            ignore::WalkState::Continue
+        })
+    });
+
+    let (pairs, dirs) = collected.into_inner();
+    Ok(WalkOutput {
+        pairs,
+        dirs,
+        ignore_rules: None,
+    })
+}

crates/fff-core/src/walk/zlob.rs

@@ -0,0 +1,123 @@
+use crate::file_picker::is_known_binary_extension_basename;
+use crate::ignore::IGNORED_DIRS;
+use crate::types::FileItem;
+use crate::walk::{WalkIgnoreRules, WalkOutput};
+use parking_lot::Mutex;
+use std::path::Path;
+use std::sync::Arc;
+use std::sync::atomic::{AtomicUsize, Ordering};
+use zlob::walk::{WalkBuilder, WalkFlags, WalkMetadata, WalkState};
+
+const PROGRESS_STEP: usize = 13;
+
+#[tracing::instrument(skip_all, name = "zlob walker", level = "info")]
+pub(crate) fn walk_collect_files(
+    base_path: &Path,
+    is_git_repo: bool,
+    follow_symlinks: bool,
+    threads: usize,
+    synced_files_count: &Arc<AtomicUsize>,
+) -> crate::Result<WalkOutput> {
+    // gitignore on; skip hidden on non-git roots (so `~/` doesn't recurse into
+    // ~/.cache, ~/.config, etc.); optionally follow symlinks.
+    let mut flags = WalkFlags::GITIGNORE;
+    if !is_git_repo {
+        flags |= WalkFlags::SKIP_HIDDEN;
+    }
+    if follow_symlinks {
+        flags |= WalkFlags::FOLLOW_SYMLINKS;
+    }
+
+    let mut builder = WalkBuilder::new(base_path)
+        .map_err(|e| crate::Error::WalkFailed(format!("WalkBuilder::new: {e:?}")))?;
+    builder
+        .options(flags)
+        .threads(threads)
+        // Bulk-fetch the only metadata FileItem needs; zlob never stats more.
+        .metadata(WalkMetadata::SIZE | WalkMetadata::MTIME);
+
+    if !is_git_repo
+        && !IGNORED_DIRS.is_empty()
+        && let Err(e) = builder.extra_ignore(IGNORED_DIRS)
+    {
+        // Interior NUL in one of the extra_ignore patterns would fail
+        // here — treat as if no extras were supplied rather than
+        // aborting the whole walk.
+        tracing::warn!(?e, "zlob extra_ignore rejected; walking without it");
+    }
+
+    // Single lock for both collections: every entry is either a file or a
+    // dir, so this keeps one mutex acquisition per entry.
+    let collected = Mutex::new((Vec::new(), Vec::new()));
+
+    let outcome = match builder.run(|entry| {
+        if !entry.is_file() {
+            // unlike ripgrep walker zlob doesnt show .git files
+            if entry.is_dir() {
+                let rel_bytes = entry.relative_path_bytes();
+                if !rel_bytes.is_empty() {
+                    let mut rel = String::from_utf8_lossy(rel_bytes).into_owned();
+                    rel.push('/');
+                    collected.lock().1.push(rel);
+                }
+            }
+
+            return WalkState::Continue;
+        }
+
+        let rel_bytes = entry.relative_path_bytes();
+
+        // `basename()` returns `&str` for files only.
+        let basename = entry.basename().unwrap_or("");
+        let is_binary = is_known_binary_extension_basename(basename);
+
+        let size = entry.size().unwrap_or(0);
+        // zlob reports mtime in ns since the Unix epoch; FileItem wants secs.
+        let modified = entry
+            .modified_ns()
+            .map(|ns| (ns / 1_000_000_000).max(0) as u64)
+            .unwrap_or(0);
+
+        let basename_offset = entry.basename_offset_in_relative();
+        // zlob emits '/'-separated relative paths, which is fff's canonical
+        // internal form on every platform — store them verbatim.
+        let rel_str = String::from_utf8_lossy(rel_bytes).into_owned();
+        let item = FileItem::new_raw(basename_offset, size, modified, None, is_binary);
+
+        let mut guard = collected.lock();
+        guard.0.push((item, rel_str));
+        let n = guard.0.len();
+        drop(guard);
+
+        if n % PROGRESS_STEP == 0 {
+            synced_files_count.store(n, Ordering::Relaxed);
+        }
+
+        WalkState::Continue
+    }) {
+        Ok(outcome) => outcome,
+        Err(e) => {
+            // Preserve whatever we collected before the failure so the caller
+            // can still surface a partial index instead of nothing.
+            tracing::error!(?e, "zlob walk failed");
+            return Err(crate::Error::WalkFailed(format!("{e:?}")));
+        }
+    };
+
+    let (pairs, dirs) = collected.into_inner();
+    // Always report the exact final total regardless of the last step.
+    synced_files_count.store(pairs.len(), Ordering::Relaxed);
+
+    // Retain the ignore rules only when the walk actually gathered some
+    // (git roots with .gitignore/.ignore). Otherwise callers fall back.
+    let ignore_rules = outcome
+        .rules()
+        .is_some()
+        .then(|| WalkIgnoreRules { inner: outcome });
+
+    Ok(WalkOutput {
+        pairs,
+        dirs,
+        ignore_rules,
+    })
+}

crates/fff-core/src/watcher/mod.rs

@@ -0,0 +1,9 @@
+mod background_watcher;
+pub use background_watcher::*;
+
+mod watch;
+pub use watch::*;
+
+// The harness reads rescan counters, which release builds compile out.
+#[cfg(all(test, rescan_stats))]
+mod rescan_tests;

crates/fff-core/src/watcher/rescan_tests.rs

@@ -0,0 +1,621 @@
+use std::path::PathBuf;
+use std::process::Command;
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+
+use notify::Event;
+use notify::EventKind;
+use notify::event::{
+    AccessKind, AccessMode, CreateKind, DataChange, Flag, ModifyKind, RemoveKind, RenameMode,
+};
+use notify_debouncer_full::DebouncedEvent;
+use tempfile::TempDir;
+
+use super::handle_debounced_events;
+use crate::constants::MAX_OVERFLOW_FILES;
+use crate::file_picker::{FFFMode, FilePicker, FilePickerOptions};
+use crate::git_status_worker::GitStatusWorker;
+use crate::rescan_stats::{RescanReason, RescanStats};
+use crate::shared::{SharedFilePicker, SharedFrecency};
+
+#[test]
+fn saving_an_indexed_file_stays_incremental() {
+    let f = Fixture::new();
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    f.write("src/main.rs", "fn main() { println!(); }");
+    let delta = f.feed([modify(f.path("src/main.rs"))]);
+
+    f.assert_no_rescan(&delta, "saving a tracked file");
+}
+
+#[test]
+fn editor_atomic_save_stays_incremental() {
+    let f = Fixture::new();
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    // write-to-temp + rename-over-target, the way vim/VSCode/IntelliJ save.
+    f.write("src/main.rs", "fn main() { println!(); }");
+    let target = f.path("src/main.rs");
+    let temp = f.path("src/.main.rs.swp");
+    let delta = f.feed([
+        DebouncedEvent::new(
+            Event::new(EventKind::Create(CreateKind::File)).add_path(temp.clone()),
+            Instant::now(),
+        ),
+        DebouncedEvent::new(
+            Event::new(EventKind::Modify(ModifyKind::Name(RenameMode::From)))
+                .add_path(temp.clone()),
+            Instant::now(),
+        ),
+        DebouncedEvent::new(
+            Event::new(EventKind::Modify(ModifyKind::Name(RenameMode::To)))
+                .add_path(target.clone()),
+            Instant::now(),
+        ),
+        DebouncedEvent::new(
+            Event::new(EventKind::Remove(RemoveKind::File)).add_path(temp),
+            Instant::now(),
+        ),
+    ]);
+
+    f.assert_no_rescan(&delta, "an atomic editor save");
+    assert!(f.is_indexed("src/main.rs"), "target must stay indexed");
+}
+
+#[test]
+fn creating_and_deleting_files_stays_incremental() {
+    let f = Fixture::new();
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    f.write("src/added.rs", "pub fn added() {}");
+    let created = f.feed([create(f.path("src/added.rs"))]);
+    f.assert_no_rescan(&created, "creating a file");
+    assert!(f.is_indexed("src/added.rs"));
+
+    f.remove("src/added.rs");
+    let removed = f.feed([remove_file(f.path("src/added.rs"))]);
+    f.assert_no_rescan(&removed, "deleting a file");
+    assert!(!f.is_indexed("src/added.rs"));
+}
+
+#[test]
+fn deleting_a_directory_stays_incremental() {
+    let f = Fixture::new();
+    f.write("src/main.rs", "fn main() {}");
+    f.write("src/nested/a.rs", "");
+    f.write("src/nested/b.rs", "");
+    f.index();
+
+    std::fs::remove_dir_all(f.path("src/nested")).unwrap();
+    let delta = f.feed([DebouncedEvent::new(
+        Event::new(EventKind::Remove(RemoveKind::Folder)).add_path(f.path("src/nested")),
+        Instant::now(),
+    )]);
+
+    f.assert_no_rescan(&delta, "deleting a directory");
+    assert!(!f.is_indexed("src/nested/a.rs"));
+    assert!(f.is_indexed("src/main.rs"));
+}
+
+#[test]
+fn read_only_access_events_are_ignored() {
+    let f = Fixture::new();
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    // fff's own preview + grep reads generate these; reacting to them would
+    // make the picker rescan whenever the user scrolls the result list.
+    let path = f.path("src/main.rs");
+    let delta = f.feed([
+        DebouncedEvent::new(
+            Event::new(EventKind::Access(AccessKind::Read)).add_path(path.clone()),
+            Instant::now(),
+        ),
+        DebouncedEvent::new(
+            Event::new(EventKind::Access(AccessKind::Open(AccessMode::Read)))
+                .add_path(path.clone()),
+            Instant::now(),
+        ),
+        DebouncedEvent::new(
+            Event::new(EventKind::Access(AccessKind::Close(AccessMode::Read))).add_path(path),
+            Instant::now(),
+        ),
+    ]);
+
+    f.assert_no_rescan(&delta, "read-only access events");
+}
+
+#[test]
+fn recreating_the_same_paths_does_not_consume_overflow_capacity() {
+    let f = Fixture::new();
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    // Recreated paths must reuse their overflow slots.
+    for _ in 0..8 {
+        for i in 0..200 {
+            let rel = format!("gen/out{i}.rs");
+            f.write(&rel, "generated");
+            f.feed([create(f.path(&rel))]);
+        }
+        for i in 0..200 {
+            let rel = format!("gen/out{i}.rs");
+            f.remove(&rel);
+            f.feed([remove_file(f.path(&rel))]);
+        }
+    }
+
+    let delta = f.all_rescans();
+    f.assert_no_rescan(&delta, "1600 create/delete cycles over 200 stable paths");
+    assert!(
+        f.overflow_len() <= 200,
+        "each path must claim one overflow slot at most, got {}",
+        f.overflow_len()
+    );
+}
+
+#[test]
+fn writes_inside_a_gitignored_directory_stay_incremental() {
+    let f = Fixture::with_git();
+    f.write(".gitignore", "target/\nnode_modules/\n");
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    let mut events = Vec::new();
+    for i in 0..64 {
+        let rel = format!("target/debug/artifact{i}.o");
+        f.write(&rel, "binary");
+        events.push(create(f.path(&rel)));
+    }
+
+    let delta = f.feed(events);
+    f.assert_no_rescan(&delta, "build output written into an ignored directory");
+}
+
+#[test]
+fn ignored_event_batch_above_index_capacity_stays_incremental() {
+    let f = Fixture::with_git();
+    f.write(".gitignore", "node_modules/\n");
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    let events = (0..MAX_OVERFLOW_FILES + 1)
+        .map(|i| {
+            let rel = format!("node_modules/pkg/file{i}.js");
+            f.write(&rel, "");
+            create(f.path(&rel))
+        })
+        .collect::<Vec<_>>();
+
+    let delta = f.feed(events);
+    f.assert_no_rescan(&delta, "ignored events above the index capacity");
+    assert_eq!(f.overflow_len(), 0);
+}
+
+#[test]
+fn repeated_edits_above_index_capacity_stay_incremental() {
+    let f = Fixture::new();
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    let path = f.path("src/main.rs");
+    let events = (0..MAX_OVERFLOW_FILES + 1)
+        .map(|_| modify(path.clone()))
+        .collect::<Vec<_>>();
+
+    let delta = f.feed(events);
+    f.assert_no_rescan(&delta, "repeated edits above the index capacity");
+    assert_eq!(f.overflow_len(), 0);
+}
+
+#[test]
+fn ignore_file_inside_an_ignored_directory_stays_incremental() {
+    let f = Fixture::with_git();
+    f.write(".gitignore", "node_modules/\n");
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    let ignore_files =
+        ["left-pad", "lodash", "typescript"].map(|pkg| format!("node_modules/{pkg}/.gitignore"));
+
+    for rel in &ignore_files {
+        f.write(rel, "dist\n");
+    }
+    let delta = f.feed(ignore_files.iter().map(|rel| create(f.path(rel))));
+    f.assert_no_rescan(&delta, "creating ignored .gitignore files");
+
+    for rel in &ignore_files {
+        f.write(rel, "build\n");
+    }
+    let delta = f.feed(ignore_files.iter().map(|rel| modify(f.path(rel))));
+    f.assert_no_rescan(&delta, "modifying ignored .gitignore files");
+
+    for rel in &ignore_files {
+        f.remove(rel);
+    }
+    let delta = f.feed(ignore_files.iter().map(|rel| remove_file(f.path(rel))));
+    f.assert_no_rescan(&delta, "removing ignored .gitignore files");
+}
+
+#[test]
+fn ignore_file_inside_an_indexed_directory_triggers_a_rescan() {
+    let f = Fixture::with_git();
+    f.write("src/.gitignore", ".gitignore\ngenerated/\n");
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    f.write("src/.gitignore", ".gitignore\ngenerated/\nbuild/\n");
+    let delta = f.feed([modify(f.path("src/.gitignore"))]);
+
+    assert_eq!(delta.count(RescanReason::IgnoreFileChanged), 1);
+}
+
+#[test]
+fn git_internal_churn_stays_incremental() {
+    let f = Fixture::with_git();
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    let git_dir = f.path(".git");
+    let delta = f.feed([
+        create(git_dir.join("index.lock")),
+        modify(git_dir.join("index")),
+        remove_file(git_dir.join("index.lock")),
+        modify(git_dir.join("HEAD")),
+        modify(git_dir.join("logs/HEAD")),
+        modify(git_dir.join("COMMIT_EDITMSG")),
+        modify(git_dir.join("refs/heads/main")),
+    ]);
+
+    f.assert_no_rescan(&delta, "git writing its own metadata");
+}
+
+#[test]
+fn changing_the_root_ignore_file_triggers_a_rescan() {
+    let f = Fixture::with_git();
+    f.write(".gitignore", "target/\n");
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    f.write(".gitignore", "target/\nsrc/\n");
+    let delta = f.feed([modify(f.path(".gitignore"))]);
+
+    assert_eq!(
+        delta.count(RescanReason::IgnoreFileChanged),
+        1,
+        "the indexed set depends on the root ignore rules, got {delta}"
+    );
+}
+
+#[test]
+fn kernel_event_loss_on_a_directory_triggers_a_rescan() {
+    let f = Fixture::new();
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    let delta = f.feed([DebouncedEvent::new(
+        Event::new(EventKind::Modify(ModifyKind::Any))
+            .add_path(f.path("src"))
+            .set_flag(Flag::Rescan),
+        Instant::now(),
+    )]);
+
+    assert_eq!(
+        delta.count(RescanReason::KernelEventLoss),
+        1,
+        "a dropped-events flag over a directory means unknown subtree state, got {delta}"
+    );
+}
+
+#[test]
+fn new_files_above_index_capacity_trigger_a_rescan() {
+    let f = Fixture::new();
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    let events = (0..MAX_OVERFLOW_FILES + 1)
+        .map(|i| {
+            let rel = format!("src/bulk{i}.rs");
+            f.write(&rel, "");
+            create(f.path(&rel))
+        })
+        .collect::<Vec<_>>();
+
+    let delta = f.feed(events);
+    assert_eq!(
+        delta.count(RescanReason::IndexUpdateRejected),
+        1,
+        "new files above the overflow region cannot be applied incrementally, got {delta}"
+    );
+}
+
+#[test]
+fn batch_at_the_overflow_boundary_stays_incremental() {
+    let f = Fixture::new();
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    let events = (0..MAX_OVERFLOW_FILES)
+        .map(|i| {
+            let rel = format!("src/bulk{i}.rs");
+            f.write(&rel, "");
+            create(f.path(&rel))
+        })
+        .collect::<Vec<_>>();
+
+    let delta = f.feed(events);
+    f.assert_no_rescan(&delta, "a batch exactly at the overflow limit");
+}
+
+#[test]
+fn event_batch_at_four_times_index_capacity_stays_incremental() {
+    let f = Fixture::new();
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    let path = f.path("src/main.rs");
+    let events = (0..MAX_OVERFLOW_FILES * 4)
+        .map(|_| modify(path.clone()))
+        .collect::<Vec<_>>();
+
+    let delta = f.feed(events);
+    f.assert_no_rescan(&delta, "an event batch exactly at the event limit");
+}
+
+#[test]
+fn event_batch_above_four_times_index_capacity_triggers_a_rescan() {
+    let f = Fixture::new();
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    let path = f.path("src/main.rs");
+    let events = (0..MAX_OVERFLOW_FILES * 4 + 1)
+        .map(|_| modify(path.clone()))
+        .collect::<Vec<_>>();
+
+    let delta = f.feed(events);
+    assert_eq!(
+        delta.count(RescanReason::EventBatchOverflow),
+        1,
+        "an event batch above four times the index capacity must rescan, got {delta}"
+    );
+}
+
+#[test]
+fn repeated_triggers_inside_the_cooldown_collapse_to_one_rescan() {
+    let f = Fixture::with_git();
+    f.write(".gitignore", "target/\n");
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    // Repeated batches during the cooldown must share one walk.
+    for round in 0..50 {
+        f.write(".gitignore", &format!("target/\n# round {round}\n"));
+        f.feed([modify(f.path(".gitignore"))]);
+    }
+
+    let stats = f.all_rescans();
+    assert_eq!(
+        stats.total, 1,
+        "50 triggers inside the cooldown must collapse to a single walk, got {stats}"
+    );
+    assert_eq!(
+        stats.throttled, 49,
+        "every suppressed request must be accounted for, got {stats}"
+    );
+}
+
+#[test]
+fn an_explicit_request_is_never_throttled() {
+    let f = Fixture::with_git();
+    f.write(".gitignore", "target/\n");
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    // Burn the cooldown with a watcher trigger, then confirm a user-initiated
+    // refresh still goes through.
+    f.write(".gitignore", "target/\nsrc/\n");
+    f.feed([modify(f.path(".gitignore"))]);
+
+    for _ in 0..3 {
+        f.picker.trigger_full_rescan_async(&f.frecency).unwrap();
+    }
+
+    let stats = f.all_rescans();
+    assert_eq!(
+        stats.count(RescanReason::Explicit),
+        3,
+        "explicit refreshes must bypass the throttle, got {stats}"
+    );
+    assert_eq!(stats.count_throttled(RescanReason::Explicit), 0);
+}
+
+#[test]
+fn events_after_a_suppressed_kernel_rescan_are_still_applied() {
+    let f = Fixture::new();
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    f.write("src/added.rs", "pub fn added() {}");
+    let delta = f.feed([
+        DebouncedEvent::new(
+            Event::new(EventKind::Modify(ModifyKind::Data(DataChange::Content)))
+                .add_path(f.path("src/main.rs"))
+                .set_flag(Flag::Rescan),
+            Instant::now(),
+        ),
+        create(f.path("src/added.rs")),
+    ]);
+
+    f.assert_no_rescan(&delta, "a dropped-events flag over a single tracked file");
+    assert!(
+        f.is_indexed("src/added.rs"),
+        "suppressing the rescan must not drop the rest of the batch"
+    );
+}
+
+#[test]
+fn a_throttled_ignore_file_event_is_still_applied_incrementally() {
+    let f = Fixture::with_git();
+    f.write(".gitignore", "target/\n");
+    f.write("src/main.rs", "fn main() {}");
+    f.index();
+
+    // Burn the cooldown: deleting .gitignore admits a full rescan.
+    f.remove(".gitignore");
+    let delta = f.feed([remove_file(f.path(".gitignore"))]);
+    assert_eq!(delta.count(RescanReason::IgnoreFileChanged), 1);
+    f.picker.wait_for_indexing_complete(Duration::from_secs(10));
+
+    // Recreating it inside the cooldown throttles the rescan, but the file
+    // itself must re-enter the index via the incremental fallback.
+    f.write(".gitignore", "target/\n__ignored_x/\n");
+    let delta = f.feed([create(f.path(".gitignore"))]);
+    assert_eq!(delta.total, 0, "the rescan must be throttled, got {delta}");
+    assert_eq!(delta.count_throttled(RescanReason::IgnoreFileChanged), 1);
+    assert!(
+        f.is_indexed(".gitignore"),
+        "a throttled ignore-file event must still index the file itself"
+    );
+}
+
+struct Fixture {
+    base: PathBuf,
+    picker: SharedFilePicker,
+    frecency: SharedFrecency,
+    git_workdir: Option<PathBuf>,
+    git_worker: Arc<GitStatusWorker>,
+    // Dropped last so background work started by a triggered rescan still
+    // sees the tree it was asked to walk.
+    _tmp: TempDir,
+}
+
+impl Fixture {
+    fn new() -> Self {
+        Self::build(false)
+    }
+
+    fn with_git() -> Self {
+        Self::build(true)
+    }
+
+    fn build(git: bool) -> Self {
+        let tmp = tempfile::tempdir().unwrap();
+        let base = crate::path_utils::canonicalize(tmp.path()).unwrap();
+        let git_workdir = git.then(|| {
+            let status = Command::new("git")
+                .args(["init", "-b", "main"])
+                .current_dir(&base)
+                .output()
+                .expect("git init");
+            assert!(status.status.success(), "git init failed");
+            base.clone()
+        });
+
+        Self {
+            base,
+            picker: SharedFilePicker::default(),
+            frecency: SharedFrecency::noop(),
+            git_workdir,
+            git_worker: GitStatusWorker::new(),
+            _tmp: tmp,
+        }
+    }
+
+    fn index(&self) {
+        let mut picker = FilePicker::new(FilePickerOptions {
+            base_path: self.base.to_string_lossy().into_owned(),
+            watch: false,
+            ..Default::default()
+        })
+        .unwrap();
+        picker.collect_files().unwrap();
+        self.picker.rebase_watches(&self.base);
+        *self.picker.write().unwrap() = Some(picker);
+    }
+
+    fn feed(&self, events: impl IntoIterator<Item = DebouncedEvent>) -> RescanStats {
+        let before = self.picker.rescan_stats();
+        handle_debounced_events(
+            FFFMode::Neovim,
+            events.into_iter().collect(),
+            &self.base,
+            &self.git_workdir,
+            &self.picker,
+            &self.frecency,
+            &self.git_worker,
+        );
+
+        self.picker.rescan_stats().since(&before)
+    }
+
+    fn assert_no_rescan(&self, delta: &RescanStats, what: &str) {
+        assert_eq!(delta.total, 0, "{what} must not trigger a rescan: {delta}");
+    }
+
+    fn path(&self, rel: &str) -> PathBuf {
+        self.base.join(rel)
+    }
+
+    fn write(&self, rel: &str, contents: &str) {
+        let path = self.path(rel);
+        std::fs::create_dir_all(path.parent().unwrap()).unwrap();
+        std::fs::write(path, contents).unwrap();
+    }
+
+    fn remove(&self, rel: &str) {
+        std::fs::remove_file(self.path(rel)).unwrap();
+    }
+
+    fn is_indexed(&self, rel: &str) -> bool {
+        let guard = self.picker.read().unwrap();
+        guard
+            .as_ref()
+            .and_then(|p| p.get_file_by_path(self.path(rel)))
+            .is_some_and(|file| !file.is_deleted())
+    }
+
+    fn all_rescans(&self) -> RescanStats {
+        self.picker.rescan_stats()
+    }
+
+    fn overflow_len(&self) -> usize {
+        let guard = self.picker.read().unwrap();
+        guard
+            .as_ref()
+            .map(|p| p.get_overflow_files().len())
+            .unwrap_or(0)
+    }
+}
+
+impl Drop for Fixture {
+    fn drop(&mut self) {
+        // A test that intentionally triggers a rescan leaves a walk running on
+        // the background pool; let it finish before the tree disappears.
+        self.picker
+            .wait_for_indexing_complete(Duration::from_secs(10));
+    }
+}
+
+fn event(kind: EventKind, path: PathBuf) -> DebouncedEvent {
+    DebouncedEvent::new(Event::new(kind).add_path(path), Instant::now())
+}
+
+fn create(path: PathBuf) -> DebouncedEvent {
+    event(EventKind::Create(CreateKind::File), path)
+}
+
+fn modify(path: PathBuf) -> DebouncedEvent {
+    event(
+        EventKind::Modify(ModifyKind::Data(DataChange::Content)),
+        path,
+    )
+}
+
+fn remove_file(path: PathBuf) -> DebouncedEvent {
+    event(EventKind::Remove(RemoveKind::File), path)
+}